From 58a0401b8758a5ae265a126c5701be788ddd23e3 Mon Sep 17 00:00:00 2001 From: Panakotta00 Date: Fri, 8 Nov 2024 20:38:12 +0100 Subject: [PATCH] feat: Improved Lua Syntax Highlighter --- .../Private/UI/FINLuaCodeEditor.cpp | 388 +----------------- .../FicsItNetworks/Private/UI/FINLuaLexer.cpp | 295 +++++++++++++ .../Private/UI/FINLuaSyntaxHighlighter.cpp | 143 +++++++ .../Public/UI/FINLuaCodeEditor.h | 16 +- .../FicsItNetworks/Public/UI/FINLuaSyntax.h | 123 ++++++ ThirdParty/eris | 2 +- 6 files changed, 567 insertions(+), 400 deletions(-) create mode 100644 Source/FicsItNetworks/Private/UI/FINLuaLexer.cpp create mode 100644 Source/FicsItNetworks/Private/UI/FINLuaSyntaxHighlighter.cpp create mode 100644 Source/FicsItNetworks/Public/UI/FINLuaSyntax.h diff --git a/Source/FicsItNetworks/Private/UI/FINLuaCodeEditor.cpp b/Source/FicsItNetworks/Private/UI/FINLuaCodeEditor.cpp index ad94faef9..b56b48f09 100644 --- a/Source/FicsItNetworks/Private/UI/FINLuaCodeEditor.cpp +++ b/Source/FicsItNetworks/Private/UI/FINLuaCodeEditor.cpp @@ -1,5 +1,6 @@ #include "UI/FINLuaCodeEditor.h" +#include "FINLuaSyntax.h" #include "FINUtils.h" #include "TimerManager.h" #include "Engine/World.h" @@ -152,391 +153,10 @@ int32 FFINTabRun::CalcCharacterTillNextTabStop(const FString& Text, int32 Offset return TabWidth - FillerChars; } -void FFINLuaSyntaxHighlighterTextLayoutMarshaller::SetText(const FString& SourceString, FTextLayout& TargetTextLayout) { - TArray TokenizedLines; - TArray LineRanges; - FTextRange::CalculateLineRangesFromString(SourceString, LineRanges); - - TArray Rules = TArray({ - " ", "\t", "\\.", "\\:", "\\\"", "\\\'", "\\,", "\\\\", "\\(", "\\)", "for", "in", "while", "do", "if", "then", "elseif", "else", - "end", "local", "true", "false", "not", "and", "or", "function", "return", "--\\[\\[", "\\]\\]--", "--", "\\+", "\\-", "\\/", - "\\*", "\\%", "\\[", "\\]", "\\{", "\\}", "\\=", "\\!", "\\~", "\\#", "\\>", "\\<"}); - - FString Pat; - for (const FString& Rule : Rules) { - Pat += FString::Printf(TEXT("(%s)|"), *Rule); - } - if (Rules.Num() > 0) Pat = Pat.LeftChop(1); - FRegexPattern Pattern(Pat); - - for (const FTextRange& LineRange : LineRanges) { - FSyntaxTokenizer::FTokenizedLine TokenizedLine; - TokenizedLine.Range = LineRange; - - FString Line = SourceString.Mid(LineRange.BeginIndex, LineRange.EndIndex - LineRange.BeginIndex); - FRegexMatcher Match(Pattern, Line); - int32 Start = 0; - int32 End = 0; - while (Match.FindNext()) { - int32 MatchStart = Match.GetMatchBeginning(); - End = Match.GetMatchEnding(); - if (MatchStart != Start) { - TokenizedLine.Tokens.Add(FSyntaxTokenizer::FToken(FSyntaxTokenizer::ETokenType::Literal, FTextRange(LineRange.BeginIndex + Start, LineRange.BeginIndex + MatchStart))); - } - Start = End; - TokenizedLine.Tokens.Add(FSyntaxTokenizer::FToken(FSyntaxTokenizer::ETokenType::Syntax, FTextRange(LineRange.BeginIndex + MatchStart, LineRange.BeginIndex + End))); - } - if (End < LineRange.EndIndex - LineRange.BeginIndex || TokenizedLine.Tokens.Num() < 1) { - TokenizedLine.Tokens.Add(FSyntaxTokenizer::FToken(FSyntaxTokenizer::ETokenType::Syntax, FTextRange(LineRange.BeginIndex + End, LineRange.EndIndex))); - } - TokenizedLines.Add(TokenizedLine); - } - - ParseTokens(SourceString, TargetTextLayout, TokenizedLines); -} - -bool FFINLuaSyntaxHighlighterTextLayoutMarshaller::RequiresLiveUpdate() const { - return true; -} - -FFINLuaSyntaxHighlighterTextLayoutMarshaller::FFINLuaSyntaxHighlighterTextLayoutMarshaller(const FFINLuaCodeEditorStyle* InLuaSyntaxTextStyle, FFINReflectionReferenceDecorator::FOnNavigate InNavigateDelegate) - : SyntaxTextStyle(InLuaSyntaxTextStyle), NavigateDelegate(InNavigateDelegate){ - /*TArray TokenizerRules; - for (FString Token : TArray({ - " ", "\t", ".", ":", "\"", "\'", "\\", ",", "(", ")", "for", "in", "while", "do", "if", "then", "elseif", "else", - "end", "local", "true", "false", "not", "and", "or", "function", "return", "--[[", "]]--", "--", "+", "-", "/", - "*", "%", "[", "]", "{", "}", "=", "!", "~", "#", ">", "<"})) { - TokenizerRules.Add(FSyntaxTokenizer::FRule(Token)); - } - - TokenizerRules.Sort([](const FSyntaxTokenizer::FRule& A, const FSyntaxTokenizer::FRule& B) { - return A.MatchText.Len() > B.MatchText.Len(); - });*/ -} - -void FFINLuaSyntaxHighlighterTextLayoutMarshaller::ParseTokens(const FString& SourceString, FTextLayout& TargetTextLayout, TArray TokenizedLines) { - ZoneScoped; - - TArray LinesToAdd; - LinesToAdd.Reserve(TokenizedLines.Num()); - - // Multiline State - bool bInString = false; - bool bInBlockComment = false; - - TSharedPtr Run; - for (const FSyntaxTokenizer::FTokenizedLine& TokenizedLine : TokenizedLines) { - TSharedRef ModelString = MakeShareable(new FString()); - TArray> Runs; - - auto DoNormal = [&](FTextRange Range) { - FTextBlockStyle Style = SyntaxTextStyle->NormalTextStyle; - - if (Runs.Num() > 0) { - bool bNew = Runs.Last() == Run; - const FString& Name = Run->GetRunInfo().Name; - if (Name == "SyntaxHighlight.FINLua.Normal") { - Range.BeginIndex = Run->GetTextRange().BeginIndex; - if (bNew) Runs.Pop(); - } else if (Name == TEXT("SyntaxHighlight.FINLua.ReflectionReference")) { - - FRunInfo RunInfo(TEXT("SyntaxHighlight.FINLua.ReflectionReference")); - RunInfo.MetaData = Run->GetRunInfo().MetaData; - const FString& Variant = RunInfo.MetaData[FFINReflectionReferenceDecorator::MetaDataVariantKey]; - if (!bNew) RunInfo.MetaData[FFINReflectionReferenceDecorator::MetaDataTypeKey] = UFINUtils::TextRange(*ModelString, FTextRange(Range.BeginIndex+8, Range.EndIndex)); - else RunInfo.MetaData[FFINReflectionReferenceDecorator::MetaDataTypeKey].Append(UFINUtils::TextRange(*ModelString, Range)); - - Range.BeginIndex = Run->GetTextRange().BeginIndex; - if (bNew) Runs.Pop(); - - const FString& Type = RunInfo.MetaData[FFINReflectionReferenceDecorator::MetaDataTypeKey]; - bool bValid = nullptr != FFINReflectionReferenceDecorator::ReflectionItemFromType(Variant, Type); - FHyperlinkStyle LinkStyle; - LinkStyle.SetUnderlineStyle(bValid ? SyntaxTextStyle->UnderlineStyleValid : SyntaxTextStyle->UnderlineStyleInvalid); - LinkStyle.SetTextStyle(SyntaxTextStyle->NormalTextStyle); - - Run = FFINReflectionReferenceDecorator::CreateRun(RunInfo, ModelString, &LinkStyle, NavigateDelegate, Range, true); - Runs.Add(Run.ToSharedRef()); - return; - } - } - - if (Runs.Num() >= 2) { - const FRunInfo& OpInfo = Runs[Runs.Num()-1]->GetRunInfo(); - const TSharedRef& LibRun = Runs[Runs.Num()-2]; - const FRunInfo& LibInfo = LibRun->GetRunInfo(); - - bool bReflectionReference = OpInfo.Name == TEXT("SyntaxHighlight.FINLua.Operator") && OpInfo.MetaData[TEXT("Operator")] == TEXT("."); - bReflectionReference = bReflectionReference && LibInfo.Name == TEXT("SyntaxHighlight.FINLua.Normal"); - bool bLib = UFINUtils::TextRange(*ModelString, LibRun->GetTextRange()) == TEXT("classes"); - bLib = bLib || UFINUtils::TextRange(*ModelString, LibRun->GetTextRange()) == TEXT("structs"); - bReflectionReference = bReflectionReference && bLib; - if (bReflectionReference) { - FRunInfo RunInfo(TEXT("SyntaxHighlight.FINLua.ReflectionReference")); - FString Variant = UFINUtils::TextRange(*ModelString, LibRun->GetTextRange()); - FString Type = UFINUtils::TextRange(*ModelString, Range); - if (Variant == TEXT("structs")) Variant = TEXT("struct"); - if (Variant == TEXT("classes")) Variant = TEXT("class"); - RunInfo.MetaData.Add(FFINReflectionReferenceDecorator::MetaDataVariantKey, Variant); - RunInfo.MetaData.Add(FFINReflectionReferenceDecorator::MetaDataTypeKey, Type); - - Runs.Pop(); - Range.BeginIndex = Runs[Runs.Num()-1]->GetTextRange().BeginIndex; - Runs.Pop(); - - bool bValid = nullptr != FFINReflectionReferenceDecorator::ReflectionItemFromType(Variant, Type); - FHyperlinkStyle LinkStyle; - LinkStyle.SetUnderlineStyle(bValid ? SyntaxTextStyle->UnderlineStyleValid : SyntaxTextStyle->UnderlineStyleInvalid); - LinkStyle.SetTextStyle(SyntaxTextStyle->NormalTextStyle); - - Run = FFINReflectionReferenceDecorator::CreateRun(RunInfo, ModelString, &LinkStyle, NavigateDelegate, Range, true); - Runs.Add(Run.ToSharedRef()); - - return; - } - } - - FRunInfo RunInfo(TEXT("SyntaxHighlight.FINLua.Normal")); - Run = FSlateTextRun::Create(RunInfo, ModelString, Style, Range); - Runs.Add(Run.ToSharedRef()); - }; - auto DoComment = [&](const FTextRange& Range) { - FTextBlockStyle Style = SyntaxTextStyle->CommentTextStyle; - FRunInfo RunInfo(TEXT("SyntaxHighlight.FINLua.Comment")); - RunInfo.MetaData.Add(TEXT("Splitting"), TEXT("")); - Run = FSlateTextRun::Create(RunInfo, ModelString, Style, Range); - Runs.Add(Run.ToSharedRef()); - }; - auto DoString = [&](const FTextRange& Range) { - FTextBlockStyle Style = SyntaxTextStyle->StringTextStyle; - FRunInfo RunInfo(TEXT("SyntaxHighlight.FINLua.String")); - RunInfo.MetaData.Add(TEXT("Splitting"), TEXT("")); - Run = FSlateTextRun::Create(RunInfo, ModelString, Style, Range); - Runs.Add(Run.ToSharedRef()); - }; - auto DoKeyword = [&](const FTextRange& Range) { - FTextBlockStyle Style = SyntaxTextStyle->KeywordTextStyle; - FRunInfo RunInfo(TEXT("SyntaxHighlight.FINLua.Keyword")); - Run = FSlateTextRun::Create(RunInfo, ModelString, Style, Range); - Runs.Add(Run.ToSharedRef()); - }; - auto DoTrue = [&](const FTextRange& Range) { - FTextBlockStyle Style = SyntaxTextStyle->BoolTrueTextStyle; - FRunInfo RunInfo(TEXT("SyntaxHighlight.FINLua.Keyword")); - Run = FSlateTextRun::Create(RunInfo, ModelString, Style, Range); - Runs.Add(Run.ToSharedRef()); - }; - auto DoFalse = [&](const FTextRange& Range) { - FTextBlockStyle Style = SyntaxTextStyle->BoolFalseTextStyle; - FRunInfo RunInfo(TEXT("SyntaxHighlight.FINLua.Keyword")); - Run = FSlateTextRun::Create(RunInfo, ModelString, Style, Range); - Runs.Add(Run.ToSharedRef()); - }; - auto DoNumber = [&](const FTextRange& Range) { - FTextBlockStyle Style = SyntaxTextStyle->NumberTextStyle; - FRunInfo RunInfo(TEXT("SyntaxHighlight.FINLua.Number")); - Run = FSlateTextRun::Create(RunInfo, ModelString, Style, Range); - Runs.Add(Run.ToSharedRef()); - }; - auto DoWhitespace = [&](const FTextRange& Range) { - FTextBlockStyle Style = SyntaxTextStyle->NormalTextStyle; - FRunInfo RunInfo(TEXT("SyntaxHighlight.FINLua.Whitespace")); - RunInfo.MetaData.Add(TEXT("Splitting"), TEXT("")); - if ((*ModelString)[Range.BeginIndex] == '\t') { - Run = FFINTabRun::Create(RunInfo, ModelString, Style, Range, 4); - } else { - Run = FSlateTextRun::Create(RunInfo, ModelString, Style, Range); - } - Runs.Add(Run.ToSharedRef()); - }; - auto DoOperator = [&](const FTextRange& Range, bool bColored) { - FTextBlockStyle Style = bColored ? SyntaxTextStyle->OperatorTextStyle : SyntaxTextStyle->NormalTextStyle; - FRunInfo RunInfo(TEXT("SyntaxHighlight.FINLua.Operator")); - RunInfo.MetaData.Add(TEXT("Splitting"), TEXT("")); - RunInfo.MetaData.Add("Operator", ModelString->Mid(Range.BeginIndex, Range.Len())); - Run = FSlateTextRun::Create(RunInfo, ModelString, Style, Range); - Runs.Add(Run.ToSharedRef()); - }; - auto DoFunction = [&](const FTextRange& Range, bool bDeclaration) { - FTextBlockStyle Style = bDeclaration ? SyntaxTextStyle->FunctionDeclarationTextStyle : SyntaxTextStyle->FunctionCallTextStyle; - FRunInfo RunInfo(TEXT("SyntaxHighlight.FINLua.Function")); - Run = FSlateTextRun::Create(RunInfo, ModelString, Style, Range); - Runs.Add(Run.ToSharedRef()); - }; - auto FindPrevNoWhitespaceRun = [&](int32 StartIndex = -1) { - if (StartIndex < 0) StartIndex = Runs.Num()-1; - for (int i = StartIndex; i >= 0; i--) { - if (Runs[i]->GetRunInfo().Name != "SyntaxHighlight.FINLua.Whitespace") { - return i; - } - } - return -1; - }; - - int StringStart = ModelString->Len(); - int StringEnd = ModelString->Len(); - - // Single-Line State - bool bInNumber = false; - bool bNumberHadDecimal = false; - bool bInLineComment = false; - bool bIsEscaped = false; - - for (const FSyntaxTokenizer::FToken& Token : TokenizedLine.Tokens) { - const FString TokenString = SourceString.Mid(Token.Range.BeginIndex, Token.Range.Len()); - - int Start = ModelString->Len(); - int End = Start + TokenString.Len(); - ModelString->Append(TokenString); - - // Update Escaped-State - bool bWasEscaped = bIsEscaped; - bIsEscaped = false; - - bool bIsNew = !Run.IsValid() || Start < 1 || Run->GetRunInfo().MetaData.Contains("Splitting"); - - if (bInString || bInLineComment || bInBlockComment) { - StringEnd += TokenString.Len(); - } - if (!bInBlockComment && !bInLineComment && (TokenString == "\\")) { - if (bInString) { - bIsEscaped = !bWasEscaped; - continue; - } - } - if (!bInBlockComment && !bInLineComment && (TokenString == "\"" || TokenString == "\'")) { - if (bInNumber) { - DoNumber(FTextRange(StringStart, StringEnd)); - bIsNew = true; - bInNumber = false; - } - if (bInString) { - if (Start > 0 && bWasEscaped) continue; - DoString(FTextRange(StringStart, StringEnd)); - bInString = false; - continue; - } - bInString = true; - StringStart = Start; - StringEnd = End; - continue; - } - if (!bInString) { - if (TokenString == "--[[" && !bInBlockComment && !bInLineComment) { - if (bInNumber) { - bInNumber = false; - DoNumber(FTextRange(StringStart, StringEnd)); - } - if (!bInBlockComment) { - bInBlockComment = true; - StringStart = Start; - StringEnd = End; - } - } else if (TokenString == "]]--" && bInBlockComment) { - bInBlockComment = false; - DoComment(FTextRange(StringStart, StringEnd)); - continue; - } else if (TokenString == "--" && !bInLineComment && !bInBlockComment) { - if (bInNumber) { - bInNumber = false; - DoNumber(FTextRange(StringStart, StringEnd)); - } - bInLineComment = true; - StringStart = Start; - StringEnd = End; - } - } - if (bInString || bInLineComment || bInBlockComment) continue; - if (bInNumber) { - bool bStillNumber = false; - if (TokenString == ".") { - if (!bNumberHadDecimal) { - bNumberHadDecimal = true; - bStillNumber = true; - } - } else if (FRegexMatcher(FRegexPattern("^[0-9]+$"), TokenString).FindNext()) bStillNumber = true; - if (bStillNumber) { - StringEnd += TokenString.Len(); - continue; - } - DoNumber(FTextRange(StringStart, StringEnd)); - bIsNew = true; - bInNumber = false; - } - - if (Token.Type == FSyntaxTokenizer::ETokenType::Syntax) { - if (bIsNew) { - if (TArray({"while", "for", "in", "do", "if", "then", "elseif", "else", "end", "local", "not", "and", "or", "function", "return"}).Contains(TokenString)) { - DoKeyword(FTextRange(Start, End)); - continue; - } else if (TokenString == "true") { - DoTrue(FTextRange(Start, End)); - continue; - } else if (TokenString == "false") { - DoFalse(FTextRange(Start, End)); - continue; - } - } - if (TokenString == "(") { - int Index = FindPrevNoWhitespaceRun(); - if (Index >= 0 && (Runs[Index]->GetRunInfo().Name == "SyntaxHighlight.FINLua.Normal")) { - FTextRange OldRange = Runs[Index]->GetTextRange(); - Runs.RemoveAt(Index); - int KeywordIndex = FindPrevNoWhitespaceRun(Index-1); - FString Keyword; - if (KeywordIndex >= 0) Keyword = ModelString->Mid(Runs[KeywordIndex]->GetTextRange().BeginIndex, Runs[KeywordIndex]->GetTextRange().EndIndex - Runs[KeywordIndex]->GetTextRange().BeginIndex); - DoFunction(OldRange, KeywordIndex >= 0 && Runs[KeywordIndex]->GetRunInfo().Name == "SyntaxHighlight.FINLua.Keyword" && Keyword == "function"); - TSharedRef NewRun = Runs[Runs.Num()-1]; - Runs.RemoveAt(Runs.Num()-1); - Runs.Insert(NewRun, Index); - DoOperator(FTextRange(Start, End), false); - continue; - } - } - if (TArray({" ","\t"}).Contains(TokenString)) { - DoWhitespace(FTextRange(Start, End)); - continue; - } - if (TArray({".",",",":","(",")","[","]","{","}"}).Contains(TokenString)) { - DoOperator(FTextRange(Start, End), false); - continue; - } - if (TArray({"+","-","*","/","%","#","=","~","!",">","<"}).Contains(TokenString)) { - DoOperator(FTextRange(Start, End), true); - continue; - } - } else { - if (!bIsNew) { - FTextRange ModelRange = Run->GetTextRange(); - Runs.RemoveAt(Runs.Num()-1); - DoNormal(ModelRange); - bIsNew = false; - } else if (TokenString.IsNumeric()) { - bInNumber = true; - StringStart = Start; - StringEnd = End; - continue; - } - } - if (TokenString.IsNumeric()) DoNumber(FTextRange(Start, End)); - else DoNormal(FTextRange(Start, End)); - } - - if (bInNumber) { - DoNumber(FTextRange(StringStart, StringEnd)); - } else if (bInString) { - DoString(FTextRange(StringStart, StringEnd)); - } else if (bInLineComment || bInBlockComment) { - DoComment(FTextRange(StringStart, StringEnd)); - } - - LinesToAdd.Emplace(MoveTemp(ModelString), MoveTemp(Runs)); - } - TargetTextLayout.AddLines(LinesToAdd); -} - void SFINLuaCodeEditor::Construct(const FArguments& InArgs) { - SyntaxHighlighter = MakeShared(InArgs._Style, InArgs._OnNavigateReflection); + //SyntaxHighlighter = MakeShared(InArgs._Style, InArgs._OnNavigateReflection); + SyntaxHighlighter = MakeShared(InArgs._Style); + Style = InArgs._Style; HScrollBar = SNew(SScrollBar) diff --git a/Source/FicsItNetworks/Private/UI/FINLuaLexer.cpp b/Source/FicsItNetworks/Private/UI/FINLuaLexer.cpp new file mode 100644 index 000000000..1baf96499 --- /dev/null +++ b/Source/FicsItNetworks/Private/UI/FINLuaLexer.cpp @@ -0,0 +1,295 @@ +#include "UI/FINLuaSyntax.h" +#include "Internationalization/Regex.h" + +TArray FIN_Lua_Keywords = { + +}; + +TArray FIN_Lua_MainTokens = { + {R"(\[(=*)\[)", FIN_Lua_Token_String_Long, 1}, + {R"(--\[(=*)\[)", FIN_Lua_Token_Comment_Long, 1}, + {"--", FIN_Lua_Token_Comment}, + + {"and(?![a-zA-Z0-9])", FIN_Lua_Token_Keyword}, + {"or(?![a-zA-Z0-9])", FIN_Lua_Token_Keyword}, + {"not(?![a-zA-Z0-9])", FIN_Lua_Token_Keyword}, + {"function(?![a-zA-Z0-9])", FIN_Lua_Token_Keyword}, + {"if(?![a-zA-Z0-9])", FIN_Lua_Token_Keyword}, + {"else(?![a-zA-Z0-9])", FIN_Lua_Token_Keyword}, + {"elseif(?![a-zA-Z0-9])", FIN_Lua_Token_Keyword}, + {"then(?![a-zA-Z0-9])", FIN_Lua_Token_Keyword}, + {"for(?![a-zA-Z0-9])", FIN_Lua_Token_Keyword}, + {"while(?![a-zA-Z0-9])", FIN_Lua_Token_Keyword}, + {"in(?![a-zA-Z0-9])", FIN_Lua_Token_Keyword}, + {"repeat(?![a-zA-Z0-9])", FIN_Lua_Token_Keyword}, + {"until(?![a-zA-Z0-9])", FIN_Lua_Token_Keyword}, + {"do(?![a-zA-Z0-9])", FIN_Lua_Token_Keyword}, + {"break(?![a-zA-Z0-9])", FIN_Lua_Token_Keyword}, + {"end(?![a-zA-Z0-9])", FIN_Lua_Token_Keyword}, + {"goto(?![a-zA-Z0-9])", FIN_Lua_Token_Keyword}, + {"local(?![a-zA-Z0-9])", FIN_Lua_Token_Keyword}, + {"return(?![a-zA-Z0-9])", FIN_Lua_Token_Keyword}, + + {"::([a-zA-Z_][a-zA-Z0-9_]*)::", FIN_Lua_Token_Label, 1}, + + {"true(?![a-zA-Z0-9])", FIN_Lua_Token_Boolean}, + {"false(?![a-zA-Z0-9])", FIN_Lua_Token_Boolean}, + {"nil(?![a-zA-Z0-9])", FIN_Lua_Token_Nil}, + {R"(0[xX][0-9a-fA-F]*\.[0-9a-fA-F]+([pP][+-]?[0-9]+)?)", FIN_Lua_Token_Number, 1}, + {R"(0[xX][0-9a-fA-F]+\.?[0-9a-fA-F]*([pP][+-]?[0-9]+)?)", FIN_Lua_Token_Number, 1}, + {R"([0-9]*\.[0-9]+([eE][+-]?[0-9]+)?)", FIN_Lua_Token_Number, 1}, + {R"([0-9]+\.?[0-9]*([eE][+-]?[0-9]+)?)", FIN_Lua_Token_Number, 1}, + + {R"(\+)", FIN_Lua_Token_Operator}, + {R"(\-)", FIN_Lua_Token_Operator}, + {R"(\*)", FIN_Lua_Token_Operator}, + {R"(\/)", FIN_Lua_Token_Operator}, + {R"(\%)", FIN_Lua_Token_Operator}, + {R"(\^)", FIN_Lua_Token_Operator}, + {R"(//)", FIN_Lua_Token_Operator}, + + {R"(\&)", FIN_Lua_Token_Operator}, + {R"(\~)", FIN_Lua_Token_Operator}, + {R"(\|)", FIN_Lua_Token_Operator}, + {R"(<<)", FIN_Lua_Token_Operator}, + {R"(>>)", FIN_Lua_Token_Operator}, + + {R"(\==)", FIN_Lua_Token_Operator}, + {R"(\~=)", FIN_Lua_Token_Operator}, + {R"(\<=)", FIN_Lua_Token_Operator}, + {R"(\>=)", FIN_Lua_Token_Operator}, + {R"(\<)", FIN_Lua_Token_Operator}, + {R"(\>)", FIN_Lua_Token_Operator}, + {R"(\=)", FIN_Lua_Token_Operator}, + + {R"(\#)", FIN_Lua_Token_Operator}, + {R"(\.\.)", FIN_Lua_Token_Operator}, + {R"(\.\.\.)", FIN_Lua_Token_Operator}, + + {"'", FIN_Lua_Token_Quote}, + {"\"", FIN_Lua_Token_Quote}, + + {R"(\.)", FIN_Lua_Token_Access}, + {":", FIN_Lua_Token_Access}, + + {",", FIN_Lua_Token_Comma}, + + {";", FIN_Lua_Token_Separator}, + + {R"(\()", FIN_Lua_Token_Parenthesis}, + {R"(\))", FIN_Lua_Token_Parenthesis}, + {R"(\[)", FIN_Lua_Token_ParenthesisSquare}, + {R"(\])", FIN_Lua_Token_ParenthesisSquare}, + {R"(\{)", FIN_Lua_Token_ParenthesisCurly}, + {R"(\})", FIN_Lua_Token_ParenthesisCurly}, + + {"[ \\t]+", FIN_Lua_Token_WhiteSpace}, + {"[a-zA-Z_][a-zA-Z0-9_]*", FIN_Lua_Token_Identifier}, +}; + +TArray FIN_Lua_StringTokens = { + {R"(\](=*)\])", FIN_Lua_Token_LongBracketClose, 1}, + + {R"(\\a)", FIN_Lua_Token_String_Escape}, + {R"(\\b)", FIN_Lua_Token_String_Escape}, + {R"(\\f)", FIN_Lua_Token_String_Escape}, + {R"(\\n)", FIN_Lua_Token_String_Escape}, + {R"(\\r)", FIN_Lua_Token_String_Escape}, + {R"(\\t)", FIN_Lua_Token_String_Escape}, + {R"(\\v)", FIN_Lua_Token_String_Escape}, + {R"(\\\\)", FIN_Lua_Token_String_Escape}, + {R"(\\")", FIN_Lua_Token_String_Escape}, + {R"(\\')", FIN_Lua_Token_String_Escape}, + {R"(\\z)", FIN_Lua_Token_String_Escape}, + {R"(\\x[0-9a-fA-F][0-9a-fA-F])", FIN_Lua_Token_String_Escape}, + {R"(\\[0-9][0-9]?[0-9]?)", FIN_Lua_Token_String_Escape}, + {R"(\\\{[0-9a-fA-F]+\})", FIN_Lua_Token_String_Escape}, + + {"'", FIN_Lua_Token_Quote}, + {"\"", FIN_Lua_Token_Quote}, +}; + +TArray FIN_Lua_CommentTokens = { + {R"(\](=*)\])", FIN_Lua_Token_LongBracketClose, 1}, +}; + +FFINLexerRuleSet FIN_Lua_CreateLexerPattern(TArrayView Tokens) { + TMap groups; + TMap tokenMap; + FStringBuilderBase str; + + int group = 0; + for (const auto& token : Tokens) { + str.Appendf(TEXT("(%s)|"), *token.Pattern); + groups.Add(token.Token, ++group); + tokenMap.Add(group, token.Token); + group += token.Groups; + } + + str.RemoveSuffix(1); + + FRegexPattern pattern(str.ToString()); + + return {pattern, groups, tokenMap}; +} + +TOptional> FFINLexerRuleSet::FindMatchedToken(FRegexMatcher& Matcher, const FStringView Code) const { + for (const auto& [group, token] : TokenMap) { + int32 beginning = Matcher.GetCaptureGroupBeginning(group); + if (beginning == INDEX_NONE) { + continue; + } + int32 ending = Matcher.GetCaptureGroupEnding(group); + + FTextRange matched = FTextRange(beginning, ending); + FStringView text = Code.Mid(matched.BeginIndex, matched.Len()); + + return {{token, matched}}; + } + + return {}; +} + +bool FFINLuaLexer::bInitialized = false; +FFINLexerRuleSet FFINLuaLexer::MainContext; +FFINLexerRuleSet FFINLuaLexer::StringContext; +FFINLexerRuleSet FFINLuaLexer::CommentContext; + +void FFINLuaLexer::Initialize() { + if (bInitialized) return; + + MainContext = FIN_Lua_CreateLexerPattern(FIN_Lua_MainTokens); + StringContext = FIN_Lua_CreateLexerPattern(FIN_Lua_StringTokens); + CommentContext = FIN_Lua_CreateLexerPattern(FIN_Lua_CommentTokens); +} + +const FFINLexerRuleSet& FFINLuaLexer::GetContext(EContext context) { + switch (context) { + case Context_Main: + return MainContext; + case Context_String: + return StringContext; + case Context_Comment: + return CommentContext; + } + return MainContext; +} + +void FFINLuaLexer::SetContext(EContext Context) { + CurrentContext = Context; + if (CurrentMatcher) { + const FFINLexerRuleSet& context = GetContext(CurrentContext); + //int32 beginLimit = CurrentMatcher->GetBeginLimit(); + //int32 endLimit = CurrentMatcher->GetEndLimit(); + offset += prevEnd; + prevEnd = 0; + CurrentMatcher = FRegexMatcher(context.Regex, FString(Code.Mid(offset))); + //CurrentMatcher->SetLimits(beginLimit, endLimit); + } else { + SetCode(Code); + } +} + +void FFINLuaLexer::Reset(const FStringView InCode) { + BlockLevel.Reset(); + Quote.Reset(); + Code = InCode; + CurrentMatcher = {}; + SetContext(Context_Main); +} + +void FFINLuaLexer::SetCode(const FStringView InCode) { + Code = InCode; + prevEnd = 0; + offset = 0; + bFound = false; + const FFINLexerRuleSet& context = GetContext(CurrentContext); + CurrentMatcher = FRegexMatcher(context.Regex, FString(Code)); +} +UE_DISABLE_OPTIMIZATION_SHIP +TOptional FFINLuaLexer::NextToken() { + const FFINLexerRuleSet& context = GetContext(CurrentContext); + FRegexMatcher& matcher = *CurrentMatcher; + + if (offset+prevEnd >= Code.Len()) { + // Reached end of string -> Return no Token + if (!BlockLevel.IsSet()) { + SetContext(Context_Main); + } + return {}; + } + + if (!bFound) { + if (!matcher.FindNext()) { + // No match found -> Return Invalid Token containing rest of string + FTextRange range(prevEnd+offset, Code.Len()); + prevEnd = Code.Len()-offset; + return FFINLuaToken(range); + } + + int begin = matcher.GetMatchBeginning(); + if (prevEnd != begin) { + // Match found but it skipped some characters -> Mark next call to use match & now return Invalid Token Containing skipped characters + bFound = true; + FTextRange range(prevEnd+offset, matcher.GetMatchBeginning()+offset); + prevEnd = matcher.GetMatchEnding()-1; + return FFINLuaToken(range); + } + } + bFound = false; + + prevEnd = matcher.GetMatchEnding(); + + const TOptional> optionalToken = context.FindMatchedToken(matcher, Code.Mid(offset)); + if (!optionalToken.IsSet()) { + // Matched but no Token Enum found -> Return Invalid Token with matched string + FTextRange range(matcher.GetMatchBeginning()+offset, matcher.GetMatchEnding()+offset); + return FFINLuaToken(range); + } + EFINLuaToken token = optionalToken->Get<0>(); + FTextRange textRange = optionalToken->Get<1>(); + textRange.BeginIndex += offset; + textRange.EndIndex += offset; + FStringView text = Code.Left(textRange.EndIndex).Mid(textRange.BeginIndex); + + switch (token) { + case FIN_Lua_Token_Quote: + if (Quote) { + if (Quote == text[0]) { + SetContext(Context_Main); + Quote.Reset(); + } + } else { + Quote = text[0]; + SetContext(Context_String); + } + break; + case FIN_Lua_Token_Comment_Long: { + int group = context.GroupMap[FIN_Lua_Token_Comment_Long]+1; + int level = matcher.GetCaptureGroupEnding(group) - matcher.GetCaptureGroupBeginning(group); + BlockLevel = level; + } case FIN_Lua_Token_Comment: + SetContext(Context_Comment); + break; + case FIN_Lua_Token_String_Long: { + int group = context.GroupMap[FIN_Lua_Token_String_Long]+1; + int level = matcher.GetCaptureGroupEnding(group) - matcher.GetCaptureGroupBeginning(group); + BlockLevel = level; + SetContext(Context_String); + break; + } + case FIN_Lua_Token_LongBracketClose: { + int group = context.GroupMap[FIN_Lua_Token_LongBracketClose]+1; + int level = matcher.GetCaptureGroupEnding(group) - matcher.GetCaptureGroupBeginning(group); + if (*BlockLevel == level) { + BlockLevel.Reset(); + SetContext(Context_Main); + } + break; + } + default: ; + } + + return FFINLuaToken(token, textRange); +} +UE_ENABLE_OPTIMIZATION_SHIP \ No newline at end of file diff --git a/Source/FicsItNetworks/Private/UI/FINLuaSyntaxHighlighter.cpp b/Source/FicsItNetworks/Private/UI/FINLuaSyntaxHighlighter.cpp new file mode 100644 index 000000000..1dc64af93 --- /dev/null +++ b/Source/FicsItNetworks/Private/UI/FINLuaSyntaxHighlighter.cpp @@ -0,0 +1,143 @@ +#include "FindLast.h" +#include "FINLuaSyntax.h" +#include "FINLuaCodeEditor.h" +#include "Framework/Text/IRun.h" +#include "Framework/Text/TextLayout.h" +UE_DISABLE_OPTIMIZATION_SHIP +void FFINLuaSyntaxHighlighter::SetText(const FString& SourceString, FTextLayout& TargetTextLayout) { + Lexer.Reset(TEXT("")); + + TArray lineRanges; + FTextRange::CalculateLineRangesFromString(SourceString, lineRanges); + + for (const FTextRange& lineRange : lineRanges) { + TSharedRef str = MakeShared(SourceString.Mid(lineRange.BeginIndex, lineRange.Len())); + Lexer.SetCode(*str); + TArray> runs; + + bool bFunctionDeclaration = false; + + while (true) { + FFINLuaLexer::EContext prevContext = Lexer.GetContext(); + auto optToken = Lexer.NextToken(); + if (!optToken.IsSet()) { + if (runs.IsEmpty()) { + FTextBlockStyle Style = SyntaxTextStyle->NormalTextStyle; + FRunInfo RunInfo(TEXT("SyntaxHighlight.FINLua.Normal")); + TSharedPtr Run = FSlateTextRun::Create(RunInfo, str, Style, FTextRange(0,0)); + runs.Add(Run.ToSharedRef()); + } + break; + } + auto [token, textRange] = *optToken; + FStringView text = FStringView(*str).Mid(textRange.BeginIndex, textRange.Len()); + if (token.IsSet() && Lexer.GetContext() == FFINLuaLexer::Context_Main && prevContext == FFINLuaLexer::Context_Main) { + switch (*token) { + case FIN_Lua_Token_Keyword: + case FIN_Lua_Token_Nil: + case FIN_Lua_Token_Operator: { + FTextBlockStyle Style = SyntaxTextStyle->KeywordTextStyle; + FRunInfo RunInfo(TEXT("SyntaxHighlight.FINLua.Keyword")); + TSharedPtr Run = FSlateTextRun::Create(RunInfo, str, Style, textRange); + runs.Add(Run.ToSharedRef()); + if (text == TEXT("function")) { + bFunctionDeclaration = true; + } + continue; + } + case FIN_Lua_Token_Boolean: + if (text == TEXT("true")) { + FTextBlockStyle Style = SyntaxTextStyle->BoolTrueTextStyle; + FRunInfo RunInfo(TEXT("SyntaxHighlight.FINLua.Keyword")); + TSharedPtr Run = FSlateTextRun::Create(RunInfo, str, Style, textRange); + runs.Add(Run.ToSharedRef()); + } else { + FTextBlockStyle Style = SyntaxTextStyle->BoolFalseTextStyle; + FRunInfo RunInfo(TEXT("SyntaxHighlight.FINLua.Keyword")); + TSharedPtr Run = FSlateTextRun::Create(RunInfo, str, Style, textRange); + runs.Add(Run.ToSharedRef()); + } + continue; + case FIN_Lua_Token_Number: { + FTextBlockStyle Style = SyntaxTextStyle->NumberTextStyle; + FRunInfo RunInfo(TEXT("SyntaxHighlight.FINLua.Number")); + TSharedPtr Run = FSlateTextRun::Create(RunInfo, str, Style, textRange); + runs.Add(Run.ToSharedRef()); + continue; + } + case FIN_Lua_Token_Parenthesis: + if (text == TEXT("(")) { + TSharedRef* prevRun = Algo::FindLastByPredicate(runs, [](const TSharedRef& run) { + return run->GetRunInfo().Name == TEXT("SyntaxHighlight.FINLua.Identifier"); + }); + if (prevRun) { + TSharedRef& run = *prevRun; + FTextBlockStyle style; + if (bFunctionDeclaration) { + style = SyntaxTextStyle->FunctionDeclarationTextStyle; + } else { + style = SyntaxTextStyle->FunctionCallTextStyle; + } + run = FSlateTextRun::Create(run->GetRunInfo(), str, style, run->GetTextRange()); + } + bFunctionDeclaration = false; + } + break; + case FIN_Lua_Token_Identifier: { + int num = runs.Num(); + if (num >= 2) { + int runBegin = runs[num-2]->GetTextRange().BeginIndex; + auto prevText = FStringView(*str).Left(textRange.BeginIndex).Mid(runBegin); + bool bClass = prevText == TEXT("classes."); + bool bStruct = prevText == TEXT("structs."); + if (bClass || bStruct) { + FRunInfo RunInfo(TEXT("SyntaxHighlight.FINLua.ReflectionReference")); + FString Variant = bClass ? TEXT("class") : TEXT("struct"); + FString Type = FString(text); + RunInfo.MetaData.Add(FFINReflectionReferenceDecorator::MetaDataVariantKey, Variant); + RunInfo.MetaData.Add(FFINReflectionReferenceDecorator::MetaDataTypeKey, Type); + bool bValid = nullptr != FFINReflectionReferenceDecorator::ReflectionItemFromType(Variant, Type); + FHyperlinkStyle LinkStyle; + LinkStyle.SetUnderlineStyle(bValid ? SyntaxTextStyle->UnderlineStyleValid : SyntaxTextStyle->UnderlineStyleInvalid); + LinkStyle.SetTextStyle(SyntaxTextStyle->NormalTextStyle); + runs.Pop(); + runs.Pop(); + runs.Add(FFINReflectionReferenceDecorator::CreateRun(RunInfo, str, &LinkStyle, FFINReflectionReferenceDecorator::FOnNavigate(), FTextRange(runBegin, textRange.EndIndex), true)); + continue; + } + } + FTextBlockStyle Style = SyntaxTextStyle->NormalTextStyle; + FRunInfo RunInfo(TEXT("SyntaxHighlight.FINLua.Identifier")); + TSharedPtr Run = FSlateTextRun::Create(RunInfo, str, Style, textRange); + runs.Add(Run.ToSharedRef()); + continue; + } + default: break; + } + } + + if (Lexer.GetContext() == FFINLuaLexer::Context_String || prevContext == FFINLuaLexer::Context_String) { + FTextBlockStyle Style = SyntaxTextStyle->StringTextStyle; + FRunInfo RunInfo(TEXT("SyntaxHighlight.FINLua.String")); + TSharedPtr Run = FSlateTextRun::Create(RunInfo, str, Style, textRange); + runs.Add(Run.ToSharedRef()); + continue; + } + + if (Lexer.GetContext() == FFINLuaLexer::Context_Comment || prevContext == FFINLuaLexer::Context_Comment) { + FTextBlockStyle Style = SyntaxTextStyle->CommentTextStyle; + FRunInfo RunInfo(TEXT("SyntaxHighlight.FINLua.Comment")); + TSharedPtr Run = FSlateTextRun::Create(RunInfo, str, Style, textRange); + runs.Add(Run.ToSharedRef()); + continue; + } + + FTextBlockStyle Style = SyntaxTextStyle->NormalTextStyle; + FRunInfo RunInfo(TEXT("SyntaxHighlight.FINLua.Normal")); + TSharedPtr Run = FSlateTextRun::Create(RunInfo, str, Style, textRange); + runs.Add(Run.ToSharedRef()); + } + TargetTextLayout.AddLine(FTextLayout::FNewLineData(str, runs)); + } +} +UE_ENABLE_OPTIMIZATION_SHIP \ No newline at end of file diff --git a/Source/FicsItNetworks/Public/UI/FINLuaCodeEditor.h b/Source/FicsItNetworks/Public/UI/FINLuaCodeEditor.h index e098eacc0..27324d670 100644 --- a/Source/FicsItNetworks/Public/UI/FINLuaCodeEditor.h +++ b/Source/FicsItNetworks/Public/UI/FINLuaCodeEditor.h @@ -137,26 +137,12 @@ class FICSITNETWORKS_API FFINTabRun : public ISlateRun, public TSharedFromThis TokenizedLines); - - const FFINLuaCodeEditorStyle* SyntaxTextStyle; - FFINReflectionReferenceDecorator::FOnNavigate NavigateDelegate; -}; - class SFINLuaCodeEditor : public SBorder { public: typedef FFINReflectionReferenceDecorator::FOnNavigate FOnNavigateReflection; private: - TSharedPtr SyntaxHighlighter; + TSharedPtr SyntaxHighlighter; TSharedPtr TextLayout; diff --git a/Source/FicsItNetworks/Public/UI/FINLuaSyntax.h b/Source/FicsItNetworks/Public/UI/FINLuaSyntax.h new file mode 100644 index 000000000..7bb6b7fc3 --- /dev/null +++ b/Source/FicsItNetworks/Public/UI/FINLuaSyntax.h @@ -0,0 +1,123 @@ +#pragma once + +#include "CoreMinimal.h" +#include "BaseTextLayoutMarshaller.h" +#include "Regex.h" +#include "TextLayout.h" + +struct FFINLuaCodeEditorStyle; + +enum EFINLuaToken { + FIN_Lua_Token_Operator, + FIN_Lua_Token_Keyword, + FIN_Lua_Token_Label, + + FIN_Lua_Token_Parenthesis, + FIN_Lua_Token_ParenthesisSquare, + FIN_Lua_Token_ParenthesisCurly, + FIN_Lua_Token_Quote, + FIN_Lua_Token_Comma, + FIN_Lua_Token_Access, + FIN_Lua_Token_Separator, + + FIN_Lua_Token_Nil, + FIN_Lua_Token_Boolean, + FIN_Lua_Token_Number, + + FIN_Lua_Token_WhiteSpace, + FIN_Lua_Token_Identifier, + + FIN_Lua_Token_String_Long, + FIN_Lua_Token_String_Escape, + + FIN_Lua_Token_Comment, + FIN_Lua_Token_Comment_Long, + + FIN_Lua_Token_LongBracketClose, +}; + +struct FFINLexerPattern { + FString Pattern; + EFINLuaToken Token; + int Groups = 0; +}; + +struct FFINLexerRuleSet { + FRegexPattern Regex = FRegexPattern(TEXT("")); + TMap GroupMap; + TMap TokenMap; + + TOptional> FindMatchedToken(FRegexMatcher& Matcher, const FStringView Code) const; +}; + +struct FFINLuaToken { + TOptional Token; + FTextRange TextRange; + + FFINLuaToken(EFINLuaToken Token, FTextRange TextRange) : Token(Token), TextRange(TextRange) {} + FFINLuaToken(FTextRange TextRange) : TextRange(TextRange) {} +}; + +class FFINLuaLexer { +public: + enum EContext { + Context_Main, + Context_String, + Context_Comment, + }; +private: + static bool bInitialized; + static FFINLexerRuleSet MainContext; + static FFINLexerRuleSet StringContext; + static FFINLexerRuleSet CommentContext; + + static void Initialize(); + + static const FFINLexerRuleSet& GetContext(EContext context); + + EContext CurrentContext; + TOptional BlockLevel; + TOptional Quote; + FStringView Code; + int32 prevEnd = 0; + int32 offset = 0; + TOptional CurrentMatcher; + bool bFound = false; + + void SetContext(EContext Context); + +public: + void Reset(const FStringView InCode); + void SetCode(const FStringView InCode); + + EContext GetContext() const { + return CurrentContext; + } + + FFINLuaLexer() { + Initialize(); + Reset(TEXT("")); + } + + TOptional NextToken(); +}; + +class FFINLuaSyntaxHighlighter : public FBaseTextLayoutMarshaller { +private: + const FFINLuaCodeEditorStyle* SyntaxTextStyle; + + FFINLuaLexer Lexer; + +public: + FFINLuaSyntaxHighlighter(const FFINLuaCodeEditorStyle* SyntaxTextStyle) : SyntaxTextStyle(SyntaxTextStyle) {} + + virtual bool RequiresLiveUpdate() const override { + return true; + } + + virtual void SetText(const FString& SourceString, FTextLayout& TargetTextLayout) override; + + virtual void GetText(FString& TargetString, const FTextLayout& SourceTextLayout) override { + SourceTextLayout.GetAsText(TargetString); + } +}; diff --git a/ThirdParty/eris b/ThirdParty/eris index 7b81cb7f6..ecd2a6e01 160000 --- a/ThirdParty/eris +++ b/ThirdParty/eris @@ -1 +1 @@ -Subproject commit 7b81cb7f6baf23b5d80a0eabaa2bf93bcc7cc041 +Subproject commit ecd2a6e014e57910fce6481a603f2705ae85e0ac