Skip to content

Commit

Permalink
some TokenList cleanups (#5848)
Browse files Browse the repository at this point in the history
  • Loading branch information
@firewave
firewave authored Jan 6, 2024
1 parent 328f98b commit 81700b4
Show file tree
Hide file tree
Showing 5 changed files with 36 additions and 29 deletions.
6 changes: 3 additions & 3 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -491,7 +491,7 @@ $(libcppdir)/checkautovariables.o: lib/checkautovariables.cpp lib/addoninfo.h li
$(libcppdir)/checkbool.o: lib/checkbool.cpp lib/addoninfo.h lib/astutils.h lib/check.h lib/checkbool.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h
$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/checkbool.cpp

$(libcppdir)/checkboost.o: lib/checkboost.cpp lib/check.h lib/checkboost.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/sourcelocation.h lib/standards.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h
$(libcppdir)/checkboost.o: lib/checkboost.cpp lib/addoninfo.h lib/check.h lib/checkboost.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h
$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/checkboost.cpp

$(libcppdir)/checkbufferoverrun.o: lib/checkbufferoverrun.cpp externals/tinyxml2/tinyxml2.h lib/addoninfo.h lib/astutils.h lib/check.h lib/checkbufferoverrun.h lib/color.h lib/config.h lib/ctu.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/valueflow.h lib/vfvalue.h lib/xml.h
Expand Down Expand Up @@ -593,7 +593,7 @@ $(libcppdir)/infer.o: lib/infer.cpp lib/calculate.h lib/config.h lib/errortypes.
$(libcppdir)/keywords.o: lib/keywords.cpp lib/config.h lib/keywords.h lib/standards.h lib/utils.h
$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/keywords.cpp

$(libcppdir)/library.o: lib/library.cpp externals/tinyxml2/tinyxml2.h lib/astutils.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/path.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenlist.h lib/utils.h lib/valueflow.h lib/vfvalue.h lib/xml.h
$(libcppdir)/library.o: lib/library.cpp externals/tinyxml2/tinyxml2.h lib/addoninfo.h lib/astutils.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/path.h lib/platform.h lib/settings.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenlist.h lib/utils.h lib/valueflow.h lib/vfvalue.h lib/xml.h
$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/library.cpp

$(libcppdir)/mathlib.o: lib/mathlib.cpp externals/simplecpp/simplecpp.h lib/config.h lib/errortypes.h lib/mathlib.h lib/utils.h
Expand Down Expand Up @@ -626,7 +626,7 @@ $(libcppdir)/settings.o: lib/settings.cpp externals/picojson/picojson.h lib/addo
$(libcppdir)/summaries.o: lib/summaries.cpp lib/addoninfo.h lib/analyzerinfo.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/sourcelocation.h lib/standards.h lib/summaries.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h
$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/summaries.cpp

$(libcppdir)/suppressions.o: lib/suppressions.cpp externals/tinyxml2/tinyxml2.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/mathlib.h lib/path.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h lib/xml.h
$(libcppdir)/suppressions.o: lib/suppressions.cpp externals/tinyxml2/tinyxml2.h lib/addoninfo.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/path.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h lib/xml.h
$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/suppressions.cpp

$(libcppdir)/templatesimplifier.o: lib/templatesimplifier.cpp lib/addoninfo.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h
Expand Down
4 changes: 2 additions & 2 deletions lib/tokenize.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3361,10 +3361,10 @@ bool Tokenizer::simplifyTokens1(const std::string &configuration)
if (mTimerResults) {
Timer t("Tokenizer::simplifyTokens1::createAst", mSettings->showtime, mTimerResults);
list.createAst();
list.validateAst();
list.validateAst(mSettings->debugnormal);
} else {
list.createAst();
list.validateAst();
list.validateAst(mSettings->debugnormal);
}

if (mTimerResults) {
Expand Down
27 changes: 15 additions & 12 deletions lib/tokenlist.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,9 @@ TokenList::TokenList(const Settings* settings) :
mSettings(settings)
{
mTokensFrontBack.list = this;
if (mSettings && (mSettings->enforcedLang != Settings::Language::None)) {
mLang = mSettings->enforcedLang;
}
}

TokenList::~TokenList()
Expand Down Expand Up @@ -80,12 +83,12 @@ void TokenList::deallocateTokens()

void TokenList::determineCppC()
{
if (!mSettings) {
mIsC = Path::isC(getSourceFilePath());
mIsCpp = Path::isCPP(getSourceFilePath());
} else {
mIsC = mSettings->enforcedLang == Settings::Language::C || (mSettings->enforcedLang == Settings::Language::None && Path::isC(getSourceFilePath()));
mIsCpp = mSettings->enforcedLang == Settings::Language::CPP || (mSettings->enforcedLang == Settings::Language::None && Path::isCPP(getSourceFilePath()));
// only try to determine it if it wasn't enforced
if (mLang == Settings::Language::None) {
if (Path::isC(getSourceFilePath()))
mLang = Settings::Language::C;
else if (Path::isCPP(getSourceFilePath()))
mLang = Settings::Language::CPP;
}
}

Expand Down Expand Up @@ -1722,10 +1725,10 @@ namespace {
};
}

void TokenList::validateAst() const
void TokenList::validateAst(bool print) const
{
OnException oe{[&] {
if (mSettings && mSettings->debugnormal)
if (print)
mTokensFrontBack.front->printOut();
}};
// Check for some known issues in AST to avoid crash/hang later on
Expand Down Expand Up @@ -1854,7 +1857,7 @@ void TokenList::simplifyPlatformTypes()
if (!mSettings)
return;

const bool isCPP11 = mSettings->standards.cpp >= Standards::CPP11;
const bool isCPP11 = isCPP() && (mSettings->standards.cpp >= Standards::CPP11);

enum { isLongLong, isLong, isInt } type;

Expand Down Expand Up @@ -1996,7 +1999,7 @@ void TokenList::simplifyStdType()
continue;
}

if (Token::Match(tok, "char|short|int|long|unsigned|signed|double|float") || (mSettings->standards.c >= Standards::C99 && Token::Match(tok, "complex|_Complex"))) {
if (Token::Match(tok, "char|short|int|long|unsigned|signed|double|float") || (isC() && (!mSettings || (mSettings->standards.c >= Standards::C99)) && Token::Match(tok, "complex|_Complex"))) {
bool isFloat= false;
bool isSigned = false;
bool isUnsigned = false;
Expand All @@ -2019,7 +2022,7 @@ void TokenList::simplifyStdType()
else if (Token::Match(tok2, "float|double")) {
isFloat = true;
typeSpec = tok2;
} else if (mSettings->standards.c >= Standards::C99 && Token::Match(tok2, "complex|_Complex"))
} else if (isC() && (!mSettings || (mSettings->standards.c >= Standards::C99)) && Token::Match(tok2, "complex|_Complex"))
isComplex = !isFloat || tok2->str() == "_Complex" || Token::Match(tok2->next(), "*|&|%name%"); // Ensure that "complex" is not the variables name
else if (Token::Match(tok2, "char|int")) {
if (!typeSpec)
Expand Down Expand Up @@ -2057,7 +2060,7 @@ void TokenList::simplifyStdType()

bool TokenList::isKeyword(const std::string &str) const
{
if (mIsCpp) {
if (isCPP()) {
// TODO: integrate into keywords?
// types and literals are not handled as keywords
static const std::unordered_set<std::string> cpp_types = {"bool", "false", "true"};
Expand Down
11 changes: 5 additions & 6 deletions lib/tokenlist.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,13 @@
//---------------------------------------------------------------------------

#include "config.h"
#include "settings.h"

#include <cstddef>
#include <iosfwd>
#include <string>
#include <vector>

class Settings;
class Token;
class TokenList;

Expand Down Expand Up @@ -61,12 +61,12 @@ class CPPCHECKLIB TokenList {

/** Is the code C. Used for bailouts */
bool isC() const {
return mIsC;
return mLang == Settings::Language::C;
}

/** Is the code CPP. Used for bailouts */
bool isCPP() const {
return mIsCpp;
return mLang == Settings::Language::CPP;
}

/**
Expand Down Expand Up @@ -171,7 +171,7 @@ class CPPCHECKLIB TokenList {
* Check abstract syntax tree.
* Throws InternalError on failure
*/
void validateAst() const;
void validateAst(bool print) const;

/**
* Verify that the given token is an element of the tokenlist.
Expand Down Expand Up @@ -214,8 +214,7 @@ class CPPCHECKLIB TokenList {
const Settings* const mSettings{};

/** File is known to be C/C++ code */
bool mIsC{};
bool mIsCpp{};
Settings::Language mLang{Settings::Language::None};
};

/// @}
Expand Down
17 changes: 11 additions & 6 deletions test/testtokenize.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2873,28 +2873,33 @@ class TestTokenizer : public TestFixture {
}
{
const char code[] = "float complex x;";
const char expected[] = "_Complex float x ;";
const char expected[] = "float complex x ;";
ASSERT_EQUALS(expected, tokenizeAndStringify(code));
}
{
const char code[] = "float complex x;";
const char expected[] = "_Complex float x ;";
ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c"));
}
{
const char code[] = "complex float x;";
const char expected[] = "_Complex float x ;";
ASSERT_EQUALS(expected, tokenizeAndStringify(code));
ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c"));
}
{
const char code[] = "complex long double x;";
const char expected[] = "_Complex long double x ;";
ASSERT_EQUALS(expected, tokenizeAndStringify(code));
ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c"));
}
{
const char code[] = "long double complex x;";
const char expected[] = "_Complex long double x ;";
ASSERT_EQUALS(expected, tokenizeAndStringify(code));
ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c"));
}
{
const char code[] = "double complex;";
const char expected[] = "double complex ;";
ASSERT_EQUALS(expected, tokenizeAndStringify(code));
ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c"));
}
}

Expand Down Expand Up @@ -5975,7 +5980,7 @@ class TestTokenizer : public TestFixture {
tokenList.prepareTernaryOpForAST();
tokenList.list.createAst();

tokenList.list.validateAst();
tokenList.list.validateAst(false);

// Basic AST validation
for (const Token *tok = tokenList.list.front(); tok; tok = tok->next()) {
Expand Down

0 comments on commit 81700b4

Please sign in to comment.