template simplifier: better detection of template functions (#1539)

* template simplifier: better detection of template functions

* fix comment

lib/templatesimplifier.cpp

@@ -73,8 +73,8 @@ TemplateSimplifier::TokenAndName::~TokenAndName()
         token->templateSimplifierPointers().erase(this);
 }
 
-TemplateSimplifier::TemplateSimplifier(TokenList &tokenlist, const Settings *settings, ErrorLogger *errorLogger)
+TemplateSimplifier::TemplateSimplifier(Tokenizer *tokenizer)
-    : mTokenList(tokenlist), mSettings(settings), mErrorLogger(errorLogger)
+    : mTokenizer(tokenizer), mTokenList(tokenizer->list), mSettings(tokenizer->mSettings), mErrorLogger(tokenizer->mErrorLogger)
 {
 }
 
@@ -884,25 +884,25 @@ bool TemplateSimplifier::instantiateMatch(const Token *instance, const std::size
     return true;
 }
 
-// Utility function for TemplateSimplifier::getTemplateNamePosition, that works on template member functions,
+// Utility function for TemplateSimplifier::getTemplateNamePosition, that works on template functions
-// hence this pattern: "> %type% [%type%] < ... > :: %type% ("
+bool TemplateSimplifier::getTemplateNamePositionTemplateFunction(const Token *tok, int &namepos)
-static bool getTemplateNamePositionTemplateMember(const Token *tok, int &namepos)
 {
-    namepos = 2;
+    namepos = 1;
     while (tok && tok->next()) {
         if (Token::Match(tok->next(), ";|{"))
             return false;
         else if (Token::Match(tok->next(), "%type% <")) {
             const Token *closing = tok->tokAt(2)->findClosingBracket();
-            if (closing && Token::Match(closing->next(), ":: ~| %name% (")) {
+            if (closing) {
-                if (closing->strAt(1) == "~")
+                if (closing->strAt(1) == "(" && mTokenizer->isFunctionHead(closing->tokAt(2), ";|{|:"))
-                    closing = closing->next();
+                    return true;
-                while (tok && tok->next() != closing->next()) {
+                while (tok && tok->next() && tok->next() != closing) {
                     tok = tok->next();
                     namepos++;
                 }
-                return true;
             }
+        } else if (Token::Match(tok->next(), "%type% (") && mTokenizer->isFunctionHead(tok->tokAt(2), ";|{|:")) {
+            return true;
         }
         tok = tok->next();
         namepos++;
@@ -913,31 +913,12 @@ static bool getTemplateNamePositionTemplateMember(const Token *tok, int &namepos
 int TemplateSimplifier::getTemplateNamePosition(const Token *tok, bool forward)
 {
     // get the position of the template name
-    int namepos = 0, starAmpPossiblePosition = 0;
+    int namepos = 0;
     if ((forward && Token::Match(tok, "> class|struct|union %type% :|<|;")) ||
         (!forward && Token::Match(tok, "> class|struct|union %type% {|:|<")))
         namepos = 2;
-    else if (Token::Match(tok, "> %type% *|&| %type% ("))
+    else if (!getTemplateNamePositionTemplateFunction(tok, namepos))
-        namepos = 2;
+        return -1; // Name not found
-    else if (Token::Match(tok, "> %type% %type% <") &&
-             Token::simpleMatch(tok->tokAt(3)->findClosingBracket(), "> ("))
-        namepos = 2;
-    else if (Token::Match(tok, "> %type% %type% *|&| %type% ("))
-        namepos = 3;
-    else if (getTemplateNamePositionTemplateMember(tok, namepos))
-        ;
-    else if (Token::Match(tok, "> %type% *|&| %type% :: %type% (")) {
-        namepos = 4;
-        starAmpPossiblePosition = 2;
-    } else if (Token::Match(tok, "> %type% %type% *|&| %type% :: %type% (")) {
-        namepos = 5;
-        starAmpPossiblePosition = 3;
-    } else {
-        // Name not found
-        return -1;
-    }
-    if (Token::Match(tok->tokAt(starAmpPossiblePosition ? starAmpPossiblePosition : namepos), "*|&"))
-        ++namepos;
 
     return namepos;
 }

lib/templatesimplifier.h

@@ -43,7 +43,7 @@ class TokenList;
 /** @brief Simplify templates from the preprocessed and partially simplified code. */
 class CPPCHECKLIB TemplateSimplifier {
 public:
-    TemplateSimplifier(TokenList &tokenlist, const Settings *settings, ErrorLogger *errorLogger);
+    TemplateSimplifier(Tokenizer *tokenizer);
     ~TemplateSimplifier();
 
     /**
@@ -98,7 +98,15 @@ public:
      * @return -1 to bail out or positive integer to identity the position
      * of the template name.
      */
-    static int getTemplateNamePosition(const Token *tok, bool forward = false);
+    int getTemplateNamePosition(const Token *tok, bool forward = false);
+
+    /**
+     * Get template name position
+     * @param tok The ">" token e.g. before "class"
+     * @param namepos return offset to name
+     * @return true if name found, false if not
+     * */
+    bool getTemplateNamePositionTemplateFunction(const Token *tok, int &namepos);
 
     /**
      * Simplify templates
@@ -262,6 +270,7 @@ private:
         Token *tok2,
         std::list<std::string> &typeStringsUsedInTemplateInstantiation);
 
+    Tokenizer *mTokenizer;
     TokenList &mTokenList;
     const Settings *mSettings;
     ErrorLogger *mErrorLogger;

lib/tokenize.cpp

@@ -175,7 +175,7 @@ Tokenizer::Tokenizer(const Settings *settings, ErrorLogger *errorLogger) :
     // make sure settings are specified
     assert(mSettings);
 
-    mTemplateSimplifier = new TemplateSimplifier(list, settings, errorLogger);
+    mTemplateSimplifier = new TemplateSimplifier(this);
 }
 
 Tokenizer::~Tokenizer()

lib/tokenize.h

@@ -51,6 +51,8 @@ class CPPCHECKLIB Tokenizer {
     friend class TestSimplifyTypedef;
     friend class TestTokenizer;
     friend class SymbolDatabase;
+    friend class TestSimplifyTemplate;
+    friend class TemplateSimplifier;
 
     /** Class used in Tokenizer::setVarIdPass1 */
     class VariableMap {

test/testsimplifytemplate.cpp

@@ -2305,19 +2305,16 @@ private:
     }
 
     // Helper function to unit test TemplateSimplifier::getTemplateNamePosition
-    int templateNamePositionHelper(const char code[], unsigned offset = 0, bool onlyCreateTokens = false) {
+    int templateNamePositionHelper(const char code[], unsigned offset = 0) {
         Tokenizer tokenizer(&settings, this);
 
         std::istringstream istr(code);
-        if (onlyCreateTokens)
+        tokenizer.tokenize(istr, "test.cpp", emptyString);
-            tokenizer.createTokens(istr, "test.cpp");
-        else
-            tokenizer.tokenize(istr, "test.cpp", emptyString);
 
         const Token *_tok = tokenizer.tokens();
         for (unsigned i = 0 ; i < offset ; ++i)
             _tok = _tok->next();
-        return TemplateSimplifier::getTemplateNamePosition(_tok);
+        return tokenizer.mTemplateSimplifier->getTemplateNamePosition(_tok);
     }
 
     void templateNamePosition() {
@@ -2329,29 +2326,52 @@ private:
         // Template function definitions
         ASSERT_EQUALS(2, templateNamePositionHelper("template<class T> unsigned foo() { return 0; }", 4));
         ASSERT_EQUALS(3, templateNamePositionHelper("template<class T> unsigned* foo() { return 0; }", 4));
+        ASSERT_EQUALS(4, templateNamePositionHelper("template<class T> unsigned** foo() { return 0; }", 4));
+
         ASSERT_EQUALS(3, templateNamePositionHelper("template<class T> const unsigned foo() { return 0; }", 4));
         ASSERT_EQUALS(4, templateNamePositionHelper("template<class T> const unsigned& foo() { return 0; }", 4));
+        ASSERT_EQUALS(5, templateNamePositionHelper("template<class T> const unsigned** foo() { return 0; }", 4));
+
+        ASSERT_EQUALS(4, templateNamePositionHelper("template<class T> std::string foo() { static str::string str; return str; }", 4));
+        ASSERT_EQUALS(5, templateNamePositionHelper("template<class T> std::string & foo() { static str::string str; return str; }", 4));
+        ASSERT_EQUALS(6, templateNamePositionHelper("template<class T> const std::string & foo() { static str::string str; return str; }", 4));
+
+        ASSERT_EQUALS(9, templateNamePositionHelper("template<class T> std::map<int, int> foo() { static std::map<int, int> m; return m; }", 4));
+        ASSERT_EQUALS(10, templateNamePositionHelper("template<class T> std::map<int, int> & foo() { static std::map<int, int> m; return m; }", 4));
+        ASSERT_EQUALS(11, templateNamePositionHelper("template<class T> const std::map<int, int> & foo() { static std::map<int, int> m; return m; }", 4));
         // Class template members
-        ASSERT_EQUALS(4, templateNamePositionHelper("class A { template<class T> unsigned foo(); }; "
+        ASSERT_EQUALS(4, templateNamePositionHelper(
-                      "template<class T> unsigned A::foo() { return 0; }", 19));
+                          "class A { template<class T> unsigned foo(); }; "
-        ASSERT_EQUALS(5, templateNamePositionHelper("class A { template<class T> const unsigned foo(); }; "
+                          "template<class T> unsigned A::foo() { return 0; }", 19));
-                      "template<class T> const unsigned A::foo() { return 0; }", 20));
+        ASSERT_EQUALS(5, templateNamePositionHelper(
-        TODO_ASSERT_EQUALS(7, -1, templateNamePositionHelper("class A { class B { template<class T> const unsigned foo(); }; } ; "
+                          "class A { template<class T> const unsigned foo(); }; "
-                           "template<class T> const unsigned A::B::foo() { return 0; }", 25));
+                          "template<class T> const unsigned A::foo() { return 0; }", 20));
+        ASSERT_EQUALS(7, templateNamePositionHelper(
+                          "class A { class B { template<class T> const unsigned foo(); }; } ; "
+                          "template<class T> const unsigned A::B::foo() { return 0; }", 25));
+        ASSERT_EQUALS(8, templateNamePositionHelper(
+                          "class A { class B { template<class T> const unsigned * foo(); }; } ; "
+                          "template<class T> const unsigned * A::B::foo() { return 0; }", 26));
+        ASSERT_EQUALS(9, templateNamePositionHelper(
+                          "class A { class B { template<class T> const unsigned ** foo(); }; } ; "
+                          "template<class T> const unsigned ** A::B::foo() { return 0; }", 27));
         // Template class member
-        ASSERT_EQUALS(6, templateNamePositionHelper("template<class T> class A { A(); }; "
+        ASSERT_EQUALS(6, templateNamePositionHelper(
-                      "template<class T> A<T>::A() {}", 18));
+                          "template<class T> class A { A(); }; "
-        ASSERT_EQUALS(8, templateNamePositionHelper("template<class T, class U> class A { A(); }; "
+                          "template<class T> A<T>::A() {}", 18));
-                      "template<class T, class U> A<T, U>::A() {}", 24));
+        ASSERT_EQUALS(8, templateNamePositionHelper(
-        ASSERT_EQUALS(7, templateNamePositionHelper("template<class T> class A { unsigned foo(); }; "
+                          "template<class T, class U> class A { A(); }; "
-                      "template<class T> unsigned A<T>::foo() { return 0; }", 19));
+                          "template<class T, class U> A<T, U>::A() {}", 24));
-        ASSERT_EQUALS(9, templateNamePositionHelper("template<class T, class U> class A { unsigned foo(); }; "
+        ASSERT_EQUALS(7, templateNamePositionHelper(
-                      "template<class T, class U> unsigned A<T, U>::foo() { return 0; }", 25));
+                          "template<class T> class A { unsigned foo(); }; "
-        ASSERT_EQUALS(9, templateNamePositionHelper("template<class T, class U> class A { unsigned foo(); }; "
+                          "template<class T> unsigned A<T>::foo() { return 0; }", 19));
-                      "template<class T, class U> unsigned A<T, U>::foo() { return 0; }", 25, /*onlyCreateTokens=*/true));
+        ASSERT_EQUALS(9, templateNamePositionHelper(
-        ASSERT_EQUALS(12, templateNamePositionHelper("template<class T> class v {}; "
+                          "template<class T, class U> class A { unsigned foo(); }; "
-                      "template<class T, class U> class A { unsigned foo(); }; "
+                          "template<class T, class U> unsigned A<T, U>::foo() { return 0; }", 25));
-                      "template<> unsigned A<int, v<char> >::foo() { return 0; }", 30, /*onlyCreateTokens=*/true));
+        ASSERT_EQUALS(12, templateNamePositionHelper(
+                          "template<class T> class v {}; "
+                          "template<class T, class U> class A { unsigned foo(); }; "
+                          "template<> unsigned A<int, v<char> >::foo() { return 0; }", 30));
     }
 
     void expandSpecialized1() {