tokenize.cpp

cppcheck is a static C/C++ code analyzer that checks for memory leaks, mismatching allocation-deallo

    for (Token *tok = _tokens; tok; tok = tok->next())
        tok->varId(0);

    // Set variable ids..
    unsigned int _varId = 0;
    for (Token *tok = _tokens; tok; tok = tok->next())
    {
        if (tok != _tokens && !Token::Match(tok, "[;{}(]"))
            continue;

        if (Token::Match(tok, "[;{}(] %type%"))
            tok = tok->next();

        if (Token::Match(tok, "else|return"))
            continue;

        // Determine name of declared variable..
        const char *varname = 0;
        Token *tok2 = tok->tokAt(1);
        while (tok2 && ! Token::Match(tok2, "[;[=(]"))
        {
            if (tok2->isName())
                varname = tok2->strAt(0);
            else if (tok2->str() != "*")
                break;
            tok2 = tok2->next();
        }

        // Variable declaration found => Set variable ids
        if (Token::Match(tok2, "[;[=]") && varname)
        {
            ++_varId;
            int indentlevel = 0;
            int parlevel = 0;
            bool dot = false;
            for (tok2 = tok->next(); tok2; tok2 = tok2->next())
            {
                if (!dot && tok2->str() == varname)
                    tok2->varId(_varId);
                else if (tok2->str() == "{")
                    ++indentlevel;
                else if (tok2->str() == "}")
                {
                    --indentlevel;
                    if (indentlevel < 0)
                        break;
                }
                else if (tok2->str() == "(")
                    ++parlevel;
                else if (tok2->str() == ")")
                {
                    // Is this a function parameter or a variable declared in for example a for loop?
                    if (parlevel == 0 && indentlevel == 0 && Token::Match(tok2, ") const| {"))
                        ;
                    else
                        --parlevel;
                }
                else if (parlevel < 0 && tok2->str() == ";")
                    break;
                dot = bool(tok2->str() == ".");
            }
        }
    }

    // Struct/Class members
    for (Token *tok = _tokens; tok; tok = tok->next())
    {
        if (tok->varId() != 0 &&
            Token::Match(tok->next(), ". %var%") &&
            tok->tokAt(2)->varId() == 0)
        {
            ++_varId;

            const std::string pattern(std::string(". ") + tok->strAt(2));
            for (Token *tok2 = tok; tok2; tok2 = tok2->next())
            {
                if (tok2->varId() == tok->varId() && Token::simpleMatch(tok2->next(), pattern.c_str()))
                    tok2->next()->next()->varId(_varId);
            }
        }
    }
}


//---------------------------------------------------------------------------
// Simplify token list
//---------------------------------------------------------------------------

void Tokenizer::simplifyTokenList()
{
    // Combine strings
    for (Token *tok = _tokens; tok; tok = tok->next())
    {
        while (tok->str()[0] == '"' && tok->next() && tok->next()->str()[0] == '"')
        {
            // Two strings after each other, combine them
            std::string temp = tok->str();
            temp.erase(temp.length() - 1);
            temp.append(tok->next()->str().substr(1));
            tok->str(temp.c_str());
            tok->deleteNext();
        }
    }

    // Remove unwanted keywords
    static const char* unwantedWords[] = { "unsigned", "unlikely" };
    for (Token *tok = _tokens; tok; tok = tok->next())
    {
        for (unsigned ui = 0; ui < sizeof(unwantedWords) / sizeof(unwantedWords[0]) && tok->next(); ui++)
        {
            if (tok->next()->str() == unwantedWords[ui])
            {
                tok->deleteNext();
                break;
            }
        }
    }

    // Convert + + into + and + - into -
    for (Token *tok = _tokens; tok; tok = tok->next())
    {
        while (tok->next())
        {
            if (tok->str() == "+")
            {
                if (tok->next()->str() == "+")
                {
                    tok->deleteNext();
                    continue;
                }
                else if (tok->next()->str() == "-")
                {
                    tok->str("-");
                    tok->deleteNext();
                    continue;
                }
            }
            else if (tok->str() == "-")
            {
                if (tok->next()->str() == "-")
                {
                    tok->str("+");
                    tok->deleteNext();
                    continue;
                }
                else if (tok->next()->str() == "+")
                {
                    tok->deleteNext();
                    continue;
                }
            }

            break;
        }
    }



    // Fill the map _typeSize..
    _typeSize.clear();
    _typeSize["char"] = sizeof(char);
    _typeSize["short"] = sizeof(short);
    _typeSize["int"] = sizeof(int);
    _typeSize["long"] = sizeof(long);
    _typeSize["float"] = sizeof(float);
    _typeSize["double"] = sizeof(double);
    for (Token *tok = _tokens; tok; tok = tok->next())
    {
        if (Token::Match(tok, "class|struct %var%"))
        {
            _typeSize[tok->strAt(1)] = 100;
        }
    }


    // Replace 'sizeof(var)'..
    for (Token *tok = _tokens; tok; tok = tok->next())
    {
        if (Token::Match(tok, "[;{}] %type% %var% ;") && tok->tokAt(2)->varId() > 0)
        {
            const unsigned int varid = tok->tokAt(2)->varId();

            // Replace 'sizeof(var)' with 'sizeof(type)'
            int indentlevel = 0;
            for (Token *tok2 = tok; tok2; tok2 = tok2->next())
            {
                if (tok2->str() == "{")
                    ++indentlevel;
                else if (tok2->str() == "}")
                {
                    --indentlevel;
                    if (indentlevel < 0)
                        break;
                }
                else if (Token::Match(tok2, "sizeof ( %varid% )", varid))
                {
                    tok2 = tok2->tokAt(2);
                    tok2->str(tok->strAt(1));
                }
            }
        }
    }


    // Replace 'sizeof(type)'..
    for (Token *tok = _tokens; tok; tok = tok->next())
    {
        if (tok->str() != "sizeof")
            continue;

        if (tok->strAt(1) != std::string("("))
        {
            // Add parenthesis around the sizeof
            for (Token *tempToken = tok->next(); tempToken; tempToken = tempToken->next())
            {
                if (Token::Match(tempToken, "%var%"))
                {
                    if (Token::Match(tempToken->next(), "."))
                    {
                        // We are checking a class or struct, search next varname
                        tempToken = tempToken->tokAt(1);
                        continue;
                    }
                    else if (Token::Match(tempToken->next(), "- >"))
                    {
                        // We are checking a class or struct, search next varname
                        tempToken = tempToken->tokAt(2);
                        continue;
                    }
                    else if (Token::Match(tempToken->next(), "++") ||
                             Token::Match(tempToken->next(), "--"))
                    {
                        // We have variable++ or variable--, there should be
                        // nothing after this
                        tempToken = tempToken->tokAt(2);
                    }
                    else if (Token::Match(tempToken->next(), "["))
                    {
                        // TODO: We need to find closing ], then check for
                        // dots and arrows "var[some[0]]->other"

                        // But for now, just bail out
                        break;
                    }

                    // Ok, we should be clean. Add ) after tempToken
                    tok->insertToken("(");
                    tempToken->insertToken(")");
                    break;
                }
            }
        }

        if (Token::Match(tok, "sizeof ( %type% * )"))
        {
            std::ostringstream str;
            // 'sizeof(type *)' has the same size as 'sizeof(char *)'
            str << sizeof(char *);
            tok->str(str.str().c_str());

            for (int i = 0; i < 4; i++)
            {
                tok->deleteNext();
            }
        }

        else if (Token::Match(tok, "sizeof ( %type% )"))
        {
            const char *type = tok->strAt(2);
            int size = SizeOfType(type);
            if (size > 0)
            {
                std::ostringstream str;
                str << size;
                tok->str(str.str().c_str());
                for (int i = 0; i < 3; i++)
                {
                    tok->deleteNext();
                }
            }
        }

        else if (Token::Match(tok, "sizeof ( * %var% )") || Token::Match(tok, "sizeof ( %var% [ %num% ] )"))
        {
            // Some default value..
            int sz = 100;

            unsigned int varid = tok->tokAt((tok->tokAt(2)->str() == "*") ? 3 : 2)->varId();
            if (varid != 0)
            {
                // Try to locate variable declaration..
                const Token *decltok = Token::findmatch(_tokens, "%type% %varid% [", varid);
                if (decltok)
                {
                    sz = SizeOfType(decltok->strAt(0));
                }
            }

            std::ostringstream ostr;
            ostr << sz;
            tok->str(ostr.str().c_str());
            while (tok->next()->str() != ")")
                tok->deleteNext();
            tok->deleteNext();
        }
    }

    // Replace 'sizeof(var)'
    for (Token *tok = _tokens; tok; tok = tok->next())
    {
        // type array [ num ] ;
        if (! Token::Match(tok, "%type% %var% [ %num% ] ;"))
            continue;

        int size = SizeOfType(tok->aaaa());
        if (size <= 0)
            continue;

        const unsigned int varid = tok->next()->varId();
        if (varid == 0)
            continue;

        int total_size = size * std::atoi(tok->strAt(3));

        // Replace 'sizeof(var)' with number
        int indentlevel = 0;
        for (Token *tok2 = tok->tokAt(5); tok2; tok2 = tok2->next())
        {
            if (tok2->str() == "{")
            {
                ++indentlevel;
            }

            else if (tok2->str() == "}")
            {
                --indentlevel;
                if (indentlevel < 0)
                    break;
            }

            else if (Token::Match(tok2, "sizeof ( %varid% )", varid))
            {
                std::ostringstream str;
                str << total_size;
                tok2->str(str.str().c_str());
                // Delete the other tokens..
                for (int i = 0; i < 3; i++)
                {
                    tok2->deleteNext();
                }
            }
        }
    }

    // Replace constants..
    for (Token *tok = _tokens; tok; tok = tok->next())
    {
        if (Token::Match(tok, "const %type% %var% = %num% ;"))
        {
            const char *sym = tok->strAt(2);
            const char *num = tok->strAt(4);
            int indent = 1;
            for (Token *tok2 = tok->tokAt(6); tok2; tok2 = tok2->next())
            {
                if (tok2->str() == "{")
                {
                    ++indent;
                }
                else if (tok2->str() == "}")
                {
                    --indent;
                    if (indent == 0)
                        break;
                }

                // Compare constants, but don't touch members of other structures
                else if (tok2->str() == sym &&
                         tok2->previous() &&
                         tok2->previous()->str() != ".")
                {
                    tok2->str(num);
                }
            }
        }
    }

    simplifyCasts();

    // Simplify simple calculations..
    while (simplifyCalculations())
        ;

    // Replace "*(str + num)" => "str[num]"
    for (Token *tok = _tokens; tok; tok = tok->next())
    {
        if (! strchr(";{}(=<>", tok->aaaa0()))
            continue;

        Token *next = tok->next();
        if (! next)
            break;

        if (Token::Match(next, "* ( %var% + %num% )"))
        {
            const char *str[4] = {"var", "[", "num", "]"};
            str[0] = tok->strAt(3);
            str[2] = tok->strAt(5);

            for (int i = 0; i < 4; i++)
            {
                tok = tok->next();
                tok->str(str[i]);
            }

            tok->deleteNext();
            tok->deleteNext();
        }
    }



    // Split up variable declarations if possible..
    for (Token *tok = _tokens; tok; tok = tok->next())
    {
        if (! Token::Match(tok, "[{};]"))
            continue;

        Token *type0 = tok->next();
        if (!Token::Match(type0, "%type%"))
            continue;
        if (Token::Match(type0, "else|return"))
            continue;

        Token *tok2 = NULL;
        unsigned int typelen = 0;

        if (Token::Match(type0, "%type% %var% ,|="))
        {
            if (type0->next()->str() != "operator")
            {
                tok2 = type0->tokAt(2);    // The ',' or '=' token
                typelen = 1;
            }
        }

        else if (Token::Match(type0, "%type% * %var% ,|="))
        {
            if (type0->next()->next()->str() != "operator")
            {
                tok2 = type0->tokAt(3);    // The ',' token
                typelen = 1;
            }
        }

        else if (Token::Match(type0, "%type% %var% [ %num% ] ,"))
        {
            tok2 = type0->tokAt(5);    // The ',' token
            typelen = 1;
        }

        else if (Token::Match(type0, "%type% * %var% [ %num% ] ,"))
        {
            tok2 = type0->tokAt(6);    // The ',' token
            typelen = 1;
        }

        else if (Token::Match(type0, "struct %type% %var% ,|="))
        {
            tok2 = type0->tokAt(3);
            typelen = 2;
        }

        else if (Token::Match(type0, "struct %type% * %var% ,|="))
        {
            tok2 = type0->tokAt(4);
            typelen = 2;
        }


        if (tok2)
        {
            if (tok2->str() == ",")
            {
                tok2->str(";");
                InsertTokens(tok2, type0, typelen);
            }

            else
            {
                Token *eq = tok2;

                int parlevel = 0;