MacroExpander.cpp

Branch :
Show log
Commit
Author : Olli Etuaho
Date : 2015-09-18 10:34:31
Hash : 261f5379
Message : Support parsing defined operator generated by macro expansion dEQP tests enforce that the defined operator should be parsed even when it is generated as a result of macro expansion, even though this is undefined according to the C++ preprocessor spec. Implement support for this by putting the parsing for the defined operator inside MacroExpander. The operator gets processed right after it is generated by macro expansion. Parsing the defined operator is toggled with a boolean according to the context where MacroExpander is used. BUG=angleproject:989 TEST=angle_unittests, dEQP-GLES3.functional.shaders.preprocessor.* - 2 tests start passing: dEQP-GLES3.functional.shaders.preprocessor.conditional_inclusion.basic_2* Change-Id: I780e63bd4558253657d898685d62339017564a06 Reviewed-on: https://chromium-review.googlesource.com/300970 Reviewed-by: Jamie Madill <jmadill@chromium.org> Tested-by: Olli Etuaho <oetuaho@nvidia.com>
src/compiler/preprocessor/MacroExpander.cpp
//
// Copyright (c) 2011 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//

#include "MacroExpander.h"

#include <algorithm>
#include <sstream>

#include "DiagnosticsBase.h"
#include "Token.h"

namespace pp
{

class TokenLexer : public Lexer
{
 public:
    typedef std::vector<Token> TokenVector;

    TokenLexer(TokenVector *tokens)
    {
        tokens->swap(mTokens);
        mIter = mTokens.begin();
    }

    void lex(Token *token) override
    {
        if (mIter == mTokens.end())
        {
            token->reset();
            token->type = Token::LAST;
        }
        else
        {
            *token = *mIter++;
        }
    }

 private:
    PP_DISALLOW_COPY_AND_ASSIGN(TokenLexer);

    TokenVector mTokens;
    TokenVector::const_iterator mIter;
};

MacroExpander::MacroExpander(Lexer *lexer,
                             MacroSet *macroSet,
                             Diagnostics *diagnostics,
                             bool parseDefined)
    : mLexer(lexer), mMacroSet(macroSet), mDiagnostics(diagnostics), mParseDefined(parseDefined)
{
}

MacroExpander::~MacroExpander()
{
    for (std::size_t i = 0; i < mContextStack.size(); ++i)
    {
        delete mContextStack[i];
    }
}

void MacroExpander::lex(Token *token)
{
    while (true)
    {
        const char kDefined[] = "defined";

        getToken(token);

        if (token->type != Token::IDENTIFIER)
            break;

        // Defined operator is parsed here since it may be generated by macro expansion.
        // Defined operator produced by macro expansion has undefined behavior according to C++
        // spec, which the GLSL spec references (see C++14 draft spec section 16.1.4), but this
        // behavior is needed for passing dEQP tests, which enforce stricter compatibility between
        // implementations.
        if (mParseDefined && token->text == kDefined)
        {
            bool paren = false;
            getToken(token);
            if (token->type == '(')
            {
                paren = true;
                getToken(token);
            }
            if (token->type != Token::IDENTIFIER)
            {
                mDiagnostics->report(Diagnostics::PP_UNEXPECTED_TOKEN, token->location,
                                     token->text);
                break;
            }
            auto iter              = mMacroSet->find(token->text);
            std::string expression = iter != mMacroSet->end() ? "1" : "0";

            if (paren)
            {
                getToken(token);
                if (token->type != ')')
                {
                    mDiagnostics->report(Diagnostics::PP_UNEXPECTED_TOKEN, token->location,
                                         token->text);
                    break;
                }
            }

            // We have a valid defined operator.
            // Convert the current token into a CONST_INT token.
            token->type = Token::CONST_INT;
            token->text = expression;
            break;
        }

        if (token->expansionDisabled())
            break;

        MacroSet::const_iterator iter = mMacroSet->find(token->text);
        if (iter == mMacroSet->end())
            break;

        const Macro& macro = iter->second;
        if (macro.disabled)
        {
            // If a particular token is not expanded, it is never expanded.
            token->setExpansionDisabled(true);
            break;
        }
        if ((macro.type == Macro::kTypeFunc) && !isNextTokenLeftParen())
        {
            // If the token immediately after the macro name is not a '(',
            // this macro should not be expanded.
            break;
        }

        pushMacro(macro, *token);
    }
}

void MacroExpander::getToken(Token *token)
{
    if (mReserveToken.get())
    {
        *token = *mReserveToken;
        mReserveToken.reset();
        return;
    }

    // First pop all empty macro contexts.
    while (!mContextStack.empty() && mContextStack.back()->empty())
    {
        popMacro();
    }

    if (!mContextStack.empty())
    {
        *token = mContextStack.back()->get();
    }
    else
    {
        mLexer->lex(token);
    }
}

void MacroExpander::ungetToken(const Token &token)
{
    if (!mContextStack.empty())
    {
        MacroContext *context = mContextStack.back();
        context->unget();
        assert(context->replacements[context->index] == token);
    }
    else
    {
        assert(!mReserveToken.get());
        mReserveToken.reset(new Token(token));
    }
}

bool MacroExpander::isNextTokenLeftParen()
{
    Token token;
    getToken(&token);

    bool lparen = token.type == '(';
    ungetToken(token);

    return lparen;
}

bool MacroExpander::pushMacro(const Macro &macro, const Token &identifier)
{
    assert(!macro.disabled);
    assert(!identifier.expansionDisabled());
    assert(identifier.type == Token::IDENTIFIER);
    assert(identifier.text == macro.name);

    std::vector<Token> replacements;
    if (!expandMacro(macro, identifier, &replacements))
        return false;

    // Macro is disabled for expansion until it is popped off the stack.
    macro.disabled = true;

    MacroContext *context = new MacroContext;
    context->macro = &macro;
    context->replacements.swap(replacements);
    mContextStack.push_back(context);
    return true;
}

void MacroExpander::popMacro()
{
    assert(!mContextStack.empty());

    MacroContext *context = mContextStack.back();
    mContextStack.pop_back();

    assert(context->empty());
    assert(context->macro->disabled);
    context->macro->disabled = false;
    delete context;
}

bool MacroExpander::expandMacro(const Macro &macro,
                                const Token &identifier,
                                std::vector<Token> *replacements)
{
    replacements->clear();

    // In the case of an object-like macro, the replacement list gets its location
    // from the identifier, but in the case of a function-like macro, the replacement
    // list gets its location from the closing parenthesis of the macro invocation.
    // This is tested by dEQP-GLES3.functional.shaders.preprocessor.predefined_macros.*
    SourceLocation replacementLocation = identifier.location;
    if (macro.type == Macro::kTypeObj)
    {
        replacements->assign(macro.replacements.begin(),
                             macro.replacements.end());

        if (macro.predefined)
        {
            const char kLine[] = "__LINE__";
            const char kFile[] = "__FILE__";

            assert(replacements->size() == 1);
            Token& repl = replacements->front();
            if (macro.name == kLine)
            {
                std::ostringstream stream;
                stream << identifier.location.line;
                repl.text = stream.str();
            }
            else if (macro.name == kFile)
            {
                std::ostringstream stream;
                stream << identifier.location.file;
                repl.text = stream.str();
            }
        }
    }
    else
    {
        assert(macro.type == Macro::kTypeFunc);
        std::vector<MacroArg> args;
        args.reserve(macro.parameters.size());
        if (!collectMacroArgs(macro, identifier, &args, &replacementLocation))
            return false;

        replaceMacroParams(macro, args, replacements);
    }

    for (std::size_t i = 0; i < replacements->size(); ++i)
    {
        Token& repl = replacements->at(i);
        if (i == 0)
        {
            // The first token in the replacement list inherits the padding
            // properties of the identifier token.
            repl.setAtStartOfLine(identifier.atStartOfLine());
            repl.setHasLeadingSpace(identifier.hasLeadingSpace());
        }
        repl.location = replacementLocation;
    }
    return true;
}

bool MacroExpander::collectMacroArgs(const Macro &macro,
                                     const Token &identifier,
                                     std::vector<MacroArg> *args,
                                     SourceLocation *closingParenthesisLocation)
{
    Token token;
    getToken(&token);
    assert(token.type == '(');

    args->push_back(MacroArg());
    for (int openParens = 1; openParens != 0; )
    {
        getToken(&token);

        if (token.type == Token::LAST)
        {
            mDiagnostics->report(Diagnostics::PP_MACRO_UNTERMINATED_INVOCATION,
                                 identifier.location, identifier.text);
            // Do not lose EOF token.
            ungetToken(token);
            return false;
        }

        bool isArg = false; // True if token is part of the current argument.
        switch (token.type)
        {
          case '(':
            ++openParens;
            isArg = true;
            break;
          case ')':
            --openParens;
            isArg = openParens != 0;
            *closingParenthesisLocation = token.location;
            break;
          case ',':
            // The individual arguments are separated by comma tokens, but
            // the comma tokens between matching inner parentheses do not
            // seperate arguments.
            if (openParens == 1)
                args->push_back(MacroArg());
            isArg = openParens != 1;
            break;
          default:
            isArg = true;
            break;
        }
        if (isArg)
        {
            MacroArg &arg = args->back();
            // Initial whitespace is not part of the argument.
            if (arg.empty())
                token.setHasLeadingSpace(false);
            arg.push_back(token);
        }
    }

    const Macro::Parameters &params = macro.parameters;
    // If there is only one empty argument, it is equivalent to no argument.
    if (params.empty() && (args->size() == 1) && args->front().empty())
    {
        args->clear();
    }
    // Validate the number of arguments.
    if (args->size() != params.size())
    {
        Diagnostics::ID id = args->size() < macro.parameters.size() ?
            Diagnostics::PP_MACRO_TOO_FEW_ARGS :
            Diagnostics::PP_MACRO_TOO_MANY_ARGS;
        mDiagnostics->report(id, identifier.location, identifier.text);
        return false;
    }

    // Pre-expand each argument before substitution.
    // This step expands each argument individually before they are
    // inserted into the macro body.
    for (std::size_t i = 0; i < args->size(); ++i)
    {
        MacroArg &arg = args->at(i);
        TokenLexer lexer(&arg);
        MacroExpander expander(&lexer, mMacroSet, mDiagnostics, mParseDefined);

        arg.clear();
        expander.lex(&token);
        while (token.type != Token::LAST)
        {
            arg.push_back(token);
            expander.lex(&token);
        }
    }
    return true;
}

void MacroExpander::replaceMacroParams(const Macro &macro,
                                       const std::vector<MacroArg> &args,
                                       std::vector<Token> *replacements)
{
    for (std::size_t i = 0; i < macro.replacements.size(); ++i)
    {
        const Token &repl = macro.replacements[i];
        if (repl.type != Token::IDENTIFIER)
        {
            replacements->push_back(repl);
            continue;
        }

        // TODO(alokp): Optimize this.
        // There is no need to search for macro params every time.
        // The param index can be cached with the replacement token.
        Macro::Parameters::const_iterator iter = std::find(
            macro.parameters.begin(), macro.parameters.end(), repl.text);
        if (iter == macro.parameters.end())
        {
            replacements->push_back(repl);
            continue;
        }

        std::size_t iArg = std::distance(macro.parameters.begin(), iter);
        const MacroArg &arg = args[iArg];
        if (arg.empty())
        {
            continue;
        }
        std::size_t iRepl = replacements->size();
        replacements->insert(replacements->end(), arg.begin(), arg.end());
        // The replacement token inherits padding properties from
        // macro replacement token.
        replacements->at(iRepl).setHasLeadingSpace(repl.hasLeadingSpace());
    }
}

}  // namespace pp
kc3-lang/angle/src/compiler/preprocessor/MacroExpander.cpp

Commit

kc3-lang/angle /src/compiler/preprocessor/MacroExpander.cpp
