Tokenizer.l

Branch :

Show log
Commit
Author : Nico Weber
Date : 2018-02-09 10:01:32
Hash : 41b072b6
Message : Prepare for -Wimplicit-fallthrough in ANGLE. Disable the warning for flex-generated output, which contains lots of intentional fallthrough. Fixes a bug where GL_SAMPLE_ALPHA_TO_ONE_EXT would fall through to GL_COVERAGE_MODULATION_CHROMIUM and hence behave like that. Fixes a bug in the D3D9 state management where invalidating DIRTY_BIT_POLYGON_OFFSET would also invalidate the stencil bits. One somewhat common incorrect pattern in ANGLE is nested switch statements that look like so: switch (a) { case a1: switch (b) { case b1: ... break; } case a2: ... } The assumption here seems to be that the breakk exits the outer case (here a1), while it in fact only exits the inner switch, so that we fall through to a2. In most places, I fixed this by adding an explicit `break` after the inner switch. This fixes a bug wher GL_PATH_JOIN_STYLE_CHROMIUM would fall through to GL_PATH_MITER_LIMIT_CHROMIUM in validation (but since the join style enum is always > 0, this happened to not have an effect in practice). This also fixes 87 bugs in GetLoadFunctionsMap() where invalid values would previously return an unrelated function map instead of the empty load function map. Bug: chromium:810767 Change-Id: Ib51388c73fbfc229160e2c10f8fb9364cc7c996c Reviewed-on: https://chromium-review.googlesource.com/911529 Commit-Queue: Nico Weber <thakis@chromium.org> Reviewed-by: Geoff Lang <geofflang@chromium.org>

src/compiler/preprocessor/Tokenizer.l

/*
//
// Copyright (c) 2002-2014 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//

This file contains the Lex specification for GLSL ES preprocessor.
Based on Microsoft Visual Studio 2010 Preprocessor Grammar:
http://msdn.microsoft.com/en-us/library/2scxys89.aspx

IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
*/

%top{
//
// Copyright (c) 2011-2014 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//

// This file is auto-generated by generate_parser.sh. DO NOT EDIT!
}

%{
#if defined(_MSC_VER)
#pragma warning(disable: 4005)
#endif

#include "compiler/preprocessor/Tokenizer.h"

#include "compiler/preprocessor/DiagnosticsBase.h"
#include "compiler/preprocessor/Token.h"

#if defined(__GNUC__)
// Triggered by the auto-generated yy_fatal_error function.
#pragma GCC diagnostic ignored "-Wmissing-noreturn"
#elif defined(_MSC_VER)
#pragma warning(disable: 4244)
#endif
#if defined(__clang__)
// Flex uses `/*FALLTHROUGH*/` instead of dedicated statements.
#pragma GCC diagnostic ignored "-Wimplicit-fallthrough"
#endif

// Workaround for flex using the register keyword, deprecated in C++11.
#ifdef __cplusplus
#if __cplusplus > 199711L
#define register
#endif
#endif

typedef std::string YYSTYPE;
typedef pp::SourceLocation YYLTYPE;

// Use the unused yycolumn variable to track file (string) number.
#define yyfileno yycolumn

#define YY_USER_INIT                   \
    do {                               \
        yyfileno = 0;                  \
        yylineno = 1;                  \
        yyextra->leadingSpace = false; \
        yyextra->lineStart = true;     \
    } while(0);

#define YY_USER_ACTION                                              \
    do                                                              \
    {                                                               \
        pp::Input* input = &yyextra->input;                         \
        pp::Input::Location* scanLoc = &yyextra->scanLoc;           \
        while ((scanLoc->sIndex < input->count()) &&                \
               (scanLoc->cIndex >= input->length(scanLoc->sIndex))) \
        {                                                           \
            scanLoc->cIndex -= input->length(scanLoc->sIndex++);    \
            ++yyfileno; yylineno = 1;                               \
        }                                                           \
        yylloc->file = yyfileno;                                    \
        yylloc->line = yylineno;                                    \
        scanLoc->cIndex += yyleng;                                  \
    } while(0);

#define YY_INPUT(buf, result, maxSize) \
    result = yyextra->input.read(buf, maxSize, &yylineno);

%}

%option noyywrap nounput never-interactive
%option reentrant bison-bridge bison-locations
%option prefix="pp"
%option extra-type="pp::Tokenizer::Context*"
%x COMMENT

NEWLINE     \n|\r|\r\n
IDENTIFIER  [_a-zA-Z][_a-zA-Z0-9]*
PUNCTUATOR  [][<>(){}.+-/*%^|&~=!:;,?]

DECIMAL_CONSTANT      [1-9][0-9]*[uU]?
OCTAL_CONSTANT        0[0-7]*[uU]?
HEXADECIMAL_CONSTANT  0[xX][0-9a-fA-F]+[uU]?

DIGIT                [0-9]
EXPONENT_PART        [eE][+-]?{DIGIT}+
FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")

%%

    /* Line comment */
"//"[^\r\n]*

    /* Block comment */
    /* Line breaks are just counted - not returned. */
    /* The comment is replaced by a single space. */
"/*" { BEGIN(COMMENT); }
<COMMENT>[^*\r\n]+
<COMMENT>"*"
<COMMENT>{NEWLINE} {
    if (yylineno == INT_MAX)
    {
        *yylval = "Integer overflow on line number";
        return pp::Token::GOT_ERROR;
    }
    ++yylineno;
}
<COMMENT>"*/" {
    yyextra->leadingSpace = true;
    BEGIN(INITIAL);
}

# {
    // # is only valid at start of line for preprocessor directives.
    yylval->assign(1, yytext[0]);
    return yyextra->lineStart ? pp::Token::PP_HASH : pp::Token::PP_OTHER;
}

{IDENTIFIER} {
    yylval->assign(yytext, yyleng);
    return pp::Token::IDENTIFIER;
}

({DECIMAL_CONSTANT}[uU]?)|({OCTAL_CONSTANT}[uU]?)|({HEXADECIMAL_CONSTANT}[uU]?) {
    yylval->assign(yytext, yyleng);
    return pp::Token::CONST_INT;
}

({DIGIT}+{EXPONENT_PART}[fF]?)|({FRACTIONAL_CONSTANT}{EXPONENT_PART}?[fF]?) {
    yylval->assign(yytext, yyleng);
    return pp::Token::CONST_FLOAT;
}

    /* Anything that starts with a {DIGIT} or .{DIGIT} must be a number. */
    /* Rule to catch all invalid integers and floats. */
({DIGIT}+[_a-zA-Z0-9.]*)|("."{DIGIT}+[_a-zA-Z0-9.]*) {
    yylval->assign(yytext, yyleng);
    return pp::Token::PP_NUMBER;
}

"++" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_INC;
}
"--" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_DEC;
}
"<<" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_LEFT;
}
">>" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_RIGHT;
}
"<=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_LE;
}
">=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_GE;
}
"==" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_EQ;
}
"!=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_NE;
}
"&&" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_AND;
}
"^^" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_XOR;
}
"||" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_OR;
}
"+=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_ADD_ASSIGN;
}
"-=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_SUB_ASSIGN;
}
"*=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_MUL_ASSIGN;
}
"/=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_DIV_ASSIGN;
}
"%=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_MOD_ASSIGN;
}
"<<=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_LEFT_ASSIGN;
}
">>=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_RIGHT_ASSIGN;
}
"&=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_AND_ASSIGN;
}
"^=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_XOR_ASSIGN;
}
"|=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_OR_ASSIGN;
}

{PUNCTUATOR} {
    yylval->assign(1, yytext[0]);
    return yytext[0];
}

[ \t\v\f]+   { yyextra->leadingSpace = true; }

{NEWLINE} {
    if (yylineno == INT_MAX)
    {
        *yylval = "Integer overflow on line number";
        return pp::Token::GOT_ERROR;
    }
    ++yylineno;
    yylval->assign(1, '\n');
    return '\n';
}

. {
    yylval->assign(1, yytext[0]);
    return pp::Token::PP_OTHER;
}

<*><<EOF>> {
    // YY_USER_ACTION is not invoked for handling EOF.
    // Set the location for EOF token manually.
    pp::Input* input = &yyextra->input;
    pp::Input::Location* scanLoc = &yyextra->scanLoc;
    yy_size_t sIndexMax = input->count() ? input->count() - 1 : 0;
    if (scanLoc->sIndex != sIndexMax)
    {
        // We can only reach here if there are empty strings at the
        // end of the input.
        scanLoc->sIndex = sIndexMax; scanLoc->cIndex = 0;
        // FIXME: this is not 64-bit clean.
        yyfileno = static_cast<int>(sIndexMax); yylineno = 1;
    }
    yylloc->file = yyfileno;
    yylloc->line = yylineno;
    yylval->clear();

    // Line number overflows fake EOFs to exit early, check for this case.
    if (yylineno == INT_MAX) {
        yyextra->diagnostics->report(pp::Diagnostics::PP_TOKENIZER_ERROR,
                pp::SourceLocation(yyfileno, yylineno),
                "Integer overflow on line number");
    }
    else if (YY_START == COMMENT)
    {
        yyextra->diagnostics->report(pp::Diagnostics::PP_EOF_IN_COMMENT,
                                     pp::SourceLocation(yyfileno, yylineno),
                                     "EOF while in a comment");
    }
    yyterminate();
}

%%

namespace pp {

Tokenizer::Tokenizer(Diagnostics *diagnostics) : mHandle(nullptr), mMaxTokenSize(256)
{
    mContext.diagnostics = diagnostics;
}

Tokenizer::~Tokenizer()
{
    destroyScanner();
}

bool Tokenizer::init(size_t count, const char * const string[], const int length[])
{
    if ((count > 0) && (string == 0))
        return false;

    mContext.input = Input(count, string, length);
    return initScanner();
}

void Tokenizer::setFileNumber(int file)
{
    // We use column number as file number.
    // See macro yyfileno.
    yyset_column(file, mHandle);
}

void Tokenizer::setLineNumber(int line)
{
    yyset_lineno(line, mHandle);
}

void Tokenizer::setMaxTokenSize(size_t maxTokenSize)
{
    mMaxTokenSize = maxTokenSize;
}

void Tokenizer::lex(Token *token)
{
    int tokenType = yylex(&token->text, &token->location, mHandle);

    if (tokenType == Token::GOT_ERROR)
    {
        mContext.diagnostics->report(Diagnostics::PP_TOKENIZER_ERROR, token->location, token->text);
        token->type = Token::LAST;
    }
    else
    {
        token->type = tokenType;
    }

    if (token->text.size() > mMaxTokenSize)
    {
        mContext.diagnostics->report(Diagnostics::PP_TOKEN_TOO_LONG,
                                     token->location, token->text);
        token->text.erase(mMaxTokenSize);
    }

    token->flags = 0;

    token->setAtStartOfLine(mContext.lineStart);
    mContext.lineStart = token->type == '\n';

    token->setHasLeadingSpace(mContext.leadingSpace);
    mContext.leadingSpace = false;
}

bool Tokenizer::initScanner()
{
    if ((mHandle == nullptr) && yylex_init_extra(&mContext, &mHandle))
        return false;

    yyrestart(0, mHandle);
    return true;
}

void Tokenizer::destroyScanner()
{
    if (mHandle == nullptr)
        return;

    yylex_destroy(mHandle);
    mHandle = nullptr;
}

}  // namespace pp

kc3-lang/angle/src/compiler/preprocessor/Tokenizer.l

Commit

kc3-lang/angle /src/compiler/preprocessor/Tokenizer.l