Tokenizer.l

Branch :

Show log
Commit
Author : Jamie Madill
Date : 2015-08-17 16:51:33
Hash : 3b040eb8
Message : Revert "Add full support for line continuation in the preprocessor" Warning in the Linux/Mac builders: In file included from ../../third_party/angle/src/tests/preprocessor_tests/input_test.cpp:7: In file included from ../../third_party/angle/src/tests/preprocessor_tests/PreprocessorTest.h:7: ../../testing/gtest/include/gtest/gtest.h:1392:16: error: comparison of integers of different signs: 'const int' and 'const unsigned long' [-Werror,-Wsign-compare] if (expected == actual) { ~~~~~~~~ ^ ~~~~~~ ../../testing/gtest/include/gtest/gtest.h:1422:12: note: in instantiation of function template specialization 'testing::internal::CmpHelperEQ<int, unsigned long>' requested here return CmpHelperEQ(expected_expression, actual_expression, expected, ^ ../../third_party/angle/src/tests/preprocessor_tests/input_test.cpp:171:5: note: in instantiation of function template specialization 'testing::internal::EqHelper<false>::Compare<int, unsigned long>' requested here EXPECT_EQ(3, input.read(buf, maxSize, &lineNo)); ^ BUG=angleproject:1125 This reverts commit c1157d1963170c7411eb6c32e2b2fbce02c5a170. Change-Id: Ic6fa286d190b006cccc5154d86e21ecc03175763 Reviewed-on: https://chromium-review.googlesource.com/294080 Reviewed-by: Jamie Madill <jmadill@chromium.org> Tested-by: Jamie Madill <jmadill@chromium.org>

src/compiler/preprocessor/Tokenizer.l

/*
//
// Copyright (c) 2002-2014 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//

This file contains the Lex specification for GLSL ES preprocessor.
Based on Microsoft Visual Studio 2010 Preprocessor Grammar:
http://msdn.microsoft.com/en-us/library/2scxys89.aspx

IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
*/

%top{
//
// Copyright (c) 2011-2014 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//

// This file is auto-generated by generate_parser.sh. DO NOT EDIT!
}

%{
#if defined(_MSC_VER)
#pragma warning(disable: 4005)
#endif

#include "Tokenizer.h"

#include "DiagnosticsBase.h"
#include "Token.h"

#if defined(__GNUC__)
// Triggered by the auto-generated yy_fatal_error function.
#pragma GCC diagnostic ignored "-Wmissing-noreturn"
#elif defined(_MSC_VER)
#pragma warning(disable: 4244)
#endif

// Workaround for flex using the register keyword, deprecated in C++11.
#ifdef __cplusplus
#if __cplusplus > 199711L
#define register
#endif
#endif

typedef std::string YYSTYPE;
typedef pp::SourceLocation YYLTYPE;

// Use the unused yycolumn variable to track file (string) number.
#define yyfileno yycolumn

#define YY_USER_INIT                   \
    do {                               \
        yyfileno = 0;                  \
        yylineno = 1;                  \
        yyextra->leadingSpace = false; \
        yyextra->lineStart = true;     \
    } while(0);

#define YY_USER_ACTION                                              \
    do                                                              \
    {                                                               \
        pp::Input* input = &yyextra->input;                         \
        pp::Input::Location* scanLoc = &yyextra->scanLoc;           \
        while ((scanLoc->sIndex < input->count()) &&                \
               (scanLoc->cIndex >= input->length(scanLoc->sIndex))) \
        {                                                           \
            scanLoc->cIndex -= input->length(scanLoc->sIndex++);    \
            ++yyfileno; yylineno = 1;                               \
        }                                                           \
        yylloc->file = yyfileno;                                    \
        yylloc->line = yylineno;                                    \
        scanLoc->cIndex += yyleng;                                  \
    } while(0);

#define YY_INPUT(buf, result, maxSize) \
    result = yyextra->input.read(buf, maxSize);

%}

%option noyywrap nounput never-interactive
%option reentrant bison-bridge bison-locations
%option prefix="pp"
%option extra-type="pp::Tokenizer::Context*"
%x COMMENT

NEWLINE     \n|\r|\r\n
IDENTIFIER  [_a-zA-Z][_a-zA-Z0-9]*
PUNCTUATOR  [][<>(){}.+-/*%^|&~=!:;,?]

DECIMAL_CONSTANT      [1-9][0-9]*[uU]?
OCTAL_CONSTANT        0[0-7]*[uU]?
HEXADECIMAL_CONSTANT  0[xX][0-9a-fA-F]+[uU]?

DIGIT                [0-9]
EXPONENT_PART        [eE][+-]?{DIGIT}+
FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")

%%

    /* Line comment */
"//"[^\r\n]*

    /* Block comment */
    /* Line breaks are just counted - not returned. */
    /* The comment is replaced by a single space. */
"/*" { BEGIN(COMMENT); }
<COMMENT>[^*\r\n]+
<COMMENT>"*"
<COMMENT>{NEWLINE} { ++yylineno; }
<COMMENT>"*/" {
    yyextra->leadingSpace = true;
    BEGIN(INITIAL);
}

# {
    // # is only valid at start of line for preprocessor directives.
    yylval->assign(1, yytext[0]);
    return yyextra->lineStart ? pp::Token::PP_HASH : pp::Token::PP_OTHER;
}

{IDENTIFIER} {
    yylval->assign(yytext, yyleng);
    return pp::Token::IDENTIFIER;
}

({DECIMAL_CONSTANT}[uU]?)|({OCTAL_CONSTANT}[uU]?)|({HEXADECIMAL_CONSTANT}[uU]?) {
    yylval->assign(yytext, yyleng);
    return pp::Token::CONST_INT;
}

({DIGIT}+{EXPONENT_PART}[fF]?)|({FRACTIONAL_CONSTANT}{EXPONENT_PART}?[fF]?) {
    yylval->assign(yytext, yyleng);
    return pp::Token::CONST_FLOAT;
}

    /* Anything that starts with a {DIGIT} or .{DIGIT} must be a number. */
    /* Rule to catch all invalid integers and floats. */
({DIGIT}+[_a-zA-Z0-9.]*)|("."{DIGIT}+[_a-zA-Z0-9.]*) {
    yylval->assign(yytext, yyleng);
    return pp::Token::PP_NUMBER;
}

"++" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_INC;
}
"--" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_DEC;
}
"<<" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_LEFT;
}
">>" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_RIGHT;
}
"<=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_LE;
}
">=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_GE;
}
"==" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_EQ;
}
"!=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_NE;
}
"&&" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_AND;
}
"^^" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_XOR;
}
"||" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_OR;
}
"+=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_ADD_ASSIGN;
}
"-=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_SUB_ASSIGN;
}
"*=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_MUL_ASSIGN;
}
"/=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_DIV_ASSIGN;
}
"%=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_MOD_ASSIGN;
}
"<<=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_LEFT_ASSIGN;
}
">>=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_RIGHT_ASSIGN;
}
"&=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_AND_ASSIGN;
}
"^=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_XOR_ASSIGN;
}
"|=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_OR_ASSIGN;
}

{PUNCTUATOR} {
    yylval->assign(1, yytext[0]);
    return yytext[0];
}

[ \t\v\f]+   { yyextra->leadingSpace = true; }

{NEWLINE} {
    ++yylineno;
    yylval->assign(1, '\n');
    return '\n';
}

\\{NEWLINE} { ++yylineno; }

. {
    yylval->assign(1, yytext[0]);
    return pp::Token::PP_OTHER;
}

<*><<EOF>> {
    // YY_USER_ACTION is not invoked for handling EOF.
    // Set the location for EOF token manually.
    pp::Input* input = &yyextra->input;
    pp::Input::Location* scanLoc = &yyextra->scanLoc;
    yy_size_t sIndexMax = input->count() ? input->count() - 1 : 0;
    if (scanLoc->sIndex != sIndexMax)
    {
        // We can only reach here if there are empty strings at the
        // end of the input.
        scanLoc->sIndex = sIndexMax; scanLoc->cIndex = 0;
        // FIXME: this is not 64-bit clean.
        yyfileno = static_cast<int>(sIndexMax); yylineno = 1;
    }
    yylloc->file = yyfileno;
    yylloc->line = yylineno;
    yylval->clear();

    if (YY_START == COMMENT)
    {
        yyextra->diagnostics->report(pp::Diagnostics::PP_EOF_IN_COMMENT,
                                     pp::SourceLocation(yyfileno, yylineno),
                                     "");
    }
    yyterminate();
}

%%

namespace pp {

Tokenizer::Tokenizer(Diagnostics *diagnostics)
    : mHandle(0),
      mMaxTokenSize(256)
{
    mContext.diagnostics = diagnostics;
}

Tokenizer::~Tokenizer()
{
    destroyScanner();
}

bool Tokenizer::init(size_t count, const char * const string[], const int length[])
{
    if ((count > 0) && (string == 0))
        return false;

    mContext.input = Input(count, string, length);
    return initScanner();
}

void Tokenizer::setFileNumber(int file)
{
    // We use column number as file number.
    // See macro yyfileno.
    yyset_column(file, mHandle);
}

void Tokenizer::setLineNumber(int line)
{
    yyset_lineno(line, mHandle);
}

void Tokenizer::setMaxTokenSize(size_t maxTokenSize)
{
    mMaxTokenSize = maxTokenSize;
}

void Tokenizer::lex(Token *token)
{
    token->type = yylex(&token->text, &token->location, mHandle);
    if (token->text.size() > mMaxTokenSize)
    {
        mContext.diagnostics->report(Diagnostics::PP_TOKEN_TOO_LONG,
                                     token->location, token->text);
        token->text.erase(mMaxTokenSize);
    }

    token->flags = 0;

    token->setAtStartOfLine(mContext.lineStart);
    mContext.lineStart = token->type == '\n';

    token->setHasLeadingSpace(mContext.leadingSpace);
    mContext.leadingSpace = false;
}

bool Tokenizer::initScanner()
{
    if ((mHandle == NULL) && yylex_init_extra(&mContext, &mHandle))
        return false;

    yyrestart(0, mHandle);
    return true;
}

void Tokenizer::destroyScanner()
{
    if (mHandle == NULL)
        return;

    yylex_destroy(mHandle);
    mHandle = NULL;
}

}  // namespace pp

kc3-lang/angle/src/compiler/preprocessor/Tokenizer.l

Commit

kc3-lang/angle /src/compiler/preprocessor/Tokenizer.l