Tokenizer.l

Branch :
Show log
Commit
Author : Geoff Lang
Date : 2018-04-25 14:29:00
Hash : 197d5294
Message : Wrap all preprocessor code in the angle namespace. BUG=836820 BUG=801364 Change-Id: I08b6a2f9f12b689e09df6efd916c313e71e8a051 Reviewed-on: https://chromium-review.googlesource.com/1028581 Commit-Queue: Geoff Lang <geofflang@chromium.org> Reviewed-by: Yuly Novikov <ynovikov@chromium.org>
src/compiler/preprocessor/Tokenizer.l
/*
//
// Copyright (c) 2002-2014 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//

This file contains the Lex specification for GLSL ES preprocessor.
Based on Microsoft Visual Studio 2010 Preprocessor Grammar:
http://msdn.microsoft.com/en-us/library/2scxys89.aspx

IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
*/

%top{
//
// Copyright (c) 2011-2014 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//

// This file is auto-generated by generate_parser.sh. DO NOT EDIT!
}

%{
#if defined(_MSC_VER)
#pragma warning(disable: 4005)
#endif

#include "compiler/preprocessor/Tokenizer.h"

#include "compiler/preprocessor/DiagnosticsBase.h"
#include "compiler/preprocessor/Token.h"

#if defined(__GNUC__)
// Triggered by the auto-generated yy_fatal_error function.
#pragma GCC diagnostic ignored "-Wmissing-noreturn"
#elif defined(_MSC_VER)
#pragma warning(disable: 4244)
#endif
#if defined(__clang__)
// Flex uses `/*FALLTHROUGH*/` instead of dedicated statements.
#pragma GCC diagnostic ignored "-Wimplicit-fallthrough"
#endif

// Workaround for flex using the register keyword, deprecated in C++11.
#ifdef __cplusplus
#if __cplusplus > 199711L
#define register
#endif
#endif

typedef std::string YYSTYPE;
typedef angle::pp::SourceLocation YYLTYPE;

// Use the unused yycolumn variable to track file (string) number.
#define yyfileno yycolumn

#define YY_USER_INIT                   \
    do {                               \
        yyfileno = 0;                  \
        yylineno = 1;                  \
        yyextra->leadingSpace = false; \
        yyextra->lineStart = true;     \
    } while(0);

#define YY_NO_INPUT
#define YY_USER_ACTION                                              \
    do                                                              \
    {                                                               \
        angle::pp::Input* input = &yyextra->input;                  \
        angle::pp::Input::Location* scanLoc = &yyextra->scanLoc;    \
        while ((scanLoc->sIndex < input->count()) &&                \
               (scanLoc->cIndex >= input->length(scanLoc->sIndex))) \
        {                                                           \
            scanLoc->cIndex -= input->length(scanLoc->sIndex++);    \
            ++yyfileno; yylineno = 1;                               \
        }                                                           \
        yylloc->file = yyfileno;                                    \
        yylloc->line = yylineno;                                    \
        scanLoc->cIndex += yyleng;                                  \
    } while(0);

#define YY_INPUT(buf, result, maxSize) \
    result = yyextra->input.read(buf, maxSize, &yylineno);

%}

%option noyywrap nounput never-interactive
%option reentrant bison-bridge bison-locations
%option prefix="pp"
%option extra-type="angle::pp::Tokenizer::Context*"
%x COMMENT

NEWLINE     \n|\r|\r\n
IDENTIFIER  [_a-zA-Z][_a-zA-Z0-9]*
PUNCTUATOR  [][<>(){}.+-/*%^|&~=!:;,?]

DECIMAL_CONSTANT      [1-9][0-9]*[uU]?
OCTAL_CONSTANT        0[0-7]*[uU]?
HEXADECIMAL_CONSTANT  0[xX][0-9a-fA-F]+[uU]?

DIGIT                [0-9]
EXPONENT_PART        [eE][+-]?{DIGIT}+
FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")

%%

    /* Line comment */
"//"[^\r\n]*

    /* Block comment */
    /* Line breaks are just counted - not returned. */
    /* The comment is replaced by a single space. */
"/*" { BEGIN(COMMENT); }
<COMMENT>[^*\r\n]+
<COMMENT>"*"
<COMMENT>{NEWLINE} {
    if (yylineno == INT_MAX)
    {
        *yylval = "Integer overflow on line number";
        return angle::pp::Token::GOT_ERROR;
    }
    ++yylineno;
}
<COMMENT>"*/" {
    yyextra->leadingSpace = true;
    BEGIN(INITIAL);
}

# {
    // # is only valid at start of line for preprocessor directives.
    yylval->assign(1, yytext[0]);
    return yyextra->lineStart ? angle::pp::Token::PP_HASH : angle::pp::Token::PP_OTHER;
}

{IDENTIFIER} {
    yylval->assign(yytext, yyleng);
    return angle::pp::Token::IDENTIFIER;
}

({DECIMAL_CONSTANT}[uU]?)|({OCTAL_CONSTANT}[uU]?)|({HEXADECIMAL_CONSTANT}[uU]?) {
    yylval->assign(yytext, yyleng);
    return angle::pp::Token::CONST_INT;
}

({DIGIT}+{EXPONENT_PART}[fF]?)|({FRACTIONAL_CONSTANT}{EXPONENT_PART}?[fF]?) {
    yylval->assign(yytext, yyleng);
    return angle::pp::Token::CONST_FLOAT;
}

    /* Anything that starts with a {DIGIT} or .{DIGIT} must be a number. */
    /* Rule to catch all invalid integers and floats. */
({DIGIT}+[_a-zA-Z0-9.]*)|("."{DIGIT}+[_a-zA-Z0-9.]*) {
    yylval->assign(yytext, yyleng);
    return angle::pp::Token::PP_NUMBER;
}

"++" {
    yylval->assign(yytext, yyleng);
    return angle::pp::Token::OP_INC;
}
"--" {
    yylval->assign(yytext, yyleng);
    return angle::pp::Token::OP_DEC;
}
"<<" {
    yylval->assign(yytext, yyleng);
    return angle::pp::Token::OP_LEFT;
}
">>" {
    yylval->assign(yytext, yyleng);
    return angle::pp::Token::OP_RIGHT;
}
"<=" {
    yylval->assign(yytext, yyleng);
    return angle::pp::Token::OP_LE;
}
">=" {
    yylval->assign(yytext, yyleng);
    return angle::pp::Token::OP_GE;
}
"==" {
    yylval->assign(yytext, yyleng);
    return angle::pp::Token::OP_EQ;
}
"!=" {
    yylval->assign(yytext, yyleng);
    return angle::pp::Token::OP_NE;
}
"&&" {
    yylval->assign(yytext, yyleng);
    return angle::pp::Token::OP_AND;
}
"^^" {
    yylval->assign(yytext, yyleng);
    return angle::pp::Token::OP_XOR;
}
"||" {
    yylval->assign(yytext, yyleng);
    return angle::pp::Token::OP_OR;
}
"+=" {
    yylval->assign(yytext, yyleng);
    return angle::pp::Token::OP_ADD_ASSIGN;
}
"-=" {
    yylval->assign(yytext, yyleng);
    return angle::pp::Token::OP_SUB_ASSIGN;
}
"*=" {
    yylval->assign(yytext, yyleng);
    return angle::pp::Token::OP_MUL_ASSIGN;
}
"/=" {
    yylval->assign(yytext, yyleng);
    return angle::pp::Token::OP_DIV_ASSIGN;
}
"%=" {
    yylval->assign(yytext, yyleng);
    return angle::pp::Token::OP_MOD_ASSIGN;
}
"<<=" {
    yylval->assign(yytext, yyleng);
    return angle::pp::Token::OP_LEFT_ASSIGN;
}
">>=" {
    yylval->assign(yytext, yyleng);
    return angle::pp::Token::OP_RIGHT_ASSIGN;
}
"&=" {
    yylval->assign(yytext, yyleng);
    return angle::pp::Token::OP_AND_ASSIGN;
}
"^=" {
    yylval->assign(yytext, yyleng);
    return angle::pp::Token::OP_XOR_ASSIGN;
}
"|=" {
    yylval->assign(yytext, yyleng);
    return angle::pp::Token::OP_OR_ASSIGN;
}

{PUNCTUATOR} {
    yylval->assign(1, yytext[0]);
    return yytext[0];
}

[ \t\v\f]+   { yyextra->leadingSpace = true; }

{NEWLINE} {
    if (yylineno == INT_MAX)
    {
        *yylval = "Integer overflow on line number";
        return angle::pp::Token::GOT_ERROR;
    }
    ++yylineno;
    yylval->assign(1, '\n');
    return '\n';
}

. {
    yylval->assign(1, yytext[0]);
    return angle::pp::Token::PP_OTHER;
}

<*><<EOF>> {
    // YY_USER_ACTION is not invoked for handling EOF.
    // Set the location for EOF token manually.
    angle::pp::Input* input = &yyextra->input;
    angle::pp::Input::Location* scanLoc = &yyextra->scanLoc;
    yy_size_t sIndexMax = input->count() ? input->count() - 1 : 0;
    if (scanLoc->sIndex != sIndexMax)
    {
        // We can only reach here if there are empty strings at the
        // end of the input.
        scanLoc->sIndex = sIndexMax; scanLoc->cIndex = 0;
        // FIXME: this is not 64-bit clean.
        yyfileno = static_cast<int>(sIndexMax); yylineno = 1;
    }
    yylloc->file = yyfileno;
    yylloc->line = yylineno;
    yylval->clear();

    // Line number overflows fake EOFs to exit early, check for this case.
    if (yylineno == INT_MAX) {
        yyextra->diagnostics->report(angle::pp::Diagnostics::PP_TOKENIZER_ERROR,
                angle::pp::SourceLocation(yyfileno, yylineno),
                "Integer overflow on line number");
    }
    else if (YY_START == COMMENT)
    {
        yyextra->diagnostics->report(angle::pp::Diagnostics::PP_EOF_IN_COMMENT,
                                     angle::pp::SourceLocation(yyfileno, yylineno),
                                     "EOF while in a comment");
    }
    yyterminate();
}

%%

namespace angle {

namespace pp {

Tokenizer::Tokenizer(Diagnostics *diagnostics) : mHandle(nullptr), mMaxTokenSize(256)
{
    mContext.diagnostics = diagnostics;
}

Tokenizer::~Tokenizer()
{
    destroyScanner();
}

bool Tokenizer::init(size_t count, const char * const string[], const int length[])
{
    if ((count > 0) && (string == 0))
        return false;

    mContext.input = Input(count, string, length);
    return initScanner();
}

void Tokenizer::setFileNumber(int file)
{
    // We use column number as file number.
    // See macro yyfileno.
    yyset_column(file, mHandle);
}

void Tokenizer::setLineNumber(int line)
{
    yyset_lineno(line, mHandle);
}

void Tokenizer::setMaxTokenSize(size_t maxTokenSize)
{
    mMaxTokenSize = maxTokenSize;
}

void Tokenizer::lex(Token *token)
{
    int tokenType = yylex(&token->text, &token->location, mHandle);

    if (tokenType == Token::GOT_ERROR)
    {
        mContext.diagnostics->report(Diagnostics::PP_TOKENIZER_ERROR, token->location, token->text);
        token->type = Token::LAST;
    }
    else
    {
        token->type = tokenType;
    }

    if (token->text.size() > mMaxTokenSize)
    {
        mContext.diagnostics->report(Diagnostics::PP_TOKEN_TOO_LONG,
                                     token->location, token->text);
        token->text.erase(mMaxTokenSize);
    }

    token->flags = 0;

    token->setAtStartOfLine(mContext.lineStart);
    mContext.lineStart = token->type == '\n';

    token->setHasLeadingSpace(mContext.leadingSpace);
    mContext.leadingSpace = false;
}

bool Tokenizer::initScanner()
{
    if ((mHandle == nullptr) && yylex_init_extra(&mContext, &mHandle))
        return false;

    yyrestart(0, mHandle);
    return true;
}

void Tokenizer::destroyScanner()
{
    if (mHandle == nullptr)
        return;

    yylex_destroy(mHandle);
    mHandle = nullptr;
}

}  // namespace pp

} // namespace angle
kc3-lang/angle/src/compiler/preprocessor/Tokenizer.l

Commit

kc3-lang/angle /src/compiler/preprocessor/Tokenizer.l
