Tokenizer.l

Branch :

Show log
Commit
Author : Olli Etuaho
Date : 2015-08-14 14:16:19
Hash : 26e355b8
Message : Add full support for line continuation in the preprocessor Re-landing earlier change with constant signedness fixed (was causing build issues on Linux). Line continuation in ESSL 3.00 needs to be processed before tokenization, since tokens can span the line continuation. On the other hand, ANGLE's tokenizer keeps track of line numbers, and whenever a line continuation appears the line number still needs to be incremented by one, just like on a regular newline. That's why line continuation is now implemented as follows: when the shader strings are concatenated in Input, they are also checked for line continuation. Whenever line continuation is encountered, the string is cut before that point. When the tokenizer asks for more input, the string starting from the character after line continuation is passed to it, and the line number is incremented from Input. This way the tokenizer can parse tokens that span multiple lines - it never sees the line continuation - but still keeps track of the line number correctly. Relevant spec is in ESSL 3.00 section 3.2 "Source strings". Support for line continuation also applies to ESSL 1.00. ESSL 3.00 spec section 1.5 says that line continuation support is mandated when an ESSL 1.00 shader is used with the OpenGL ES 3.0 API, and is optional when ESSL 1.00 is used with the OpenGL ES 2.0 API. TEST=dEQP-GLES3.functional.shaders.preprocessor.line_continuation.* (all pass), angle_unittests BUG=angleproject:1125 Change-Id: Ic086aacac53cd75bf93c0fda782416501d2f842b Reviewed-on: https://chromium-review.googlesource.com/294200 Reviewed-by: Jamie Madill <jmadill@chromium.org> Tested-by: Olli Etuaho <oetuaho@nvidia.com>

src/compiler/preprocessor/Tokenizer.l

/*
//
// Copyright (c) 2002-2014 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//

This file contains the Lex specification for GLSL ES preprocessor.
Based on Microsoft Visual Studio 2010 Preprocessor Grammar:
http://msdn.microsoft.com/en-us/library/2scxys89.aspx

IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
*/

%top{
//
// Copyright (c) 2011-2014 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//

// This file is auto-generated by generate_parser.sh. DO NOT EDIT!
}

%{
#if defined(_MSC_VER)
#pragma warning(disable: 4005)
#endif

#include "Tokenizer.h"

#include "DiagnosticsBase.h"
#include "Token.h"

#if defined(__GNUC__)
// Triggered by the auto-generated yy_fatal_error function.
#pragma GCC diagnostic ignored "-Wmissing-noreturn"
#elif defined(_MSC_VER)
#pragma warning(disable: 4244)
#endif

// Workaround for flex using the register keyword, deprecated in C++11.
#ifdef __cplusplus
#if __cplusplus > 199711L
#define register
#endif
#endif

typedef std::string YYSTYPE;
typedef pp::SourceLocation YYLTYPE;

// Use the unused yycolumn variable to track file (string) number.
#define yyfileno yycolumn

#define YY_USER_INIT                   \
    do {                               \
        yyfileno = 0;                  \
        yylineno = 1;                  \
        yyextra->leadingSpace = false; \
        yyextra->lineStart = true;     \
    } while(0);

#define YY_USER_ACTION                                              \
    do                                                              \
    {                                                               \
        pp::Input* input = &yyextra->input;                         \
        pp::Input::Location* scanLoc = &yyextra->scanLoc;           \
        while ((scanLoc->sIndex < input->count()) &&                \
               (scanLoc->cIndex >= input->length(scanLoc->sIndex))) \
        {                                                           \
            scanLoc->cIndex -= input->length(scanLoc->sIndex++);    \
            ++yyfileno; yylineno = 1;                               \
        }                                                           \
        yylloc->file = yyfileno;                                    \
        yylloc->line = yylineno;                                    \
        scanLoc->cIndex += yyleng;                                  \
    } while(0);

#define YY_INPUT(buf, result, maxSize) \
    result = yyextra->input.read(buf, maxSize, &yylineno);

%}

%option noyywrap nounput never-interactive
%option reentrant bison-bridge bison-locations
%option prefix="pp"
%option extra-type="pp::Tokenizer::Context*"
%x COMMENT

NEWLINE     \n|\r|\r\n
IDENTIFIER  [_a-zA-Z][_a-zA-Z0-9]*
PUNCTUATOR  [][<>(){}.+-/*%^|&~=!:;,?]

DECIMAL_CONSTANT      [1-9][0-9]*[uU]?
OCTAL_CONSTANT        0[0-7]*[uU]?
HEXADECIMAL_CONSTANT  0[xX][0-9a-fA-F]+[uU]?

DIGIT                [0-9]
EXPONENT_PART        [eE][+-]?{DIGIT}+
FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")

%%

    /* Line comment */
"//"[^\r\n]*

    /* Block comment */
    /* Line breaks are just counted - not returned. */
    /* The comment is replaced by a single space. */
"/*" { BEGIN(COMMENT); }
<COMMENT>[^*\r\n]+
<COMMENT>"*"
<COMMENT>{NEWLINE} { ++yylineno; }
<COMMENT>"*/" {
    yyextra->leadingSpace = true;
    BEGIN(INITIAL);
}

# {
    // # is only valid at start of line for preprocessor directives.
    yylval->assign(1, yytext[0]);
    return yyextra->lineStart ? pp::Token::PP_HASH : pp::Token::PP_OTHER;
}

{IDENTIFIER} {
    yylval->assign(yytext, yyleng);
    return pp::Token::IDENTIFIER;
}

({DECIMAL_CONSTANT}[uU]?)|({OCTAL_CONSTANT}[uU]?)|({HEXADECIMAL_CONSTANT}[uU]?) {
    yylval->assign(yytext, yyleng);
    return pp::Token::CONST_INT;
}

({DIGIT}+{EXPONENT_PART}[fF]?)|({FRACTIONAL_CONSTANT}{EXPONENT_PART}?[fF]?) {
    yylval->assign(yytext, yyleng);
    return pp::Token::CONST_FLOAT;
}

    /* Anything that starts with a {DIGIT} or .{DIGIT} must be a number. */
    /* Rule to catch all invalid integers and floats. */
({DIGIT}+[_a-zA-Z0-9.]*)|("."{DIGIT}+[_a-zA-Z0-9.]*) {
    yylval->assign(yytext, yyleng);
    return pp::Token::PP_NUMBER;
}

"++" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_INC;
}
"--" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_DEC;
}
"<<" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_LEFT;
}
">>" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_RIGHT;
}
"<=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_LE;
}
">=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_GE;
}
"==" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_EQ;
}
"!=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_NE;
}
"&&" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_AND;
}
"^^" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_XOR;
}
"||" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_OR;
}
"+=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_ADD_ASSIGN;
}
"-=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_SUB_ASSIGN;
}
"*=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_MUL_ASSIGN;
}
"/=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_DIV_ASSIGN;
}
"%=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_MOD_ASSIGN;
}
"<<=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_LEFT_ASSIGN;
}
">>=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_RIGHT_ASSIGN;
}
"&=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_AND_ASSIGN;
}
"^=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_XOR_ASSIGN;
}
"|=" {
    yylval->assign(yytext, yyleng);
    return pp::Token::OP_OR_ASSIGN;
}

{PUNCTUATOR} {
    yylval->assign(1, yytext[0]);
    return yytext[0];
}

[ \t\v\f]+   { yyextra->leadingSpace = true; }

{NEWLINE} {
    ++yylineno;
    yylval->assign(1, '\n');
    return '\n';
}

\\{NEWLINE} { ++yylineno; }

. {
    yylval->assign(1, yytext[0]);
    return pp::Token::PP_OTHER;
}

<*><<EOF>> {
    // YY_USER_ACTION is not invoked for handling EOF.
    // Set the location for EOF token manually.
    pp::Input* input = &yyextra->input;
    pp::Input::Location* scanLoc = &yyextra->scanLoc;
    yy_size_t sIndexMax = input->count() ? input->count() - 1 : 0;
    if (scanLoc->sIndex != sIndexMax)
    {
        // We can only reach here if there are empty strings at the
        // end of the input.
        scanLoc->sIndex = sIndexMax; scanLoc->cIndex = 0;
        // FIXME: this is not 64-bit clean.
        yyfileno = static_cast<int>(sIndexMax); yylineno = 1;
    }
    yylloc->file = yyfileno;
    yylloc->line = yylineno;
    yylval->clear();

    if (YY_START == COMMENT)
    {
        yyextra->diagnostics->report(pp::Diagnostics::PP_EOF_IN_COMMENT,
                                     pp::SourceLocation(yyfileno, yylineno),
                                     "");
    }
    yyterminate();
}

%%

namespace pp {

Tokenizer::Tokenizer(Diagnostics *diagnostics)
    : mHandle(0),
      mMaxTokenSize(256)
{
    mContext.diagnostics = diagnostics;
}

Tokenizer::~Tokenizer()
{
    destroyScanner();
}

bool Tokenizer::init(size_t count, const char * const string[], const int length[])
{
    if ((count > 0) && (string == 0))
        return false;

    mContext.input = Input(count, string, length);
    return initScanner();
}

void Tokenizer::setFileNumber(int file)
{
    // We use column number as file number.
    // See macro yyfileno.
    yyset_column(file, mHandle);
}

void Tokenizer::setLineNumber(int line)
{
    yyset_lineno(line, mHandle);
}

void Tokenizer::setMaxTokenSize(size_t maxTokenSize)
{
    mMaxTokenSize = maxTokenSize;
}

void Tokenizer::lex(Token *token)
{
    token->type = yylex(&token->text, &token->location, mHandle);
    if (token->text.size() > mMaxTokenSize)
    {
        mContext.diagnostics->report(Diagnostics::PP_TOKEN_TOO_LONG,
                                     token->location, token->text);
        token->text.erase(mMaxTokenSize);
    }

    token->flags = 0;

    token->setAtStartOfLine(mContext.lineStart);
    mContext.lineStart = token->type == '\n';

    token->setHasLeadingSpace(mContext.leadingSpace);
    mContext.leadingSpace = false;
}

bool Tokenizer::initScanner()
{
    if ((mHandle == NULL) && yylex_init_extra(&mContext, &mHandle))
        return false;

    yyrestart(0, mHandle);
    return true;
}

void Tokenizer::destroyScanner()
{
    if (mHandle == NULL)
        return;

    yylex_destroy(mHandle);
    mHandle = NULL;
}

}  // namespace pp

kc3-lang/angle/src/compiler/preprocessor/Tokenizer.l

Commit

kc3-lang/angle /src/compiler/preprocessor/Tokenizer.l