kc3-lang/angle/src/compiler/preprocessor/preprocessor.l

• Show log

  Commit

• Hash : d6053daa
  Author :
  Date : 2019-12-05T17:46:23
  

  Remove tabs from source files.
  
  WebKit's Subversion repo refuses to commit source files that contain
  tabs.
  
  Bug: angleproject:3439
  Change-Id: I0a804bcfa0375a98e19945e20297c90d31106827
  Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/1954410
  Commit-Queue: James Darpinian <jdarpinian@chromium.org>
  Reviewed-by: Shahbaz Youssefi <syoussefi@chromium.org>
  

Download

• src/compiler/preprocessor/preprocessor.l

• /*
  //
  // Copyright 2002 The ANGLE Project Authors. All rights reserved.
  // Use of this source code is governed by a BSD-style license that can be
  // found in the LICENSE file.
  //
  
  This file contains the Lex specification for GLSL ES preprocessor.
  Based on Microsoft Visual Studio 2010 Preprocessor Grammar:
  http://msdn.microsoft.com/en-us/library/2scxys89.aspx
  
  IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN scripts/run_code_generation.py
  */
  
  %top{
  // GENERATED FILE - DO NOT EDIT.
  // Generated by generate_parser.py from preprocessor.l
  //
  // Copyright 2019 The ANGLE Project Authors. All rights reserved.
  // Use of this source code is governed by a BSD-style license that can be
  // found in the LICENSE file.
  //
  // preprocessor.l:
  //   Lexer for the OpenGL shading language preprocessor.
  
  }
  
  %{
  #if defined(_MSC_VER)
  #pragma warning(disable: 4005)
  #endif
  
  #include "compiler/preprocessor/Tokenizer.h"
  
  #include "compiler/preprocessor/DiagnosticsBase.h"
  #include "compiler/preprocessor/Token.h"
  
  #if defined(__GNUC__)
  // Triggered by the auto-generated yy_fatal_error function.
  #pragma GCC diagnostic ignored "-Wmissing-noreturn"
  #elif defined(_MSC_VER)
  #pragma warning(disable: 4244)
  #endif
  #if defined(__clang__)
  // Flex uses `/*FALLTHROUGH*/` instead of dedicated statements.
  #pragma clang diagnostic ignored "-Wimplicit-fallthrough"
  #if defined(__APPLE__)
  // Older clang versions don't have -Wextra-semi-stmt, and detecting Apple clang versions is
  // difficult because they use different yet overlapping version numbers vs. regular clang.
  #pragma clang diagnostic ignored "-Wunknown-warning-option"
  #endif
  // Flex isn't semi-colon clean.
  #pragma clang diagnostic ignored "-Wextra-semi-stmt"
  #pragma clang diagnostic ignored "-Wunreachable-code"
  #endif
  
  // Workaround for flex using the register keyword, deprecated in C++11.
  #ifdef __cplusplus
  #if __cplusplus > 199711L
  #define register
  #endif
  #endif
  
  typedef std::string YYSTYPE;
  typedef angle::pp::SourceLocation YYLTYPE;
  
  // Use the unused yycolumn variable to track file (string) number.
  #define yyfileno yycolumn
  
  #define YY_USER_INIT                   \
      do {                               \
          yyfileno = 0;                  \
          yylineno = 1;                  \
          yyextra->leadingSpace = false; \
          yyextra->lineStart = true;     \
      } while(0);
  
  #define YY_NO_INPUT
  #define YY_USER_ACTION                                              \
      do                                                              \
      {                                                               \
          angle::pp::Input* input = &yyextra->input;                  \
          angle::pp::Input::Location* scanLoc = &yyextra->scanLoc;    \
          while ((scanLoc->sIndex < input->count()) &&                \
                 (scanLoc->cIndex >= input->length(scanLoc->sIndex))) \
          {                                                           \
              scanLoc->cIndex -= input->length(scanLoc->sIndex++);    \
              ++yyfileno; yylineno = 1;                               \
          }                                                           \
          yylloc->file = yyfileno;                                    \
          yylloc->line = yylineno;                                    \
          scanLoc->cIndex += yyleng;                                  \
      } while(0);
  
  #define YY_INPUT(buf, result, maxSize) \
      result = yyextra->input.read(buf, maxSize, &yylineno);
  
  %}
  
  %option noyywrap nounput never-interactive
  %option reentrant bison-bridge bison-locations
  %option prefix="pp"
  %option extra-type="angle::pp::Tokenizer::Context*"
  %x COMMENT
  
  NEWLINE     \n|\r|\r\n
  IDENTIFIER  [_a-zA-Z][_a-zA-Z0-9]*
  PUNCTUATOR  [][<>(){}.+-/*%^|&~=!:;,?]
  
  DECIMAL_CONSTANT      [1-9][0-9]*[uU]?
  OCTAL_CONSTANT        0[0-7]*[uU]?
  HEXADECIMAL_CONSTANT  0[xX][0-9a-fA-F]+[uU]?
  
  DIGIT                [0-9]
  EXPONENT_PART        [eE][+-]?{DIGIT}+
  FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
  
  %%
  
      /* Line comment */
  "//"[^\r\n]*
  
      /* Block comment */
      /* Line breaks are just counted - not returned. */
      /* The comment is replaced by a single space. */
  "/*" { BEGIN(COMMENT); }
  <COMMENT>[^*\r\n]+
  <COMMENT>"*"
  <COMMENT>{NEWLINE} {
      if (yylineno == INT_MAX)
      {
          *yylval = "Integer overflow on line number";
          return angle::pp::Token::GOT_ERROR;
      }
      ++yylineno;
  }
  <COMMENT>"*/" {
      yyextra->leadingSpace = true;
      BEGIN(INITIAL);
  }
  
  # {
      // # is only valid at start of line for preprocessor directives.
      yylval->assign(1, yytext[0]);
      return yyextra->lineStart ? angle::pp::Token::PP_HASH : angle::pp::Token::PP_OTHER;
  }
  
  {IDENTIFIER} {
      yylval->assign(yytext, yyleng);
      return angle::pp::Token::IDENTIFIER;
  }
  
  ({DECIMAL_CONSTANT}[uU]?)|({OCTAL_CONSTANT}[uU]?)|({HEXADECIMAL_CONSTANT}[uU]?) {
      yylval->assign(yytext, yyleng);
      return angle::pp::Token::CONST_INT;
  }
  
  ({DIGIT}+{EXPONENT_PART}[fF]?)|({FRACTIONAL_CONSTANT}{EXPONENT_PART}?[fF]?) {
      yylval->assign(yytext, yyleng);
      return angle::pp::Token::CONST_FLOAT;
  }
  
      /* Anything that starts with a {DIGIT} or .{DIGIT} must be a number. */
      /* Rule to catch all invalid integers and floats. */
  ({DIGIT}+[_a-zA-Z0-9.]*)|("."{DIGIT}+[_a-zA-Z0-9.]*) {
      yylval->assign(yytext, yyleng);
      return angle::pp::Token::PP_NUMBER;
  }
  
  "++" {
      yylval->assign(yytext, yyleng);
      return angle::pp::Token::OP_INC;
  }
  "--" {
      yylval->assign(yytext, yyleng);
      return angle::pp::Token::OP_DEC;
  }
  "<<" {
      yylval->assign(yytext, yyleng);
      return angle::pp::Token::OP_LEFT;
  }
  ">>" {
      yylval->assign(yytext, yyleng);
      return angle::pp::Token::OP_RIGHT;
  }
  "<=" {
      yylval->assign(yytext, yyleng);
      return angle::pp::Token::OP_LE;
  }
  ">=" {
      yylval->assign(yytext, yyleng);
      return angle::pp::Token::OP_GE;
  }
  "==" {
      yylval->assign(yytext, yyleng);
      return angle::pp::Token::OP_EQ;
  }
  "!=" {
      yylval->assign(yytext, yyleng);
      return angle::pp::Token::OP_NE;
  }
  "&&" {
      yylval->assign(yytext, yyleng);
      return angle::pp::Token::OP_AND;
  }
  "^^" {
      yylval->assign(yytext, yyleng);
      return angle::pp::Token::OP_XOR;
  }
  "||" {
      yylval->assign(yytext, yyleng);
      return angle::pp::Token::OP_OR;
  }
  "+=" {
      yylval->assign(yytext, yyleng);
      return angle::pp::Token::OP_ADD_ASSIGN;
  }
  "-=" {
      yylval->assign(yytext, yyleng);
      return angle::pp::Token::OP_SUB_ASSIGN;
  }
  "*=" {
      yylval->assign(yytext, yyleng);
      return angle::pp::Token::OP_MUL_ASSIGN;
  }
  "/=" {
      yylval->assign(yytext, yyleng);
      return angle::pp::Token::OP_DIV_ASSIGN;
  }
  "%=" {
      yylval->assign(yytext, yyleng);
      return angle::pp::Token::OP_MOD_ASSIGN;
  }
  "<<=" {
      yylval->assign(yytext, yyleng);
      return angle::pp::Token::OP_LEFT_ASSIGN;
  }
  ">>=" {
      yylval->assign(yytext, yyleng);
      return angle::pp::Token::OP_RIGHT_ASSIGN;
  }
  "&=" {
      yylval->assign(yytext, yyleng);
      return angle::pp::Token::OP_AND_ASSIGN;
  }
  "^=" {
      yylval->assign(yytext, yyleng);
      return angle::pp::Token::OP_XOR_ASSIGN;
  }
  "|=" {
      yylval->assign(yytext, yyleng);
      return angle::pp::Token::OP_OR_ASSIGN;
  }
  
  {PUNCTUATOR} {
      yylval->assign(1, yytext[0]);
      return yytext[0];
  }
  
  [ \t\v\f]+   { yyextra->leadingSpace = true; }
  
  {NEWLINE} {
      if (yylineno == INT_MAX)
      {
          *yylval = "Integer overflow on line number";
          return angle::pp::Token::GOT_ERROR;
      }
      ++yylineno;
      yylval->assign(1, '\n');
      return '\n';
  }
  
  . {
      yylval->assign(1, yytext[0]);
      return angle::pp::Token::PP_OTHER;
  }
  
  <*><<EOF>> {
      // YY_USER_ACTION is not invoked for handling EOF.
      // Set the location for EOF token manually.
      angle::pp::Input* input = &yyextra->input;
      angle::pp::Input::Location* scanLoc = &yyextra->scanLoc;
      yy_size_t sIndexMax = input->count() ? input->count() - 1 : 0;
      if (scanLoc->sIndex != sIndexMax)
      {
          // We can only reach here if there are empty strings at the
          // end of the input.
          scanLoc->sIndex = sIndexMax; scanLoc->cIndex = 0;
          // FIXME: this is not 64-bit clean.
          yyfileno = static_cast<int>(sIndexMax); yylineno = 1;
      }
      yylloc->file = yyfileno;
      yylloc->line = yylineno;
      yylval->clear();
  
      // Line number overflows fake EOFs to exit early, check for this case.
      if (yylineno == INT_MAX) {
          yyextra->diagnostics->report(angle::pp::Diagnostics::PP_TOKENIZER_ERROR,
                  angle::pp::SourceLocation(yyfileno, yylineno),
                  "Integer overflow on line number");
      }
      else if (YY_START == COMMENT)
      {
          yyextra->diagnostics->report(angle::pp::Diagnostics::PP_EOF_IN_COMMENT,
                                       angle::pp::SourceLocation(yyfileno, yylineno),
                                       "EOF while in a comment");
      }
      yyterminate();
  }
  
  %%
  
  namespace angle {
  
  namespace pp {
  
  Tokenizer::Tokenizer(Diagnostics *diagnostics) : mHandle(nullptr), mMaxTokenSize(256)
  {
      mContext.diagnostics = diagnostics;
  }
  
  Tokenizer::~Tokenizer()
  {
      destroyScanner();
  }
  
  bool Tokenizer::init(size_t count, const char * const string[], const int length[])
  {
      if ((count > 0) && (string == 0))
          return false;
  
      mContext.input = Input(count, string, length);
      return initScanner();
  }
  
  void Tokenizer::setFileNumber(int file)
  {
      // We use column number as file number.
      // See macro yyfileno.
      yyset_column(file, mHandle);
  }
  
  void Tokenizer::setLineNumber(int line)
  {
      yyset_lineno(line, mHandle);
  }
  
  void Tokenizer::setMaxTokenSize(size_t maxTokenSize)
  {
      mMaxTokenSize = maxTokenSize;
  }
  
  void Tokenizer::lex(Token *token)
  {
      int tokenType = yylex(&token->text, &token->location, mHandle);
  
      if (tokenType == Token::GOT_ERROR)
      {
          mContext.diagnostics->report(Diagnostics::PP_TOKENIZER_ERROR, token->location, token->text);
          token->type = Token::LAST;
      }
      else
      {
          token->type = tokenType;
      }
  
      if (token->text.size() > mMaxTokenSize)
      {
          mContext.diagnostics->report(Diagnostics::PP_TOKEN_TOO_LONG,
                                       token->location, token->text);
          token->text.erase(mMaxTokenSize);
      }
  
      token->flags = 0;
  
      token->setAtStartOfLine(mContext.lineStart);
      mContext.lineStart = token->type == '\n';
  
      token->setHasLeadingSpace(mContext.leadingSpace);
      mContext.leadingSpace = false;
  }
  
  bool Tokenizer::initScanner()
  {
      if ((mHandle == nullptr) && yylex_init_extra(&mContext, &mHandle))
          return false;
  
      yyrestart(0, mHandle);
      return true;
  }
  
  void Tokenizer::destroyScanner()
  {
      if (mHandle == nullptr)
          return;
  
      yylex_destroy(mHandle);
      mHandle = nullptr;
  }
  
  }  // namespace pp
  
  } // namespace angle
  
  

Download