Commit a06cbcd2 authored by jan.koester's avatar jan.koester
Browse files

flex fixed

parent 5f1471bc

lib/bison/bison.cpp

+43 −11
Changes for lib/bison/bison.cpp: 43 added lines, 11 removed lines.
Original line number Diff line number Diff line
@@ -1350,17 +1350,33 @@ std::string CodeGenerator::substituteAction(const Production &p) const { 
            continue;

        }

        if(c=='$'){

            if(i+1<n && src[i+1]=='$'){

                out += lhsTag.empty() ? "yyval" : ("(yyval."+lhsTag+")");

                i+=2; continue;

            }

            if(i+1<n && (isdigit((unsigned char)src[i+1]))){

                size_t j=i+1; std::string num;

            // "$<tag>$", "$<tag>1": the union member spelled out, for

            // mid-rule actions, whose values have no declared type

            std::string explicitTag;

            bool hasTag=false;

            size_t k=i+1;

            if(k<n && src[k]=='<'){

                size_t close=src.find('>',k);

                if(close!=std::string::npos){

                    explicitTag=src.substr(k+1,close-k-1);

                    hasTag=true;

                    k=close+1;

                }

            }

            if(k<n && src[k]=='$'){

                const std::string &tag = hasTag ? explicitTag : lhsTag;

                out += tag.empty() ? "yyval" : ("(yyval."+tag+")");

                i=k+1; continue;

            }

            if(k<n && (isdigit((unsigned char)src[k]) || (src[k]=='-' && k+1<n && isdigit((unsigned char)src[k+1])))){

                size_t j=k; std::string num;

                if(src[j]=='-') num+=src[j++];

                while(j<n && isdigit((unsigned char)src[j])) num+=src[j++];

                int N=atoi(num.c_str());

                int offset=N-len;

                int sym=rhsSymbolAt(N);

                std::string tag=symbolTypeTag(sym);

                // $0, $-1: what's on the stack below this rule

                int sym= N>0 ? rhsSymbolAt(N) : -1;

                std::string tag= hasTag ? explicitTag : (N>0 ? symbolTypeTag(sym) : std::string());

                std::string ref="yyvsp["+std::to_string(offset)+"]";

                out += tag.empty() ? ref : ("("+ref+"."+tag+")");

                i=j; continue;
@@ -1436,6 +1452,18 @@ std::string CodeGenerator::generateHeader(const CodegenOptions &opts) const { 
std::string CodeGenerator::generateSource(const CodegenOptions &opts) const {

    std::string out;

    out += "/* Generated by fresh bison. Do not edit. */\n\n";

    // Like real bison: the grammar's actions keep saying yylex, yyerror and

    // yylval whatever %name-prefix says, the preprocessor renames them.

    if(g_.nameprefix!="yy"){

        for(const char *name: {"yyparse","yylex","yyerror","yydebug","yynerrs"})

            out += "#define "+std::string(name)+" "+prefixed(name)+"\n";

        if(!g_.pureParser){

            out += "#define yylval "+prefixed("yylval")+"\n";

            out += "#define yychar "+prefixed("yychar")+"\n";

            if(g_.locations) out += "#define yylloc "+prefixed("yylloc")+"\n";

        }

        out += "\n";

    }

    out += g_.prologueText;

    out += "\n#include <stdlib.h>\n#include <string.h>\n#include <stdio.h>\n\n";

    if(opts.emitHeader) out += "#include \""+opts.baseName+".h\"\n\n";
@@ -1443,9 +1471,13 @@ std::string CodeGenerator::generateSource(const CodegenOptions &opts) const { 


    out += tablesText();

    out += "\n#define YY_ACCEPT_CODE 2147483647\n";

    out += "#define YYEMPTY (-2)\n\n";

    out += "#define YYEMPTY (-2)\n";

    out += "#define YYEOF 0\n";

    out += "#define yyclearin (yychar = YYEMPTY)\n\n";



    if(g_.locations){

        // also for a grammar's own YYLLOC_DEFAULT, which is written with it

        out += "#ifndef YYRHSLOC\n# define YYRHSLOC(Rhs, K) ((Rhs)[K])\n#endif\n";

        // Default location-merge, guarded so a grammar's own #define (in its

        // prologue, already emitted above) wins -- e.g. PostgreSQL's real

        // gram.y sets "#define YYLTYPE int" (just a scan position, not a
@@ -1457,7 +1489,6 @@ std::string CodeGenerator::generateSource(const CodegenOptions &opts) const { 
        // 'first_line' in something not a structure or union"). The

        // default below is real bison's own.

        out += "#ifndef YYLLOC_DEFAULT\n";

        out += "# define YYRHSLOC(Rhs, K) ((Rhs)[K])\n";

        out += "# define YYLLOC_DEFAULT(Current, Rhs, N) \\\n"

               "    do \\\n"

               "      if (N) { \\\n"
@@ -1487,7 +1518,8 @@ std::string CodeGenerator::generateSource(const CodegenOptions &opts) const { 
    std::string llocName = g_.pureParser ? "yylloc" : prefixed("yylloc");



    std::string errArgs;

    if(g_.locations) errArgs += "&"+llocName+", ";

    // the location only goes to a pure parser's yyerror

    if(g_.locations && g_.pureParser) errArgs += "&"+llocName+", ";

    if(!g_.parseParams.empty()) errArgs += paramNames+", ";

    errArgs += "\"syntax error\"";

lib/flex/flex.cpp

+47 −31
Changes for lib/flex/flex.cpp: 47 added lines, 31 removed lines.
Original line number Diff line number Diff line
@@ -329,8 +329,8 @@ bool FlexOptions::set(const std::string &name, const std::string &value) { 
    else if (key == "header-file" || key == "header") headerFile = value;

    else if (key == "extra-type") extraType = value;

    else if (key == "reentrant") reentrant = on;

    else if (key == "bison-bridge") { bisonBridge = on; if (on) reentrant = true; }

    else if (key == "bison-locations") { bisonLocations = on; if (on) { bisonBridge = true; reentrant = true; } }

    else if (key == "bison-bridge") bisonBridge = on;

    else if (key == "bison-locations") { bisonLocations = on; if (on) bisonBridge = true; }

    else if (key == "yywrap") yywrap = on;

    else if (key == "stack") stack = on;

    else if (key == "main") { main = on; if (on) yywrap = false; }
@@ -1161,6 +1161,29 @@ static void yy_init_buffer( YY_BUFFER_STATE b, FILE *file FX_PARG ); 
static FX_MAYBE_UNUSED int input( FX_PROTO );

static FX_MAYBE_UNUSED void yyunput( int c, char *yy_bp FX_PARG );



// Back to the state before the first yylex(); PostgreSQL's jsonpath scanner

// calls this between uses.

static FX_MAYBE_UNUSED int yy_init_globals( FX_PROTO )

{

    FX_GUTS

    FX_G->yy_buffer_stack = NULL;

    FX_G->yy_buffer_stack_top = 0;

    FX_G->yy_buffer_stack_max = 0;

    FX_G->yy_c_buf_p = NULL;

    FX_G->yy_init = 0;

    FX_G->yy_start = 0;

    FX_G->yy_start_stack_ptr = 0;

    FX_G->yy_start_stack_depth = 0;

    FX_G->yy_start_stack = NULL;

    FX_G->yy_more_flag = 0;

    FX_G->fx_states = NULL;

    FX_G->fx_states_cap = 0;

    yyin = NULL;

    yyout = NULL;

    yylineno = 1;

    return 0;

}



static void yy_fatal_error( const char *msg FX_PARG )

{

    FX_UNUSED;
@@ -2006,13 +2029,13 @@ std::string CodeGenerator::generateSource() { 
    out << "    char *yy_c_buf_p;\n    int yy_init;\n    int yy_start;\n    int yy_did_buffer_switch_on_eof;\n"

           "    int yy_start_stack_ptr;\n    int yy_start_stack_depth;\n    int *yy_start_stack;\n"

           "    int yy_more_flag;\n    int yy_more_len;\n    int *fx_states;\n    long fx_states_cap;\n";

    if (o.bisonBridge) out << "    YYSTYPE *yylval_r;\n";

    if (o.bisonLocations) out << "    YYLTYPE *yylloc_r;\n";

    if (o.reentrant && o.bisonBridge) out << "    YYSTYPE *yylval_r;\n";

    if (o.reentrant && o.bisonLocations) out << "    YYLTYPE *yylloc_r;\n";

    out << "};\n\n";

    if (!o.reentrant) out << "static struct fx_guts fx_globals;\n\n";



    if (o.bisonBridge) out << "#define yylval FX_G->yylval_r\n";

    if (o.bisonLocations) out << "#define yylloc FX_G->yylloc_r\n";

    if (o.reentrant && o.bisonBridge) out << "#define yylval FX_G->yylval_r\n";

    if (o.reentrant && o.bisonLocations) out << "#define yylloc FX_G->yylloc_r\n";

    if (o.reentrant) {

        out << "YY_EXTRA_TYPE yyget_extra( yyscan_t yyscanner );\nvoid yyset_extra( YY_EXTRA_TYPE user_defined, yyscan_t yyscanner );\n"

               "int yylex_init_extra( YY_EXTRA_TYPE user_defined, yyscan_t *scanner );\n"
@@ -2037,7 +2060,7 @@ std::string CodeGenerator::generateSource() { 
    else out << "((file) ? (isatty( fileno( file ) ) > 0) : 0)\n";



    if (o.reentrant) out << "#define FX_DESTROY_SCANNER yyfree( yyscanner, yyscanner )\n";

    else out << "#define FX_DESTROY_SCANNER memset( FX_G, 0, sizeof(*FX_G) )\n";

    else out << "#define FX_DESTROY_SCANNER yy_init_globals()\n";



    out << kAfterPrologueCode << "\n";

    if (o.stack) out << kStartStackCode << "\n";
@@ -2078,14 +2101,13 @@ std::string CodeGenerator::generateSource() { 


    // ---- the scanner ----

    out << "#ifndef YY_DECL\n#define YY_DECL_IS_OURS 1\n";

    if (o.reentrant) {

    {

        std::string params;

        if (o.bisonBridge) params += "YYSTYPE *yylval_param, ";

        if (o.bisonLocations) params += "YYLTYPE *yylloc_param, ";

        out << "extern int yylex( " << params << "yyscan_t yyscanner );\n"

            << "#define YY_DECL int yylex( " << params << "yyscan_t yyscanner )\n";

    } else {

        out << "extern int yylex( void );\n#define YY_DECL int yylex( void )\n";

        if (o.bisonBridge) params += "YYSTYPE *yylval_param";

        if (o.bisonLocations) params += ", YYLTYPE *yylloc_param";

        if (o.reentrant) params += params.empty() ? "yyscan_t yyscanner" : ", yyscan_t yyscanner";

        if (params.empty()) params = "void";

        out << "extern int yylex( " << params << " );\n#define YY_DECL int yylex( " << params << " )\n";

    }

    out << "#endif\n\n";
@@ -2102,8 +2124,9 @@ std::string CodeGenerator::generateSource() { 
           "    long fx_acc_off, fx_more;\n";

    if (useReject) out << "    long fx_pos, fx_lp;\n";

    out << "    FX_GUTS\n";

    if (o.bisonBridge) out << "    yylval = yylval_param;\n";

    if (o.bisonLocations) out << "    yylloc = yylloc_param;\n";

    // without a scanner struct the actions see plain locals of these names

    if (o.bisonBridge) out << (o.reentrant ? "    yylval = yylval_param;\n" : "    YYSTYPE *yylval = yylval_param;\n");

    if (o.bisonLocations) out << (o.reentrant ? "    yylloc = yylloc_param;\n" : "    YYLTYPE *yylloc = yylloc_param;\n");

    out << "    if ( ! FX_G->yy_init ) {\n"

           "        FX_G->yy_init = 1;\n"

           "#ifdef YY_USER_INIT\n        YY_USER_INIT;\n#endif\n"
@@ -2236,34 +2259,27 @@ std::string CodeGenerator::generateSource() { 
           "        YY_USER_ACTION\n"

           "fx_do_action:\n"

           "        switch ( yy_act ) {\n";

    // "|" shares the next rule's action

    // In file order: "|" shares the next rule's action, also an <<EOF>> rule's.

    for (size_t i = 0; i < s.rules.size(); ++i) {

        if (s.rules[i].isEofRule) continue;

        if (s.rules[i].isEofRule) {

            for (size_t st = 0; st < numStates; ++st)

                if (eofByState[st] == (int)i) out << "        case YY_STATE_EOF(" << st << "):\n";

        } else {

            out << "        case " << i << ":\n";

        }

        if (s.rules[i].actionText == "|") continue;

        out << "        {\n" << s.rules[i].actionText << "\n        }\n        YY_BREAK\n";

    }

    out << "        case " << defaultAct << ":\n"

           "            ECHO;\n"

           "            YY_BREAK\n";

    bool defaultEof = false;

    for (size_t st = 0; st < numStates; ++st) {

        if (eofByState[st] >= 0) continue;

        out << "        case YY_STATE_EOF(" << st << "):\n";

        defaultEof = true;

    }

    bool defaultEof = false;

    for (size_t st = 0; st < numStates; ++st) if (eofByState[st] < 0) defaultEof = true;

    if (defaultEof) out << "            yyterminate();\n";

    for (size_t i = 0; i < s.rules.size(); ++i) {

        if (!s.rules[i].isEofRule) continue;

        bool used = false;

        for (size_t st = 0; st < numStates; ++st) {

            if (eofByState[st] != (int)i) continue;

            out << "        case YY_STATE_EOF(" << st << "):\n";

            used = true;

        }

        if (!used) continue;

        out << "        {\n" << s.rules[i].actionText << "\n        }\n        YY_BREAK\n";

    }

    out << "        default:\n"

           "            YY_FATAL_ERROR( \"fatal flex scanner internal error--no action found\" );\n"

           "        }\n"

lib/flex/test/run.sh

+51 −0
Changes for lib/flex/test/run.sh: 51 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -317,6 +317,57 @@ int main(void){ 
E

check bison-bridge+text-in-buffer bb.l '' '42@2'



cat > "$T/bb2.l" <<'E'

%option bison-bridge noyywrap

%{

typedef int YYSTYPE;

%}

%%

[0-9]+   { *yylval = atoi(yytext); return 1; }

.|\n     ;

%%

int main(void){

    YYSTYPE v;

    while(yylex(&v)) printf("%d;", v);

    return 0;

}

E

check bison-bridge-not-reentrant bb2.l '4 5' '4;5;'



cat > "$T/again.l" <<'E'

%option noyywrap

%%

[a-z]+  printf("[%s:%d]", yytext, yylineno);

\n      ;

.       ;

%%

int main(void){

    YY_BUFFER_STATE h = yy_scan_string("a\nb");

    yylex();

    yy_delete_buffer(h);

    yylex_destroy();

    h = yy_scan_string("c");

    yylex();

    yy_delete_buffer(h);

    yy_init_globals();

    return 0;

}

E

check destroy-and-reuse again.l '' '[a:1][b:2][c:1]'



cat > "$T/bareof.l" <<'E'

%option noyywrap

%x S

%%

s          BEGIN(S);

<S>x   |

<S>.   |

<S><<EOF>>  { printf("[%d]", yyleng); yyless(0); BEGIN(INITIAL); }

y          printf("Y");

\n         ;

E

check bar-into-eof-rule bareof.l 'syssx' '[1]Y[1][1]x'



cat > "$T/fatal.l" <<'E'

%option noyywrap noyyalloc noyyrealloc noyyfree

%{