new flex

// meaning: "flex -oscripts/kconfig/lexer.lex.c -L scripts/kconfig/lexer.l"

// (scripts/Makefile.host: "cmd_flex = $(LEX) -o$@ -L $<").

//

// Only the CLI surface real-world callers actually use is supported:

// "flex [-L] -o OUTFILE INFILE" (attached "-oFILE", as the kernel invokes

// it, or separate "-o FILE"). "-L" (suppress #line directives) is accepted

// and is a no-op -- this generator never emits #line directives at all.

// Switches that select tables or diagnostics are accepted and change nothing

// (-C..., -b, -d, -f, -p, -v, -w, ...); this generator always builds one kind

// of table, and never emits #line directives (-L is implied).

#include <iostream>

#include <fstream>

#include <sstream>

#include <string>

#include <utility>

#include <vector>

#include "shell.h"

#include "flex/flex.h"

        FlexCommand():ShellCommand("flex"){}

        int runCommand(int argc,const char **argv){

            std::string outfile, infile;

            fresh::flex::FlexOptions cli;

            std::vector<std::pair<std::string,std::string>> settings;

            std::vector<std::string> infiles;

            bool toStdout = false;

            for(int i=1;i<argc;++i){

                std::string a=argv[i];

                if(a=="-L") continue;

                if(a=="-o"){

                    if(i+1>=argc){ std::cerr << "flex: -o requires an argument" << std::endl; return 2; }

                    outfile=argv[++i];

                if(a=="--version" || a=="-V"){ std::cout << "flex 2.6.4" << std::endl; return 0; }

                if(a=="--help" || a=="-h" || a=="-?"){

                    std::cout << "usage: flex [-LiItRsBIl78] [-o FILE] [-P PREFIX] [--header-file=FILE] [FILE...]" << std::endl;

                    return 0;

                }

                if(a=="-" ){ infiles.push_back(a); continue; }

                if(a.compare(0,2,"--")==0){

                    std::string name=a.substr(2), value;

                    size_t eq=name.find('=');

                    if(eq!=std::string::npos){ value=name.substr(eq+1); name=name.substr(0,eq); }

                    if(name=="stdout"){ toStdout=true; continue; }

                    if(name=="outfile" || name=="prefix" || name=="header-file"){

                        if(eq==std::string::npos){

                            if(i+1>=argc){ std::cerr << "flex: " << a << " requires an argument" << std::endl; return 2; }

                            value=argv[++i];

                        }

                    }

                    if(!cli.set(name,value)){ std::cerr << "flex: unknown option: " << a << std::endl; return 2; }

                    settings.emplace_back(name,value);

                    continue;

                }

                if(a.size()>2 && a[0]=='-' && a[1]=='o'){ outfile=a.substr(2); continue; }

                if(!a.empty() && a[0]=='-'){

                    std::cerr << "flex: unknown option: " << a << std::endl;

                if(a.size()>1 && a[0]=='-'){

                    bool stop=false;

                    for(size_t k=1;k<a.size() && !stop;++k){

                        char c=a[k];

                        auto argument=[&](std::string &to){

                            if(k+1<a.size()) to=a.substr(k+1);

                            else if(i+1<argc) to=argv[++i];

                            else return false;

                            stop=true;

                            return true;

                        };

                        std::string value;

                        switch(c){

                        case 'o': if(!argument(value)){ std::cerr << "flex: -o requires an argument" << std::endl; return 2; }

                                  settings.emplace_back("outfile",value); break;

                        case 'P': if(!argument(value)){ std::cerr << "flex: -P requires an argument" << std::endl; return 2; }

                                  settings.emplace_back("prefix",value); break;

                        case 'S': case 'T': case 'n': if(c=='S') argument(value); break;

                        case 'C': stop=true; break;

                        case 'i': settings.emplace_back("case-insensitive",""); break;

                        case 'I': settings.emplace_back("always-interactive",""); break;

                        case 'B': settings.emplace_back("never-interactive",""); break;

                        case 'R': settings.emplace_back("reentrant",""); break;

                        case 's': settings.emplace_back("nodefault",""); break;

                        case 't': toStdout=true; break;

                        case 'L': case 'b': case 'd': case 'f': case 'F': case 'l': case 'p': case 'v': case 'w':

                        case 'c': case '7': case '8': case 'X': case 'D': case '+':

                            if(c=='+'){ std::cerr << "flex: C++ scanners are not supported" << std::endl; return 2; }

                            break;

                        default:

                            std::cerr << "flex: unknown option: -" << c << std::endl;

                            return 2;

                        }

                infile=a;

                    }

            if(infile.empty() || outfile.empty()){

                std::cerr << "flex: usage: flex [-L] -o OUTFILE INFILE" << std::endl;

                return 2;

                    continue;

                }

                infiles.push_back(a);

            }

            std::stringstream ss;

            if(infiles.empty() || (infiles.size()==1 && infiles[0]=="-")) ss << std::cin.rdbuf();

            else for(auto &infile: infiles){

                std::ifstream in(infile.c_str());

                if(!in.is_open()){

                    std::cerr << "flex: " << infile << ": cannot open" << std::endl;

                    return 1;

                }

            std::stringstream ss; ss<<in.rdbuf();

                ss << in.rdbuf();

            }

            fresh::flex::LexReader reader(ss.str(), infile);

            fresh::flex::LexReader reader(ss.str(), infiles.empty() ? "<stdin>" : infiles[0]);

            fresh::flex::LexSpec spec;

            if(!reader.read(spec)){

                for(auto &e: reader.errors()) std::cerr << "flex: " << e << std::endl;

                return 1;

            }

            for(auto &s: settings) spec.options.set(s.first,s.second);

            fresh::flex::CodeGenerator codegen(spec);

            std::string source = codegen.generateSource();

                return 1;

            }

            if(!spec.options.headerFile.empty()){

                std::ofstream outH(spec.options.headerFile.c_str());

                if(!outH.is_open()){

                    std::cerr << "flex: " << spec.options.headerFile << ": cannot open for writing" << std::endl;

                    return 1;

                }

                outH << codegen.generateHeader();

            }

            if(toStdout){ std::cout << source; return 0; }

            std::string outfile=spec.options.outfile;

            if(outfile.empty()) outfile="lex."+spec.options.prefix+".c";

            std::ofstream outC(outfile.c_str());

            if(!outC.is_open()){

                std::cerr << "flex: " << outfile << ": cannot open for writing" << std::endl;

// everything that actually understands .l files lives here, same division

// of labor as lib/bison.

//

// Scope is deliberately bounded to what the Linux kernel's real

// scripts/kconfig/lexer.l (the actual target -- see kernel.spec's %build)

// uses: %x exclusive start conditions, <SC>{ ... } block scoping, <<EOF>>

// per-state rules, named definitions ({name}), POSIX bracket expressions

// incl. [:blank:]-style classes, trailing context (r1/r2), the yy_*

// buffer-stack API (yy_create_buffer/yy_switch_to_buffer/yy_delete_buffer),

// input()/unput(), BEGIN(), yylineno. NOT supported (not used by the

// target): %array, REJECT, yymore(), yy_scan_string/yy_scan_buffer, multiple

// input files via -f/--stdout skeleton customization, %top.

// Compatible with flex 2.6 for C scanners: reentrant ones, %option prefix,

// bison-bridge/-locations, the whole buffer API, yytext inside the input

// buffer, yymore(), REJECT, yyless()/unput()/input(), %array, start-condition

// stacks, trailing context, group flags and class set operators. Tested on

// the kernel's and PostgreSQL's scanners. Not there: C++ scanners (%option

// c++), which are an error, and serialized tables (--tables-file).

#pragma once

    int line = 0;

};

// %option settings; command line switches set the same ones.

struct FlexOptions {

    std::string prefix = "yy";

    std::string outfile, headerFile, extraType;

    bool reentrant = false;

    bool bisonBridge = false, bisonLocations = false;

    bool yywrap = true;

    bool stack = false;

    bool main = false;

    bool caseless = false;

    bool nodefault = false;

    bool array = false;

    bool noUnistd = false;

    bool noYyalloc = false, noYyrealloc = false, noYyfree = false;

    int interactive = -1;                   // -1: ask isatty()

    // One option by name, "no" prefix included. False if it isn't known.

    bool set(const std::string &name, const std::string &value);

};

struct LexSpec {

    std::vector<StartCondition> conditions; // conditions[0] is always INITIAL

    std::map<std::string,std::string> defs; // {name} -> raw pattern text (definitions section)

    std::vector<std::string> options;       // %option tokens, lowercased, as-seen (informational only)

    FlexOptions options;

    std::string top;                        // %top{ ... } block(s), verbatim, concatenated

    std::string prologue;                   // definitions-section %{ ... %} block(s), verbatim, concatenated

    std::string rulesPreamble;               // indented code appearing before the first real rule (verbatim)

    std::vector<Rule> rules;

    // the inner text (braces excluded); pos_ is left just past the closing

    // '}'.

    std::string readBraceBlock();

    std::string readComment();  // pos_ at "/*"; leaves it just past "*/"

    void parseOptions(const std::string &text);

    void parseDefinitionsSection();

    void parseRulesSection();

    // whitespace before the action): either a brace-delimited block or the

    // rest of the line. Consumes through end of line either way.

    void readActionInto(Rule &r);

    // Handles the rule(s) scoped to a "<SC1,SC2>" (or "<*>") prefix already

    // consumed by the caller: either a "<SC>{ ... }" block of multiple rules

    // (terminated by a line whose only content is "}"), or a single rule on

    // the same line.

    void applyConditionBlock(const std::vector<int> &conditions);

    bool atConditionPrefix() const;

    void parseScoped(const std::vector<int> &outer);

};

// ---- Pattern (regex) model ---------------------------------------------------

// ---- Bytecode ("Pike VM") ----------------------------------------------------

// A tiny Thompson-NFA bytecode, simulated at scan time via the classic

// "thread list" algorithm (Pike VM / Russ Cox's construction) rather than

// precompiled into a minimized DFA table the way real flex does it: with a

// handful of rules and kilobyte-scale Kconfig input files, raw NFA

// simulation is already far faster than the actual C compilation this feeds

// into, and it sidesteps having to implement DFA subset construction +

// equivalence-class minimization + table compression correctly. Emitted

// once as generic interpreter C code (FRESH_FLEX_RUNTIME_C in flex.cpp),

// shared by every compiled rule in the output file; each rule contributes

// only its own flat `fx_instr[]` array.

// A tiny Thompson-NFA bytecode. CodeGenerator subset-constructs a DFA from

// the rules' programs; those with trailing context also keep theirs in the

// output, where a Pike VM finds the split inside a finished match.

enum class Op { Char, Class, Any, Bol, Split, Jmp, Mark, Match };

struct Instr {

    Op op;

    int x = 0, y = 0; // Char: x=byte value. Class: x=class table index. Split: x,y=targets. Jmp: x=target.

    int x = 0, y = 0; // Char: x=byte value. Class: x=class table index. Any: x=matches newline too. Split: x,y=targets. Jmp: x=target. Match: x=rule.

};

struct CompiledProgram {

    std::vector<Instr> code;

    bool isEofRule = false;

    bool hasTrailing = false;

};

// Parses one already-isolated pattern token's text (e.g. "[ \t]*\n/[^ \t\n]")

public:

    // `defs` is LexSpec::defs (name -> raw pattern text, for {name}

    // expansion); `errors` collects diagnostics the same way LexReader does.

    PatternCompiler(const std::map<std::string,std::string> &defs, std::vector<std::string> &errors);

    PatternCompiler(const std::map<std::string,std::string> &defs, std::vector<std::string> &errors, bool caseless);

    // Parses `text` (no surrounding whitespace) into `out`. Returns false on

    // a malformed pattern (records an error).

    size_t p_ = 0;

    std::string context_;

    bool failed_ = false;

    bool defaultCaseless_ = false;

    bool caseless_ = false, dotAll_ = false, freeSpacing_ = false; // (?i (?s (?x

    void failHere(const std::string &msg);

    char cpeek(size_t ahead=0) const;

    ReNode *parseBracketClass();

    ReNode *parseQuotedString();

    ReNode *parseNamedDef();

    ReNode *parseGroup();

    ReNode *makeChar(unsigned char ch);

    void foldCase(CharClass &cc);

    void skipFreeSpacing();

    unsigned char parseEscapedChar(); // consumes a backslash escape, returns the resulting byte

    int internClass(CharClass &&cc);

    void addPosixClass(CharClass &cc, const std::string &name); // "alpha","digit","blank", etc.

    // Returns the generated .c file content, or "" if a pattern failed to

    // compile (check errors() in that case).

    std::string generateSource();

    std::string generateHeader();           // for --header-file

    const std::vector<std::string> &errors() const { return errors_; }

private: