我无法获取令牌的字符串值

I can't get the string value of a token

本文关键字：字符串 令牌 获取      更新时间：2023-10-16

我尝试用Boost Spirit实现一个小编程语言的Lexer。

我必须得到一个令牌的值，我得到一个bad_get异常:

抛出'boost::bad_get'实例后终止调用
what(): boost::bad_get: failed value

我在执行

时得到这个异常
std::string contents = "void";
base_iterator_type first = contents.begin();
base_iterator_type last = contents.end();
SimpleLexer<lexer_type> lexer;
iter = lexer.begin(first, last);
end = lexer.end();
std::cout << "Value = " << boost::get<std::string>(iter->value()) << std::endl;

我的词法分析器定义如下:

typedef std::string::iterator base_iterator_type;
typedef boost::spirit::lex::lexertl::token<base_iterator_type, boost::mpl::vector<unsigned int, std::string>> Tok;
typedef lex::lexertl::actor_lexer<Tok> lexer_type;
template<typename L>
class SimpleLexer : public lex::lexer<L> {
    private:
    public:
        SimpleLexer() {
            keyword_for = "for";
            keyword_while = "while";
            keyword_if = "if";
            keyword_else = "else";
            keyword_false = "false";
            keyword_true = "true";
            keyword_from = "from";
            keyword_to = "to";
            keyword_foreach = "foreach";
            word = "[a-zA-Z]+";
            integer = "[0-9]+";
            litteral = "...";
            left_parenth = '('; 
            right_parenth = ')'; 
            left_brace = '{'; 
            right_brace = '}'; 
            stop = ';';
            comma = ',';
            swap = "<>";
            assign = '=';
            addition = '+';
            subtraction = '-';
            multiplication = '*';
            division = '/';
            modulo = '%';
            equals = "==";
            not_equals = "!=";
            greater = '>';
            less = '<';
            greater_equals = ">=";
            less_equals = "<=";
            whitespaces = "[ \t\n]+";
            comments = "\/\*[^*]*\*+([^/*][^*]*\*+)*\/";
            //Add keywords
            this->self += keyword_for | keyword_while | keyword_true | keyword_false | keyword_if | keyword_else | keyword_from | keyword_to | keyword_foreach;
            this->self += integer | litteral | word;
            this->self += equals | not_equals | greater_equals | less_equals | greater | less ;
            this->self += left_parenth | right_parenth | left_brace | right_brace;
            this->self += comma | stop;
            this->self += assign | swap | addition | subtraction | multiplication | division | modulo;
            //Ignore whitespaces and comments
            this->self += whitespaces [lex::_pass = lex::pass_flags::pass_ignore];
            this->self += comments [lex::_pass = lex::pass_flags::pass_ignore]; 
        }
        lex::token_def<std::string> word, litteral, integer;
        lex::token_def<lex::omit> left_parenth, right_parenth, left_brace, right_brace;
        lex::token_def<lex::omit> stop, comma;
        lex::token_def<lex::omit> assign, swap, addition, subtraction, multiplication, division, modulo;
        lex::token_def<lex::omit> equals, not_equals, greater, less, greater_equals, less_equals;
        //Keywords
        lex::token_def<lex::omit> keyword_if, keyword_else, keyword_for, keyword_while, keyword_from, keyword_to, keyword_foreach;
        lex::token_def<lex::omit> keyword_true, keyword_false;
        //Ignored tokens
        lex::token_def<lex::omit> whitespaces;
        lex::token_def<lex::omit> comments;
};

是否有其他方法来获取Token的值?

你总是可以使用'default'标记data(它是源迭代器类型的iterator_range)。

std::string tokenvalue(iter->value().begin(), iter->value().end());

在研究了boost存储库中的测试用例之后，我发现了许多事情:

  • 这是由设计
  • 有一个更简单的方法
  • 更容易的方式在Lex语义操作(例如使用_1)和Qi中使用词法分析器令牌时自动实现;赋值将自动转换为Qi属性类型
  • 这(确实)得到了文档
    • 中提到的"懒惰，一次性，评估"语义。

问题在于令牌数据是可变的，它从原始输入迭代器范围开始。只有在'和'强制赋值之后，转换后的属性才缓存在变体中。您可以见证转换:

lexer_type::iterator_type iter = lexer.begin(first, last);
lexer_type::iterator_type end = lexer.end();
assert(0 == iter->value().which());
std::cout << "Value = " << boost::get<boost::iterator_range<base_iterator_type> >(iter->value()) << std::endl;
std::string s;
boost::spirit::traits::assign_to(*iter, s);
assert(1 == iter->value().which());
std::cout << "Value = " << s << std::endl;

正如你所看到的，这里的属性分配是强制的，直接使用assign_to trait实现。

完整的工作演示:

#include <boost/spirit/include/lex_lexertl.hpp>
#include <iostream>
#include <string>
namespace lex = boost::spirit::lex;
typedef std::string::iterator base_iterator_type;
typedef boost::spirit::lex::lexertl::token<base_iterator_type, boost::mpl::vector<int, std::string>> Tok;
typedef lex::lexertl::actor_lexer<Tok> lexer_type;
template<typename L>
class SimpleLexer : public lex::lexer<L> {
    private:
    public:
        SimpleLexer() {
            word = "[a-zA-Z]+";
            integer = "[0-9]+";
            literal = "...";
            this->self += integer | literal | word;
        }
        lex::token_def<std::string> word, literal;
        lex::token_def<int> integer;
};
int main(int argc, const char* argv[]) {
    SimpleLexer<lexer_type> lexer;
    std::string contents = "void";
    base_iterator_type first = contents.begin();
    base_iterator_type last = contents.end();
    lexer_type::iterator_type iter = lexer.begin(first, last);
    lexer_type::iterator_type end = lexer.end();
    assert(0 == iter->value().which());
    std::cout << "Value = " << boost::get<boost::iterator_range<base_iterator_type> >(iter->value()) << std::endl;
    std::string s;
    boost::spirit::traits::assign_to(*iter, s);
    assert(2 == iter->value().which());
    std::cout << "Value = " << s << std::endl;
    return 0;
}
相关文章：