token.h

用python写的分词程序,实现的是最大匹配方法,简单易用

#ifndef _TOKEN_H_
#define _TOKEN_H_

namespace rmmseg
{
    struct Token
    {
        Token(const char *txt, int len)
            :text(txt), length(len) { }
        // `text' may or may not be nul-terminated, its length
        // should be stored in the `length' field.
        //
        // if length is 0, this is an empty token
        const char *text;
        int length;
    };
}

#endif /* _TOKEN_H_ */