tokenizer.h

使用具有增量学习的监控式学习方法。包括几个不同的分类算法。

#ifndef TOKENIZER_H
#define TOKENIZER_H

#include <stdio.h>
#include "languages.h"
#include "stemmer.h"
#include "stopword.h"

typedef struct tokenizer_ tokenizer;

tokenizer *
tokenizer_new (const char *str);

tokenizer *
tokenizer_alpha_new (void);

tokenizer *
tokenizer_ws_new (void);

tokenizer *
tokenizer_ngram_new (void);

tokenizer *
tokenizer_null_new (void);

int
tokenizer_set_minmax (tokenizer *tok, int min, int max);

void
tokenizer_set_stopwords (tokenizer *tok, word_stopper *ws);

void
tokenizer_set_stemmer (tokenizer *tok, stemmer_functions *sf);

void
tokenizer_set_languages (tokenizer *tok, languages *langs);

int
tokenizer_set_language (tokenizer *tok, const char *lang);

void
tokenizer_set_text (tokenizer *tok, const char *text, int size,
		    const char *code);

const char *
tokenizer_next_token (tokenizer *tok);

int
tokenizer_save (tokenizer *tok, FILE *f);

#endif