File size: 2,430 Bytes
26d5b81 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 | #ifndef NEUROFLOW_TOKENIZER_HPP
#define NEUROFLOW_TOKENIZER_HPP
#include <algorithm>
#include <fstream>
#include <iostream>
#include <memory>
#include <queue>
#include <sstream>
#include <string>
#include <unordered_map>
#include <vector>
namespace neuroflow {
class Tokenizer {
public:
virtual ~Tokenizer() = default;
virtual std::vector<size_t> encode(const std::string& text, size_t max_len = 0) = 0;
virtual std::string decode(const std::vector<size_t>& ids) = 0;
size_t vocab_size() const { return vocab_size_; }
size_t pad_id() const { return pad_id_; }
size_t unk_id() const { return unk_id_; }
size_t bos_id() const { return bos_id_; }
size_t eos_id() const { return eos_id_; }
protected:
size_t vocab_size_ = 128000;
size_t pad_id_ = 0;
size_t unk_id_ = 1;
size_t bos_id_ = 2;
size_t eos_id_ = 3;
size_t max_input_len_ = 128;
};
class BPETokenizer : public Tokenizer {
public:
BPETokenizer(const std::string& config_path);
BPETokenizer();
std::vector<size_t> encode(const std::string& text, size_t max_len = 0) override;
std::string decode(const std::vector<size_t>& ids) override;
void add_vocab(const std::string& token, size_t id);
void set_vocab_size(size_t vs);
private:
std::unordered_map<std::string, size_t> vocab_;
std::unordered_map<size_t, std::string> id_to_token_;
std::vector<std::pair<std::string, std::string>> merges_;
std::unordered_map<std::string, size_t> merge_ranks_;
std::vector<uint8_t> utf8_to_bytes(const std::string& text);
std::string bytes_to_utf8(const std::string& raw);
std::string apply_bpe(const std::string& token);
void parse_config(const std::string& content);
};
class WordPieceTokenizer : public Tokenizer {
public:
WordPieceTokenizer();
WordPieceTokenizer(const std::string& config_path);
std::vector<size_t> encode(const std::string& text, size_t max_len = 0) override;
std::string decode(const std::vector<size_t>& ids) override;
void add_vocab(const std::string& token, size_t id);
void set_vocab_size(size_t vs);
private:
std::unordered_map<std::string, size_t> vocab_;
std::unordered_map<size_t, std::string> id_to_token_;
void parse_vocab(const std::string& content);
};
} // namespace neuroflow
#endif // NEUROFLOW_TOKENIZER_HPP |