File tree 1 file changed +5
-8
lines changed
1 file changed +5
-8
lines changed Original file line number Diff line number Diff line change @@ -36,12 +36,11 @@ class HFTokenizer : public Tokenizer {
36
36
return ret;
37
37
}
38
38
39
- // use i32 to be consistent with sentencepiece
40
- std::vector<int32_t > Encode (const std::string& text) final {
41
- return Encode (text, false );
42
- }
39
+ // use i32 to be consistent with sentencepiece
40
+ std::vector<int32_t > Encode (const std::string& text) final { return Encode (text, false ); }
43
41
44
- std::vector<std::vector<int32_t >> EncodeBatch (const std::vector<std::string>& texts, bool add_special_tokens) final {
42
+ std::vector<std::vector<int32_t >> EncodeBatch (const std::vector<std::string>& texts,
43
+ bool add_special_tokens) {
45
44
std::vector<const char *> texts_raw;
46
45
std::vector<size_t > seq_lens;
47
46
size_t num_seqs = texts.size ();
@@ -78,9 +77,7 @@ class HFTokenizer : public Tokenizer {
78
77
return std::string (data, len);
79
78
}
80
79
81
- std::string Decode (const std::vector<int32_t >& ids) final {
82
- return Decode (ids, false );
83
- }
80
+ std::string Decode (const std::vector<int32_t >& ids) final { return Decode (ids, false ); }
84
81
85
82
size_t GetVocabSize () final {
86
83
size_t size;
You can’t perform that action at this time.
0 commit comments