Mila 0.13.48
Deep Neural Network Library
Loading...
Searching...
No Matches
Mila::Data::CharVocabulary Member List

This is the complete list of members for Mila::Data::CharVocabulary, including all inherited members.

addRegularTokens(const std::vector< unsigned char > &sorted_bytes)Mila::Data::CharVocabularyinlineprivate
addSpecialTokensFromConfig()Mila::Data::CharVocabularyinlineprivate
buildFromText(const std::string &corpus)Mila::Data::CharVocabularyinlineprivate
char_to_idx_Mila::Data::CharVocabularyprivate
charToIndex(char c) constMila::Data::CharVocabularyinline
CharVocabulary()=deleteMila::Data::CharVocabulary
CharVocabulary(const CharVocabularyConfig &config)Mila::Data::CharVocabularyinlineexplicitprivate
config_Mila::Data::CharVocabularyprivate
extractUniqueBytes(const std::string &text) constMila::Data::CharVocabularyinlineprivate
getConfig() constMila::Data::CharVocabularyinline
getSize() const overrideMila::Data::CharVocabularyinlinevirtual
hasSpecialTokens() constMila::Data::CharVocabularyinline
idToToken(TokenId id) const overrideMila::Data::CharVocabularyinlinevirtual
idx_to_char_Mila::Data::CharVocabularyprivate
indexToChar(TokenId idx) constMila::Data::CharVocabularyinline
load(const fs::path &path)Mila::Data::CharVocabularyinlinestatic
loadContent(std::istream &file)Mila::Data::CharVocabularyinlineprivate
normalizeText(const std::string &text) constMila::Data::CharVocabularyinlineprivate
pad_token_id_Mila::Data::CharVocabularyprivate
padTokenId() constMila::Data::CharVocabularyinline
save(const fs::path &path) const overrideMila::Data::CharVocabularyinline
Mila::Data::TokenizerVocabulary::save(const std::filesystem::path &path) const =0Mila::Data::TokenizerVocabularypure virtual
saveContent(std::ostream &file) constMila::Data::CharVocabularyinlineprivate
sortBytes(const std::unordered_map< unsigned char, bool > &unique_bytes) constMila::Data::CharVocabularyinlineprivate
tokenToId(const std::string &token) const overrideMila::Data::CharVocabularyinlinevirtual
train(const std::string &corpus, const CharVocabularyConfig &config)Mila::Data::CharVocabularyinlinestatic
trainFromFile(const fs::path &corpus_path, const CharVocabularyConfig &config)Mila::Data::CharVocabularyinlinestatic
unk_token_id_Mila::Data::CharVocabularyprivate
unkTokenId() constMila::Data::CharVocabularyinline
~TokenizerVocabulary()=defaultMila::Data::TokenizerVocabularyvirtual