Mila::Data::CharTokenizer Member List

This is the complete list of members for Mila::Data::CharTokenizer, including all inherited members.

CharTokenizer(CharVocabulary vocab)	Mila::Data::CharTokenizer	inlineexplicit
decode(std::span< const TokenId > tokens) override	Mila::Data::CharTokenizer	inlinevirtual
encode(const std::string &text) override	Mila::Data::CharTokenizer	inlinevirtual
getBosTokenId() const override	Mila::Data::CharTokenizer	inlinevirtual
getEosTokenId() const override	Mila::Data::CharTokenizer	inlinevirtual
getPadTokenId() const override	Mila::Data::CharTokenizer	inlinevirtual
getVocabSize() const override	Mila::Data::CharTokenizer	inlinevirtual
isValidToken(TokenId tokenId) const override	Mila::Data::CharTokenizer	inlinevirtual
load(const std::filesystem::path &path)	Mila::Data::CharTokenizer	inlinestatic
tokenToString(TokenId tokenId) const override	Mila::Data::CharTokenizer	inlinevirtual
vocab_	Mila::Data::CharTokenizer	private
~Tokenizer()=default	Mila::Data::Tokenizer	virtual