TokenizerType.ixx File Reference

#include <cstdint>
#include <string_view>
#include <string>

Namespaces
namespace	Mila
	Mila main API namespace.
namespace	Mila::Data

Enumerations
enum class	Mila::Data::TokenizerType : uint8_t {   Mila::Data::Unknown = 0 , Mila::Data::Char , Mila::Data::Bpe , Mila::Data::SentencePiece ,   Mila::Data::Word , Mila::Data::Unigram }
	Tokenizer type discriminator used across tokenizer and vocabulary types. More...

Functions
TokenizerType	Mila::Data::stringToTokenizerType (std::string_view s) noexcept
	Parse a string into TokenizerType.
std::string_view	Mila::Data::tokenizerTypeToString (TokenizerType t) noexcept
	Convert TokenizerType to a stable string representation.