BpeVocabulary.ixx File Reference

BPE vocabulary for GPT-2, Llama 3.x, and Mistral model families. More...

#include <string>
#include <string_view>
#include <vector>
#include <unordered_map>
#include <unordered_set>
#include <optional>
#include <filesystem>
#include <fstream>
#include <sstream>
#include <algorithm>
#include <stdexcept>
#include <cstdint>
#include <chrono>
#include <iostream>
#include <iomanip>
import Serialization.Metadata;
import Data.FileHeader;
import Data.BpePreTokenizationMode;
import Data.TokenizerVocabulary;
import Data.BpeVocabularyConfig;
import Data.Tokenizer;
import Data.SpecialTokens;

Classes
class	Mila::Data::BpeVocabulary
	Unified Byte Pair Encoding (BPE) vocabulary. More...
struct	Mila::Data::BpeVocabulary::PairHash
struct	Mila::Data::BpeVocabulary::PairViewHash

Namespaces
namespace	Mila
	Mila main API namespace.
namespace	Mila::Data

Typedefs
using	Mila::Data::TokenId

Detailed Description

BPE vocabulary for GPT-2, Llama 3.x, and Mistral model families.