MultiHeadAttention.ixx File Reference

Implementation of multi-head attention mechanism for transformer architectures. More...

#include <memory>
#include <vector>
#include <string>
#include <iostream>
#include <sstream>
#include <type_traits>
#include <cmath>
import Serialization.ModelArchive;
import Compute.Precision;
import Compute.CudaMemoryResource;
import Compute.CpuMemoryResource;
import Compute.MemoryResource;
import Dnn.Modules.Attention:Config;
import Dnn.Module;
import Dnn.Tensor;
import Compute.OperationBase;
import Compute.UnaryOperation;
import Dnn.TensorTraits;
import Dnn.TensorHelpers;
import Compute.DeviceContext;
import Compute.DeviceType;
import Compute.CudaDevice;
import Compute.CpuDevice;
import Compute.OperationAttributes;
import Compute.OperationRegistry;

Classes
class	Mila::Dnn::MultiHeadAttention< TDeviceType, TInput, TOutput >
	Multi-head attention module for transformer architectures. More...

Namespaces
namespace	Mila
namespace	Mila::Dnn

Typedefs
template<typename TInput = float, typename TOutput = TInput>
using	Mila::Dnn::CpuMultiHeadAttention = MultiHeadAttention< DeviceType::Cpu, TInput, TOutput >
	Type alias for CPU-based multi-head attention module with customizable tensor types.
template<typename TInput = float, typename TOutput = TInput>
using	Mila::Dnn::CudaMultiHeadAttention = MultiHeadAttention< DeviceType::Cuda, TInput, TOutput >
	Type alias for CUDA-based multi-head attention module with customizable tensor types.

Detailed Description

Implementation of multi-head attention mechanism for transformer architectures.