#include <layer_cache.hpp>

Classes
struct	LookupResult

Public Member Functions
	LayerKVCacheIndex (std::size_t max_entries=4096, std::size_t max_bytes=256 1024 1024)

LookupResult	find_deepest (const std::string &model_id, const std::string &decode_hash, const std::vector< std::int32_t > &tokens, std::int32_t total_layers, std::uint64_t arch_version, const std::string &cache_namespace={}) const

void	insert (LayerCheckpoint checkpoint)

void	invalidate_model (const std::string &model_id)

void	invalidate_arch (std::uint64_t arch_version)

void	clear ()

std::size_t	size () const

std::size_t	max_entries () const
	Capacity in entries passed at construction.

std::size_t	max_bytes () const
	Byte budget passed at construction.

std::size_t	estimated_bytes () const
	Sum of the resident checkpoints' `estimated_bytes`.

Detailed Description

Per-layer KV checkpoint tier.

Eviction: least-recently-used under two bounds, max_entries and max_bytes (the sum of each checkpoint's caller-supplied estimated_bytes). A find_deepest hit and insert both refresh recency; entries are dropped oldest-first until both bounds hold.

Constructor & Destructor Documentation

◆ LayerKVCacheIndex()

continuum::runtime::LayerKVCacheIndex::LayerKVCacheIndex	(	std::size_t	max_entries = `4096`,
		std::size_t	max_bytes = `256 1024 1024`
	)

explicit

Member Function Documentation

◆ clear()

void continuum::runtime::LayerKVCacheIndex::clear ( )

◆ estimated_bytes()

std::size_t continuum::runtime::LayerKVCacheIndex::estimated_bytes ( ) const

Sum of the resident checkpoints' estimated_bytes.

◆ find_deepest()

LookupResult continuum::runtime::LayerKVCacheIndex::find_deepest	(	const std::string &	model_id,
		const std::string &	decode_hash,
		const std::vector< std::int32_t > &	tokens,
		std::int32_t	total_layers,
		std::uint64_t	arch_version,
		const std::string &	cache_namespace = `{}`
	)		const

◆ insert()

void continuum::runtime::LayerKVCacheIndex::insert ( LayerCheckpoint checkpoint )

◆ invalidate_arch()

void continuum::runtime::LayerKVCacheIndex::invalidate_arch ( std::uint64_t arch_version )

◆ invalidate_model()

void continuum::runtime::LayerKVCacheIndex::invalidate_model ( const std::string & model_id )

◆ max_bytes()

std::size_t continuum::runtime::LayerKVCacheIndex::max_bytes ( ) const

inline

Byte budget passed at construction.

◆ max_entries()

std::size_t continuum::runtime::LayerKVCacheIndex::max_entries ( ) const