conformance.hpp

Go to the documentation of this file.

#pragma once
 
#include <continuum/backend/backend.hpp>
 
#include <cstdint>
#include <string>
#include <vector>
 
namespace continuum::backend {
 
enum class ConformanceStatus { Pass, Fail, Skip };
 
struct ConformanceCheck {
  std::string name;
  ConformanceStatus status = ConformanceStatus::Skip;
  std::string detail;
};
 
struct ConformanceOptions {
  std::string model_id = "conformance/model";
  std::string prompt = "You are a careful assistant. Answer briefly. Question: what is a prefix cache?";
  std::int32_t prefix_len = -1;
  std::int32_t max_tokens = 16;
  bool expect_deterministic = true;
  std::string tensor_op = "identity";
};
 
struct ConformanceReport {
  std::string backend;
  std::vector<ConformanceCheck> checks;
 
  bool passed() const;
  std::string summary() const;
};
 
ConformanceReport RunBackendConformance(Backend& backend, const std::string& name,
                                        const ConformanceOptions& options = {});
 
}  // namespace continuum::backend