ares_eval

// internal/ares_eval
type Duration time.Duration  // YAML/JSON-unmarshalable duration

type TestCase struct {
    ID            string
    Name          string
    Input         string
    ExpectedOutput string
    ExpectedTools []string
    Timeout       Duration
    Metadata      map[string]interface{}
    Tags          []string
}

type TestSuite struct {
    Name        string
    Description string
    TestCases   []TestCase
    Tags        []string
}

type TestResult struct {
    TestCaseID  string
    ActualOutput string
    ToolsUsed   []string
    Duration    time.Duration
    TokensUsed  int
    Error       string
    Metrics     map[string]float64
    Timestamp   time.Time
}

type EvalScore struct {
    Metric  string
    Score   float64
    Details string
}

type Evaluator interface {
    Evaluate(ctx context.Context, testCase TestCase, result TestResult) ([]EvalScore, error)
}

type ExactMatchEvaluator struct{}
func NewExactMatchEvaluator() *ExactMatchEvaluator

type KeywordPresenceEvaluator struct{ Keywords []string }
func NewKeywordPresenceEvaluator(keywords []string) *KeywordPresenceEvaluator

type ToolUsageEvaluator struct{}
func NewToolUsageEvaluator() *ToolUsageEvaluator

type LLMClient interface {
    Generate(ctx context.Context, prompt string) (string, error)
}

type ScaleType int  // ScaleOneToTen, ScaleOneToFive, ScalePassFail

type LLMJudgeEvaluator struct { /* unexported */ }
type LLMJudgeOption func(*LLMJudgeEvaluator)
func WithPrompt(tmpl string) LLMJudgeOption
func WithScale(scale ScaleType) LLMJudgeOption
func WithChinesePrompt() LLMJudgeOption
func WithEnglishPrompt() LLMJudgeOption
func WithDimensionAveraging() LLMJudgeOption
func NewLLMJudgeEvaluator(client LLMClient, opts ...LLMJudgeOption) (*LLMJudgeEvaluator, error)
func (e *LLMJudgeEvaluator) Evaluate(ctx context.Context, tc TestCase, result TestResult) ([]EvalScore, error)
func (e *LLMJudgeEvaluator) Name() string

type EvaluatorRegistry struct { /* unexported */ }
func NewEvaluatorRegistry() *EvaluatorRegistry
func (r *EvaluatorRegistry) Register(name string, eval Evaluator) error
func (r *EvaluatorRegistry) Get(name string) (Evaluator, bool)
func (r *EvaluatorRegistry) Names() []string

type TestRunner interface {
    RunSuite(ctx context.Context, suite TestSuite) ([]TestResult, error)
    RunSingle(ctx context.Context, testCase TestCase) (TestResult, error)
}

type AgentExecutor interface {
    Execute(ctx context.Context, input string) (output string, toolsUsed []string, tokensUsed int, err error)
}

type AgentTestRunner struct { /* unexported */ }
func NewAgentTestRunner(executor AgentExecutor) (*AgentTestRunner, error)
func (r *AgentTestRunner) SetRegistry(registry *EvaluatorRegistry)
func (r *AgentTestRunner) RunSuite(ctx context.Context, suite TestSuite) ([]TestResult, error)
func (r *AgentTestRunner) RunSingle(ctx context.Context, testCase TestCase) (TestResult, error)
func (r *AgentTestRunner) RunAndEvaluate(ctx context.Context, suite TestSuite, evaluatorName string) ([]TestResult, [][]EvalScore, error)

type ConcurrentRunnerConfig struct {
    MaxParallel int
    Timeout     time.Duration
}
func DefaultConcurrentRunnerConfig() ConcurrentRunnerConfig

type ConcurrentRunner struct { /* unexported */ }
func NewConcurrentRunner(inner TestRunner, config ConcurrentRunnerConfig) (*ConcurrentRunner, error)
func (r *ConcurrentRunner) RunSuite(ctx context.Context, suite TestSuite) ([]TestResult, error)
func (r *ConcurrentRunner) RunSingle(ctx context.Context, testCase TestCase) (TestResult, error)

type AgentConfig struct {
    Name         string
    Model        string
    Parameters   map[string]any
    SystemPrompt string
}

type ComparisonResult struct {
    ConfigName       string
    Config           AgentConfig
    Results          []TestResult
    AggregatedScores map[string]float64
    OverallScore     float64
    TotalDuration    time.Duration
    PassRate         float64
    Error            string
}

type LeaderboardEntry struct {
    Rank       int
    ConfigName string
    Overall    float64
    Scores     map[string]float64
}

type ComparisonReport struct {
    SuiteName   string
    Timestamp   time.Time
    Results     []ComparisonResult
    Leaderboard []LeaderboardEntry
    Summary     ComparisonSummary
}

type ComparisonRunner struct { /* unexported */ }
func NewComparisonRunner(suite *TestSuite, runnerFactory func(AgentConfig) (TestRunner, error)) *ComparisonRunner
func (r *ComparisonRunner) Run(ctx context.Context, configs []AgentConfig) (*ComparisonReport, error)
func (r *ComparisonRunner) GenerateMarkdown(report *ComparisonReport) (string, error)
func GenerateLeaderboard(results []ComparisonResult) []LeaderboardEntry

type Loader struct{}
func NewLoader() *Loader
func (l *Loader) Load(path string) (*TestSuite, error)
func (l *Loader) LoadDir(dir string) ([]TestSuite, error)

type ReportGenerator struct{}
func NewReportGenerator() *ReportGenerator
func (g *ReportGenerator) GenerateMarkdown(suite TestSuite, results []TestResult, scores [][]EvalScore) (string, error)
func (g *ReportGenerator) GenerateJSON(suite TestSuite, results []TestResult, scores [][]EvalScore) (string, error)
func (g *ReportGenerator) SaveReport(path string, content string) error
func RunEvaluation(ctx context.Context, loader *Loader, runner TestRunner, evaluator Evaluator, suitePath string) ([]TestResult, [][]EvalScore, error)

// internal/ares_eval/service
type Service struct { /* unexported */ }
type Option func(*Service)
func WithAgentExecutor(exec ares_eval.AgentExecutor) Option
func NewService(repo EvalResultRepository, opts ...Option) (*Service, error)
func (s *Service) RunEval(ctx context.Context, req *RunEvalRequest) (*RunEvalResponse, error)
func (s *Service) GetResults(ctx context.Context, runID string) (*GetResultsResponse, error)
func (s *Service) GetLeaderboard(ctx context.Context, limit, offset int) (*LeaderboardResponse, error)
func (s *Service) GetComparison(ctx context.Context, runIDs []string) (*ComparisonResponse, error)

type EvalResult struct {
    ID            string
    RunID         string
    ConfigName    string
    SuiteName     string
    TestCaseID    string
    TestCaseName  string
    Score         float64
    Dimensions    map[string]float64
    Status        string
    ErrorMessage  *string
    DurationMs    int
    CreatedAt     time.Time
    UpdatedAt     time.Time
}

type EvalResultRepository interface {
    Store(ctx context.Context, result *EvalResult) error
    StoreBatch(ctx context.Context, results []*EvalResult) error
    GetByRunID(ctx context.Context, runID string) ([]*EvalResult, error)
    GetLeaderboard(ctx context.Context, limit, offset int) ([]*LeaderboardEntry, int, error)
    GetComparison(ctx context.Context, runIDs []string) ([]*ComparisonRow, error)
}

type Handler struct { /* unexported */ }
func NewHandler(svc *Service) *Handler
func (h *Handler) HandleRunEval(w http.ResponseWriter, r *http.Request)
func (h *Handler) HandleGetResults(w http.ResponseWriter, r *http.Request)
func (h *Handler) HandleGetLeaderboard(w http.ResponseWriter, r *http.Request)
func (h *Handler) HandleGetComparison(w http.ResponseWriter, r *http.Request)