1cfe5aee76
- Created tables for AI benchmark runs and results to facilitate model evaluation. - Added expected answers column to benchmark results. - Introduced tables for internet connection change cases and vTiger archive management, including records, relations, and checkpoints. - Implemented triggers to enforce append-only behavior for vTiger archive records and files. - Enhanced solution management with soft delete capabilities for sag_solutions and knowledge_articles. feat(scripts): add CRM benchmarking script for Ollama models - Developed a Python script to benchmark CRM models exposed through Ollama, including various test cases and scoring mechanisms. test(tests): add comprehensive tests for new features - Implemented tests for internet change case service, vTiger archive functionality, and sag solution knowledge management. - Ensured coverage for edge cases and error handling in the new features.
42 lines
1.5 KiB
SQL
42 lines
1.5 KiB
SQL
CREATE TABLE IF NOT EXISTS ai_benchmark_runs (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
status VARCHAR(24) NOT NULL DEFAULT 'queued',
|
|
models JSONB NOT NULL DEFAULT '[]'::jsonb,
|
|
test_keys JSONB NOT NULL DEFAULT '[]'::jsonb,
|
|
total_cases INTEGER NOT NULL DEFAULT 0,
|
|
completed_cases INTEGER NOT NULL DEFAULT 0,
|
|
created_by INTEGER REFERENCES users(user_id) ON DELETE SET NULL,
|
|
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
started_at TIMESTAMPTZ,
|
|
completed_at TIMESTAMPTZ,
|
|
error_text TEXT
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS ai_benchmark_results (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
run_id BIGINT NOT NULL REFERENCES ai_benchmark_runs(id) ON DELETE CASCADE,
|
|
model VARCHAR(160) NOT NULL,
|
|
test_key VARCHAR(100) NOT NULL,
|
|
test_name VARCHAR(200) NOT NULL,
|
|
category VARCHAR(80) NOT NULL,
|
|
score INTEGER NOT NULL DEFAULT 0,
|
|
max_score INTEGER NOT NULL DEFAULT 0,
|
|
duration_ms INTEGER NOT NULL DEFAULT 0,
|
|
prompt_tokens INTEGER,
|
|
response_tokens INTEGER,
|
|
tokens_per_second NUMERIC(10, 2),
|
|
passed_checks JSONB NOT NULL DEFAULT '[]'::jsonb,
|
|
failed_checks JSONB NOT NULL DEFAULT '[]'::jsonb,
|
|
response_json JSONB,
|
|
response_text TEXT,
|
|
error_text TEXT,
|
|
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
UNIQUE(run_id, model, test_key)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_ai_benchmark_runs_created
|
|
ON ai_benchmark_runs(created_at DESC);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_ai_benchmark_results_run
|
|
ON ai_benchmark_results(run_id, model);
|