User Guide
Reference
Changelog
API
/
eval.list_custom_metrics()
API Index
Getting Started
ChatBot
Conversation
Message
ChatBot Methods
ChatBot.model()
ChatBot.provider_model()
ChatBot.preset()
ChatBot.temperature()
ChatBot.max_tokens()
ChatBot.persona()
ChatBot.system_prompt()
ChatBot.get_system_prompt()
ChatBot.avoid()
ChatBot.get_avoid_topics()
ChatBot.tools()
ChatBot.add_tools()
ChatBot.verbose()
ChatBot.chat()
ChatBot.start_conversation()
ChatBot.continue_conversation()
ChatBot.show()
ChatBot.get_config_summary()
ChatBot.get_config()
ChatBot.prompt_builder()
ChatBot.structured_prompt()
ChatBot.chain_prompts()
ChatBot.check_llm_status()
ChatBot.quick_start()
ChatBot.create_chat_session()
Prompt Engineering
PromptBuilder
PromptBuilder.persona()
PromptBuilder.task_context()
PromptBuilder.focus_on()
PromptBuilder.critical_constraint()
PromptBuilder.constraint()
PromptBuilder.core_analysis()
PromptBuilder.structured_section()
PromptBuilder.avoid_topics()
PromptBuilder.output_format()
PromptBuilder.example()
PromptBuilder.final_emphasis()
PromptBuilder.vocabulary()
PromptBuilder.pathways()
PromptBuilder.print()
Priority
PromptSection
VocabularyTerm
architectural_analysis_prompt()
code_review_prompt()
debugging_prompt()
Conversation Pathways
Pathways
Pathways.state()
Pathways.required()
Pathways.optional()
Pathways.tools()
Pathways.success_condition()
Pathways.next_state()
Pathways.branch_on()
Pathways.fallback()
Pathways.agent()
Pathways.register_agent()
Pathways.get_agent()
Pathways.visualize()
Conversation Methods
Conversation.add_message()
Conversation.add_user_message()
Conversation.add_assistant_message()
Conversation.add_system_message()
Conversation.get_messages()
Conversation.get_last_message()
Conversation.clear_messages()
Conversation.set_context_window()
Conversation.get_context_messages()
Conversation.get_message_count()
Conversation.to_dict()
Conversation.from_dict()
File Attachments
Attachments
Attachments.with_prompt()
Attachments.to_chat_contents()
Attachments.get_metadata()
Attachments.summary()
AttachmentMetadata
Tools
tool()
ToolCategory
ToolContext
ToolResult
get_global_registry()
load_tool_box()
get_builtin_tool()
load_selected_tools()
Testing & Validation
autotest_avoid_topics()
autotest_pathways()
TestResults
TestResults.to_dataframe()
TestResults.to_great_table()
PathwayTestResults
ToolEnabledConversation
create_tool_conversation()
Guardrails
Guard
Guard.check()
GuardAction
GuardPhase
GuardResult
GuardPipeline
GuardPipeline.add()
GuardPipeline.run()
GuardPipelineResult
guardrail()
ChatBot.guardrail()
ChatBot.guard_stats()
no_pii()
max_response_length()
max_input_length()
tone_check()
disclaimer_required()
must_cite_sources()
keyword_block()
Evaluation
eval.ChatBot
eval.CustomMetric
eval.EvalCase
eval.EvalScore
eval.EvalResult
eval.EvalResults
eval.BenchmarkResult
eval.EvalDimension
eval.eval_metric()
eval.list_custom_metrics()
eval.clear_custom_metrics()
eval.eval()
eval.eval_regression()
eval.eval_suite()
eval.eval_model_update()
eval.benchmark_persona()
eval.scorecard_table()
eval.sweep_table()
eval.artifact_correctness_scorer()
eval.HAS_PANDAS
eval.HAS_GREAT_TABLES
eval.DEFAULT_DIMENSIONS
eval.ScorerFn
eval_regression()
eval_model_update()
eval_suite()
EvalCase
EvalDimension
EvalResults
EvalResults.summary()
EvalResults.passed()
EvalResults.regressions()
EvalResults.scores_by_variant()
EvalResults.to_dataframe()
EvalResults.to_great_table()
EvalResults.to_scorecard()
EvalResult
EvalScore
scorecard_table()
sweep_table()
Model Profiles
ModelProfile
ModelProfile.key
ModelProfile.name
ModelProfile.supports()
CostTier
get_model_profile()
list_models()
register_model()
model_profiles_table()
Ollama Detection
OllamaStatus
detect_ollama()
list_ollama_models()
sync_ollama_models()
Context Window Management
ContextWindow
ContextWindow.fit_messages()
ContextWindow.fit_prompt()
ContextWindow.count_tokens()
ContextWindow.fits()
ContextWindow.overflow()
FitResult
PromptFitResult
FitStrategy
estimate_tokens()
ContextWindow Methods
ContextWindow.max_tokens
ContextWindow.input_budget
ContextWindow.reserve_output
ContextWindow.count_tokens()
ContextWindow.fits()
ContextWindow.overflow()
ContextWindow.fit_messages()
ContextWindow.fit_prompt()
Prompt Optimization
optimize_prompt()
OptimizationLevel
OptimizeResult
Hybrid Routing
route()
classify_complexity()
Router
Router.route()
RoutingStrategy
RoutingResult
TaskComplexity
Consensus Mode
consensus.ModelResponse
consensus.Disagreement
consensus.ConsensusResult
consensus.ConsensusStrategy
consensus.find_disagreements()
consensus.consensus()
find_disagreements()
ConsensusStrategy
ConsensusResult
ModelResponse
Disagreement
Cascade Consensus
cascade.Router
cascade.ModelResponse
cascade.CascadeRound
cascade.CascadeResult
cascade.ConsensusStrategy
cascade.RoutingStrategy
cascade.TaskComplexity
cascade.consensus()
cascade.estimate_confidence()
cascade.cascade()
cascade.Responder
estimate_confidence()
CascadeResult
CascadeRound
Memory Tiers
MemoryStore
MemoryStore.remember()
MemoryStore.recall()
MemoryStore.forget()
MemoryStore.search()
WorkingMemory
ShortTermMemory
LongTermMemory
MemoryEntry
MemoryTier
WorkingMemory Methods
WorkingMemory.set()
WorkingMemory.get()
WorkingMemory.has()
WorkingMemory.delete()
WorkingMemory.keys()
WorkingMemory.entries()
WorkingMemory.search()
WorkingMemory.clear()
WorkingMemory.__len__()
ShortTermMemory Methods
ShortTermMemory.max_entries
ShortTermMemory.default_ttl
ShortTermMemory.set()
ShortTermMemory.get()
ShortTermMemory.has()
ShortTermMemory.delete()
ShortTermMemory.keys()
ShortTermMemory.entries()
ShortTermMemory.search()
ShortTermMemory.clear()
ShortTermMemory.__len__()
LongTermMemory Methods
LongTermMemory.path
LongTermMemory.set()
LongTermMemory.get()
LongTermMemory.has()
LongTermMemory.delete()
LongTermMemory.keys()
LongTermMemory.entries()
LongTermMemory.search()
LongTermMemory.clear()
LongTermMemory.close()
LongTermMemory.__len__()
Agent
Agent
Agent.from_persona()
Agent.respond()
Agent.remember()
Agent.recall()
Agent.retention
Agent.conversation
Agent.reset_conversation()
Agent Methods
Agent.__post_init__()
Agent.from_persona()
Agent.respond()
Agent.remember()
Agent.recall()
Agent.retention
Agent.conversation
Agent.reset_conversation()
Human-in-the-Loop
human_review()
approve()
reject()
revise()
HumanReview
HumanReview.status
HumanReview.decision
HumanReview.is_pending
HumanReview.is_resolved
HumanReview.is_approved
HumanReview.final_content
ReviewDecision
ReviewStatus
ReviewQueue
HumanReview Methods
HumanReview.status
HumanReview.decision
HumanReview.is_pending
HumanReview.is_resolved
HumanReview.is_approved
HumanReview.final_content
ReviewQueue Methods
ReviewQueue.add()
ReviewQueue.pending()
ReviewQueue.resolved()
ReviewQueue.by_agent()
ReviewQueue.by_status()
ReviewQueue.all
ReviewQueue.__len__()
Skill System
SkillDefinition
create_skill()
get_skill()
list_skills()
register_skill()
load_skill()
skill_categories()
Structured Outputs
ExtractResult
extract()
schema_to_dict()
Persona Traits
TraitDefinition
apply_trait()
create_trait()
get_trait()
list_traits()
register_trait()
load_trait()
trait_categories()
MCP Bridge
MCPToolInfo
MCPBridgeServer
MCPBridgeServer.tool_names()
MCPBridgeServer.build()
MCPBridgeServer.run()
tools_to_mcp_server()
mcp_tool_to_talk_box()
list_mcp_tools()
discover_mcp_tools()
Knowledge Graph
KnowledgeGraph
KnowledgeGraph.add_node()
KnowledgeGraph.get_node()
KnowledgeGraph.delete_node()
KnowledgeGraph.list_nodes()
KnowledgeGraph.node_count()
KnowledgeGraph.add_edge()
KnowledgeGraph.get_edges()
KnowledgeGraph.delete_edge()
KnowledgeGraph.edge_count()
KnowledgeGraph.neighbors()
KnowledgeGraph.search()
KnowledgeGraph.stats()
KnowledgeGraph.health()
KnowledgeGraph.pending_questions()
KnowledgeGraph.answer_question()
KnowledgeGraph.dismiss_question()
KnowledgeGraph.question_stats()
KnowledgeGraph.clear()
KnowledgeGraph.close()
Node
Edge
NodeType
cosine_similarity()
KnowledgeGraph Methods
KnowledgeGraph.name
KnowledgeGraph.ontology
KnowledgeGraph.set_ontology()
KnowledgeGraph.path
KnowledgeGraph.add_node()
KnowledgeGraph.get_node()
KnowledgeGraph.delete_node()
KnowledgeGraph.list_nodes()
KnowledgeGraph.node_count()
KnowledgeGraph.add_edge()
KnowledgeGraph.get_edges()
KnowledgeGraph.delete_edge()
KnowledgeGraph.edge_count()
KnowledgeGraph.neighbors()
KnowledgeGraph.search()
KnowledgeGraph.extract_subgraph()
KnowledgeGraph.stats()
KnowledgeGraph.health()
KnowledgeGraph.pending_questions()
KnowledgeGraph.answer_question()
KnowledgeGraph.dismiss_question()
KnowledgeGraph.question_stats()
KnowledgeGraph.decision_trail()
KnowledgeGraph.revert_decision()
KnowledgeGraph.clear()
KnowledgeGraph.clear_layer()
KnowledgeGraph.export_base()
KnowledgeGraph.export_layers()
KnowledgeGraph.compose()
KnowledgeGraph.close()
KnowledgeGraph.__repr__()
KG Visualization
VisNode
VisEdge
VisGraph
VisGraph.node_count
VisGraph.edge_count
VisGraph.to_dict()
prepare_vis_graph()
to_mermaid()
to_html()
visualize()
Connectors
Document
Connector
Connector.scan()
MarkdownDir
MarkdownDir.scan()
DirectoryConnector
DirectoryConnector.scan()
AppleNotes
AppleNotes.scan()
SyncResult
SyncResult.total
connector()
sync()
Enrichment
ExtractedEntity
ExtractedRelationship
EnrichmentResult
EnrichmentResult.entity_names
EnrichmentResult.entity_count
EnrichmentResult.topic_count
EnrichmentResult.relationship_count
EnrichmentPipeline
EnrichmentPipeline.run()
PipelineResult
PipelineResult.total
EnrichmentConfig
regex_enricher()
Compliance Export
export_json()
export_html()
CLI
cli.main()
cli.info()
cli.personas()
cli.models()
cli.test_cmd()
Configuration
TalkBoxConfig
TalkBoxConfig.resolve()
TalkBoxConfig.to_dict()
TalkBoxConfig.validate_model()
TalkBoxConfig.is_cloud_model()
ProfileConfig
ProfileConfig.to_dict()
ResolvedConfig
TUIMode
load_config()
save_config()
load_profile()
save_profile()
list_profiles()
eval.list_custom_metrics()
Return all registered custom metrics.
Usage
Source
eval
.
list_custom_metrics
()