lightbench.evaluators

lightbench.evaluators#

The evaluators package contains various evaluators for different types of data and tasks, such as code evaluation and text evaluation.

Evaluators#

class lightbench.evaluators.evaluator.Evaluator(verbose=False)#

Bases: ABC

abstractmethod cleanup()#
abstractmethod print_summary()#
abstractmethod run(prompts)#
class lightbench.evaluators.code_evaluator.CodeEvaluator(model_loader: LLMServiceLoader, num_test_limit: int | None = None, few_shot=False, verbose=False)#

Bases: Evaluator

cleanup()#
print_summary()#
run()#
class lightbench.evaluators.text_evaluator.TextEvaluator(model_loader: LLMServiceLoader, judge: LLMJudge, num_test_limit: int | None = None, verbose: bool = False)#

Bases: Evaluator

cleanup()#
print_summary()#
run()#
Contents