diff --git a/packages/uipath/samples/calculator/evaluations/evaluators/llm-judge-semantic-similarity.json b/packages/uipath/samples/calculator/evaluations/evaluators/llm-judge-semantic-similarity.json index 900d85c67..1c7ada6c1 100644 --- a/packages/uipath/samples/calculator/evaluations/evaluators/llm-judge-semantic-similarity.json +++ b/packages/uipath/samples/calculator/evaluations/evaluators/llm-judge-semantic-similarity.json @@ -6,7 +6,7 @@ "evaluatorConfig": { "name": "LLMJudgeOutputEvaluator", "targetOutputKey": "*", - "model": "gpt-4.1-2025-04-14", + "model": "anthropic.claude-sonnet-4-5-20250929-v1:0", "prompt": "Compare the following outputs and evaluate their semantic similarity.\n\nActual Output: {{ActualOutput}}\nExpected Output: {{ExpectedOutput}}\n\nProvide a score from 0-100 where 100 means semantically identical and 0 means completely different.", "temperature": 0.0, "defaultEvaluationCriteria": { diff --git a/packages/uipath/testcases/calculator-evals/src/assert.py b/packages/uipath/testcases/calculator-evals/src/assert.py index a3ae42088..784c07ee5 100644 --- a/packages/uipath/testcases/calculator-evals/src/assert.py +++ b/packages/uipath/testcases/calculator-evals/src/assert.py @@ -74,8 +74,9 @@ def main() -> None: # Allow 0 scores for Faithfulness and ContextPrecision evaluators # since calculator doesn't use RAG/context grounding + normalized_evaluator_name = evaluator_name.replace(" ", "") is_rag_evaluator = any( - x in evaluator_name + x in normalized_evaluator_name for x in ["Faithfulness", "ContextPrecision"] )