From d38d0cccb8c0564e955855f70f5caddc626598d7 Mon Sep 17 00:00:00 2001 From: Henrique Chaves Date: Thu, 14 Nov 2024 16:34:04 -0300 Subject: [PATCH 1/3] Make RagasMetric compatible with Ragas v0.2 --- giskard/rag/metrics/ragas_metrics.py | 34 +++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/giskard/rag/metrics/ragas_metrics.py b/giskard/rag/metrics/ragas_metrics.py index 211e8eb4bf..b86fb17f2b 100644 --- a/giskard/rag/metrics/ragas_metrics.py +++ b/giskard/rag/metrics/ragas_metrics.py @@ -1,6 +1,9 @@ from typing import Optional, Sequence import logging +import warnings + +import ragas from ...llm.client import ChatMessage, LLMClient, get_default_client from ...llm.embeddings import BaseEmbedding, get_default_embedding @@ -110,14 +113,33 @@ def __call__(self, question_sample: dict, answer: AgentAnswer) -> dict: ) return {self.name: 0} - ragas_sample = { - "question": question_sample["question"], - "answer": answer.message, - "contexts": answer.documents, - "ground_truth": question_sample["reference_answer"], - } + ragas_sample = self.prepare_ragas_sample(question_sample, answer) + return {self.name: self.metric.score(ragas_sample)} + @classmethod + def prepare_ragas_sample(self, question_sample: dict, answer: AgentAnswer) -> dict: + if ragas.__version__.startswith("0.1"): + warnings.warn( + "You are using an old version of RAGAS." + "Support for RAGAS v0.1 is deprecated and may be removed in future versions." + "Please consider updating to the latest version.", + DeprecationWarning, + ) + return { + "question": question_sample["question"], + "answer": answer.message, + "contexts": answer.documents, + "ground_truth": question_sample["reference_answer"], + } + else: + return { + "user_input": question_sample["question"], + "response": answer.message, + "retrieved_contexts": answer.documents, + "reference": question_sample["reference_answer"], + } + ragas_context_precision = RagasMetric(name="RAGAS Context Precision", metric=context_precision, requires_context=True) ragas_faithfulness = RagasMetric(name="RAGAS Faithfulness", metric=faithfulness, requires_context=True) From e8a3522a3e77ea91746b21625488a05d18b99a7f Mon Sep 17 00:00:00 2001 From: Henrique Chaves Date: Thu, 14 Nov 2024 18:06:42 -0300 Subject: [PATCH 2/3] Replace warnings with logger --- giskard/rag/metrics/ragas_metrics.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/giskard/rag/metrics/ragas_metrics.py b/giskard/rag/metrics/ragas_metrics.py index b86fb17f2b..521886bde5 100644 --- a/giskard/rag/metrics/ragas_metrics.py +++ b/giskard/rag/metrics/ragas_metrics.py @@ -1,7 +1,6 @@ from typing import Optional, Sequence import logging -import warnings import ragas @@ -107,7 +106,7 @@ def __call__(self, question_sample: dict, answer: AgentAnswer) -> dict: self.metric.init(run_config) if self.requires_context and answer.documents is None: - logger.warn( + logger.warning( f"No retrieved documents are passed to the evaluation function, computation of {self.name} cannot be done without it." "Make sure you pass 'retrieved_documents' to the evaluate function or that the 'answer_fn' return documents alongside the answer." ) @@ -117,14 +116,13 @@ def __call__(self, question_sample: dict, answer: AgentAnswer) -> dict: return {self.name: self.metric.score(ragas_sample)} - @classmethod - def prepare_ragas_sample(self, question_sample: dict, answer: AgentAnswer) -> dict: + @staticmethod + def prepare_ragas_sample(question_sample: dict, answer: AgentAnswer) -> dict: if ragas.__version__.startswith("0.1"): - warnings.warn( - "You are using an old version of RAGAS." - "Support for RAGAS v0.1 is deprecated and may be removed in future versions." - "Please consider updating to the latest version.", - DeprecationWarning, + logger.warning( + f"{DeprecationWarning.__name__}: You are using an older version (v0.1) of `ragas` package. " + "Support for v0.1 is deprecated and may be removed in future versions. " + "Please consider updating `ragas` to a later version." ) return { "question": question_sample["question"], From af63db81fd7f86985a6dc5c52b6cc7c9ec6127ad Mon Sep 17 00:00:00 2001 From: Henrique Chaves Date: Fri, 15 Nov 2024 10:19:47 -0300 Subject: [PATCH 3/3] Update prepare_ragas_sample --- giskard/rag/metrics/ragas_metrics.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/giskard/rag/metrics/ragas_metrics.py b/giskard/rag/metrics/ragas_metrics.py index 521886bde5..4e5a56e09a 100644 --- a/giskard/rag/metrics/ragas_metrics.py +++ b/giskard/rag/metrics/ragas_metrics.py @@ -130,13 +130,13 @@ def prepare_ragas_sample(question_sample: dict, answer: AgentAnswer) -> dict: "contexts": answer.documents, "ground_truth": question_sample["reference_answer"], } - else: - return { - "user_input": question_sample["question"], - "response": answer.message, - "retrieved_contexts": answer.documents, - "reference": question_sample["reference_answer"], - } + + return { + "user_input": question_sample["question"], + "response": answer.message, + "retrieved_contexts": answer.documents, + "reference": question_sample["reference_answer"], + } ragas_context_precision = RagasMetric(name="RAGAS Context Precision", metric=context_precision, requires_context=True)