Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[GSK-3940] Make RagasMetric compatible with Ragas v0.2 #2073

Merged
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 27 additions & 7 deletions giskard/rag/metrics/ragas_metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

import logging

import ragas

from ...llm.client import ChatMessage, LLMClient, get_default_client
from ...llm.embeddings import BaseEmbedding, get_default_embedding
from ..base import AgentAnswer
Expand Down Expand Up @@ -104,20 +106,38 @@ def __call__(self, question_sample: dict, answer: AgentAnswer) -> dict:

self.metric.init(run_config)
if self.requires_context and answer.documents is None:
logger.warn(
logger.warning(
f"No retrieved documents are passed to the evaluation function, computation of {self.name} cannot be done without it."
"Make sure you pass 'retrieved_documents' to the evaluate function or that the 'answer_fn' return documents alongside the answer."
)
return {self.name: 0}

ragas_sample = {
"question": question_sample["question"],
"answer": answer.message,
"contexts": answer.documents,
"ground_truth": question_sample["reference_answer"],
}
ragas_sample = self.prepare_ragas_sample(question_sample, answer)

return {self.name: self.metric.score(ragas_sample)}

@staticmethod
def prepare_ragas_sample(question_sample: dict, answer: AgentAnswer) -> dict:
if ragas.__version__.startswith("0.1"):
logger.warning(
f"{DeprecationWarning.__name__}: You are using an older version (v0.1) of `ragas` package. "
"Support for v0.1 is deprecated and may be removed in future versions. "
"Please consider updating `ragas` to a later version."
)
return {
"question": question_sample["question"],
"answer": answer.message,
"contexts": answer.documents,
"ground_truth": question_sample["reference_answer"],
}
else:
henchaves marked this conversation as resolved.
Show resolved Hide resolved
return {
"user_input": question_sample["question"],
"response": answer.message,
"retrieved_contexts": answer.documents,
"reference": question_sample["reference_answer"],
}


ragas_context_precision = RagasMetric(name="RAGAS Context Precision", metric=context_precision, requires_context=True)
ragas_faithfulness = RagasMetric(name="RAGAS Faithfulness", metric=faithfulness, requires_context=True)
Expand Down
Loading