Ragas

Common errors

data_samples = {
    'user_input': ['What is RAG?'],
    'response': ['RAG is Retrieval-Augmented Generation.'],
    'retrieved_contexts': [['RAG combines retrieval and generation models.']],
    'ground_truth': ['RAG is a technique to improve LLM outputs.']
}
# Then proceed to create EvaluationDataset
# dataset = EvaluationDataset(data_samples)

from ragas.metrics import Faithfulness
from ragas.llms import RagasLLM
from langchain_openai import ChatOpenAI

openai_model = ChatOpenAI(model="gpt-3.5-turbo")
ragas_llm = RagasLLM(llm=openai_model)

# Initialize the metric as a class instance with the LLM
faithfulness_metric = Faithfulness(llm=ragas_llm)

# Then pass the instance to evaluate
# result = evaluate(dataset, metrics=[faithfulness_metric])

from ragas.metrics import ResponseRelevancy
# Then instantiate it with an LLM as shown above:
# response_relevancy_metric = ResponseRelevancy(llm=ragas_llm)

pip install ragas
# If you need specific integrations like OpenAI:
pip install ragas[openai]

Warnings

Install compatibility stale last tested: 2026-05-12

Imports

• evaluate (v0.2+ style)
  wrong

  # v0.1 style — deprecated, removed in v1.0
  from datasets import Dataset
  from ragas import evaluate
  from ragas.metrics import faithfulness, answer_relevancy
  
  data = {
      'question': ['When was the first Super Bowl?'],
      'answer': ['Jan 15, 1967'],
      'contexts': [['The game was played on January 15, 1967.']]
  }
  ds = Dataset.from_dict(data)
  result = evaluate(ds, metrics=[faithfulness, answer_relevancy])

  correct

  from ragas import EvaluationDataset, SingleTurnSample, evaluate
  from ragas.metrics import Faithfulness, ResponseRelevancy
  from ragas.llms import LangchainLLMWrapper
  from langchain_openai import ChatOpenAI
  
  llm = LangchainLLMWrapper(ChatOpenAI(model='gpt-4o-mini'))
  
  samples = [
      SingleTurnSample(
          user_input='When was the first Super Bowl?',
          response='The first Super Bowl was held on Jan 15, 1967.',
          retrieved_contexts=[
              'The First AFL-NFL World Championship Game was played on January 15, 1967.'
          ]
      )
  ]
  
  dataset = EvaluationDataset(samples=samples)
  
  result = evaluate(
      dataset,
      metrics=[
          Faithfulness(llm=llm),
          ResponseRelevancy(llm=llm)
      ]
  )
  print(result)

  v0.1 field names: question/answer/contexts. v0.2+ field names: user_input/response/retrieved_contexts. answer_relevancy renamed to ResponseRelevancy. Metrics are now class instances with llm= argument.

• single metric scoring
  wrong

  from ragas.metrics import faithfulness  # lowercase — deprecated singleton
  from ragas import evaluate
  # Using deprecated singleton instance

  correct

  from ragas import SingleTurnSample
  from ragas.metrics import Faithfulness
  from ragas.llms import LangchainLLMWrapper
  from langchain_openai import ChatOpenAI
  import asyncio
  
  llm = LangchainLLMWrapper(ChatOpenAI(model='gpt-4o-mini'))
  scorer = Faithfulness(llm=llm)
  
  sample = SingleTurnSample(
      user_input='What year was Python created?',
      response='Python was created in 1991.',
      retrieved_contexts=['Python was first released in 1991 by Guido van Rossum.']
  )
  
  # Async score
  score = asyncio.run(scorer.single_turn_ascore(sample))
  print(score)

  Import Faithfulness (class) not faithfulness (deprecated singleton). Initialize with llm= argument. Use single_turn_ascore() for single samples.

Quickstart stale last tested: 2026-04-23

Ragas v0.2+ RAG evaluation with EvaluationDataset and class-based metrics.

# pip install ragas langchain-openai
from ragas import EvaluationDataset, SingleTurnSample, evaluate
from ragas.metrics import Faithfulness, ResponseRelevancy, LLMContextRecall
from ragas.llms import LangchainLLMWrapper
from langchain_openai import ChatOpenAI
import os

os.environ['OPENAI_API_KEY'] = 'your-key'

llm = LangchainLLMWrapper(ChatOpenAI(model='gpt-4o-mini'))

samples = [
    SingleTurnSample(
        user_input='What is the capital of France?',
        response='The capital of France is Paris.',
        retrieved_contexts=['Paris is the capital and most populous city of France.'],
        reference='Paris'  # ground truth — needed for recall
    )
]

dataset = EvaluationDataset(samples=samples)

result = evaluate(
    dataset,
    metrics=[
        Faithfulness(llm=llm),
        ResponseRelevancy(llm=llm),
        LLMContextRecall(llm=llm)
    ]
)
print(result)
# {'faithfulness': 1.0, 'response_relevancy': 0.97, 'context_recall': 1.0}