​

SDK Migration Guide

# Setup Scorecard
from scorecard.client import Scorecard
client = Scorecard(api_key="YOUR_API_KEY")

# Setup Scorecard
from scorecard_ai import Scorecard
client = Scorecard(api_key="YOUR_API_KEY")

# Run and evaluate
run = client.run_tests(
  input_testset_id=123
  scoring_config_id=789,
  model_invocation=lambda prompt: call_model(prompt),
)

# Run and evaluate
from scorecard_ai.lib import run_and_evaluate
run = run_and_evaluate(
    client=client,
    project_id="123",
    testset_id="456",
    metric_ids=["789"],
    system=lambda system_input: call_model(system_input["prompt"])
)

testset = client.testset.create(
    name="Testset name",
    description="Optional Testset description",
    using_retrieval=False,
)

testset = client.testsets.create(
    project_id="1234",
    name="Testset name",
    description="Required Testset description",
    field_mapping={
        # Inputs represent the input to the AI system.
        "inputs": ["userQuery"],
        # Labels represent the expected output of the AI system.
        "labels": ["ideal"],
        # Metadata fields are used for grouping Testcases, but not seen by the AI system.
        "metadata": [],
    },
    json_schema={
        "type": "object",
        "properties": {
            # The original message.
            "userQuery": {"type": "string"},
            # The ideal model response
            "ideal": {"type": "string"},
        },
        "required": ["userQuery", "ideal"],
    },
)

client.testcase.create(
  testset_id=testset.id,
  user_query="What's 2+2 in English?",
  ideal="Four",
)

client.testcases.create(
  testset_id=testset.id,
  items=[
      {
          "json_data": {
              "userQuery": "What's 2+2 in English?",
              "ideal": "Four",
          },
      },
  ]
)