Using the Scorecard SDK

1
#@title 👉 API Keys
2
3
SCORECARD_API_KEY = "" #@param { type: "string" }

1
#@title Install dependencies
2
#@markdown In order to keep the notebook working for all future users, we pin the dependency versions.
3
4
!pip install scorecard-ai==0.1.12
5
!pip install openai==1.11.1

1
#@title Imports
2
3
from openai import OpenAI
4
from scorecard.client import Scorecard

1
#@title 1. Create a basic Testset with Custom Schema
2
#@markdown Here we'll create a basic Testset that gets stored in Scorecard.
3
4
from scorecard.types import CustomSchema, CustomVariable
5
import json
6
7
client = Scorecard(
8
    api_key=SCORECARD_API_KEY,
9
)
10
11
# Create a Testset
12
testset = client.testset.create(
13
    name="Story Opening Lines",
14
    description="Demo of a Testset created via Scorecard Python SDK",
15
    using_retrieval=False,
16
    custom_schema=CustomSchema(
17
        variables=[
18
            CustomVariable(
19
                name="inputs",
20
                description="Custom Parameters for this Testset.",
21
                role="input",
22
                data_type="json_object",
23
            ),
24
        ]
25
    ),
26
)
27
28
# Add three Testcases
29
client.testcase.create(
30
    testset_id=testset.id,
31
    user_query="query",
32
    custom_inputs={
33
        "inputs": json.dumps(
34
            {
35
                "key": "value",
36
                "link": "https://www.google.com",
37
                "number": 1,
38
                "nested": {"key": "value"},
39
            }
40
        ),
41
    },
42
)
43
44
print("Visit the Scorecard UI to view your Testset:")
45
print(f"https://app.getscorecard.ai/view-dataset/{testset.id}")