feat!: version 13 - dataset evaluators

.github/workflows/playwright.yaml

@@ -4,12 +4,16 @@ permissions:
 
 on:
   push:
-    branches: [main, feat/annotations]
+    branches:
+      - main
+      - version-13
     paths:
       - "src/**"
       - "app/**"
   pull_request:
-    branches: [main, feat/annotations]
+    branches:
+      - main
+      - version-13
     paths:
       - "src/**"
       - "app/**"

.github/workflows/python-CI.yml

@@ -6,7 +6,7 @@ on:
   push:
     branches:
       - main
-      - feat/version-12
+      - version-13
   pull_request:
     paths:
       - "**/*.py"

.github/workflows/typescript-CI.yml

@@ -6,7 +6,7 @@ on:
   push:
     branches:
       - main
-      - feat/version-12
+      - version-13
   pull_request:
     paths:
       - "app/**"

.github/workflows/typescript-packages-CI.yml

@@ -6,7 +6,7 @@ on:
   push:
     branches:
       - main
-      - feat/version-12
+      - version-13
   pull_request:
     paths:
       - "js/**"

app/schema.graphql

@@ -147,6 +147,12 @@ interface ApiKey {
   expiresAt: DateTime
 }
 
+input AssignEvaluatorToDatasetInput {
+  datasetId: ID!
+  evaluatorId: ID!
+  inputMapping: EvaluatorInputMappingInput = null
+}
+
 enum AuthMethod {
   LOCAL
   OAUTH2
@@ -174,6 +180,21 @@ type BoundedFloatInvocationParameter implements InvocationParameterBase {
   maxValue: Float!
 }
 
+type BuiltInEvaluator implements Evaluator & Node {
+  """The Globally Unique ID of this object"""
+  id: ID!
+  name: Identifier!
+  description: String
+  metadata: JSON!
+  kind: EvaluatorKind!
+  createdAt: DateTime!
+  updatedAt: DateTime!
+  inputSchema: JSON
+  isAssignedToDataset(datasetId: ID = null): Boolean!
+  datasetInputMapping(datasetId: ID = null): EvaluatorInputMapping
+  user: User
+}
+
 enum CanonicalParameterName {
   TEMPERATURE
   MAX_COMPLETION_TOKENS
@@ -227,6 +248,7 @@ input ChatCompletionInput {
   template: PromptTemplateOptions
   promptName: Identifier = null
   repetitions: Int!
+  evaluators: [PlaygroundEvaluatorInput!]! = []
 }
 
 input ChatCompletionMessageInput {
@@ -270,6 +292,7 @@ input ChatCompletionOverDatasetInput {
   experimentDescription: String = null
   experimentMetadata: JSON = {}
   promptName: Identifier = null
+  evaluators: [PlaygroundEvaluatorInput!]! = []
 }
 
 type ChatCompletionOverDatasetMutationExamplePayload {
@@ -292,6 +315,7 @@ type ChatCompletionRepetition {
   toolCalls: [ChatCompletionToolCall!]!
   span: Span
   errorMessage: String
+  evaluations: [ExperimentRunAnnotation!]!
 }
 
 type ChatCompletionSubscriptionError implements ChatCompletionSubscriptionPayload {
@@ -377,6 +401,26 @@ input ClusterInput {
   id: ID
 }
 
+type CodeEvaluator implements Evaluator & Node {
+  """The Globally Unique ID of this object"""
+  id: ID!
+  name: Identifier!
+  description: String
+  metadata: JSON!
+  kind: EvaluatorKind!
+  createdAt: DateTime!
+  updatedAt: DateTime!
+  inputSchema: JSON
+  isAssignedToDataset(datasetId: ID = null): Boolean!
+  datasetInputMapping(datasetId: ID = null): EvaluatorInputMapping
+  user: User
+}
+
+type CodeEvaluatorMutationPayload {
+  evaluator: CodeEvaluator!
+  query: Query!
+}
+
 union ContentPart = TextContentPart | ToolCallContentPart | ToolResultContentPart
 
 input ContentPartInput @oneOf {
@@ -440,6 +484,12 @@ input CreateChatPromptVersionInput {
   tags: [SetPromptVersionTagInput!] = null
 }
 
+input CreateCodeEvaluatorInput {
+  datasetId: ID
+  name: Identifier!
+  description: String
+}
+
 input CreateDatasetInput {
   name: String!
   description: String
@@ -473,6 +523,14 @@ input CreateDatasetSplitWithExamplesInput {
   exampleIds: [ID!]!
 }
 
+input CreateLLMEvaluatorInput {
+  datasetId: ID
+  name: Identifier!
+  description: String
+  promptVersion: ChatPromptVersionInput!
+  outputConfig: CategoricalAnnotationConfigInput!
+}
+
 input CreateModelMutationInput {
   name: String!
   provider: String = null
@@ -624,6 +682,7 @@ type Dataset implements Node {
   experiments(first: Int = 50, last: Int, after: String, before: String, filterCondition: String, filterIds: [ID!]): ExperimentConnection!
   experimentAnnotationSummaries: [DatasetExperimentAnnotationSummary!]!
   labels: [DatasetLabel!]!
+  evaluators(first: Int = 50, last: Int, after: String, before: String, sort: EvaluatorSort, filter: EvaluatorFilter): EvaluatorConnection!
   lastUpdatedAt: DateTime
 }
 
@@ -892,6 +951,15 @@ type DeleteDatasetSplitsMutationPayload {
   query: Query!
 }
 
+input DeleteEvaluatorsInput {
+  evaluatorIds: [ID!]!
+}
+
+type DeleteEvaluatorsPayload {
+  evaluatorIds: [ID!]!
+  query: Query!
+}
+
 input DeleteExperimentsInput {
   experimentIds: [ID!]!
 }
@@ -1231,6 +1299,88 @@ input EvalResultKey {
   attr: EvalAttr!
 }
 
+type EvaluationChunk implements ChatCompletionSubscriptionPayload {
+  datasetExampleId: ID
+  repetitionNumber: Int
+  experimentRunEvaluation: ExperimentRunAnnotation
+  spanEvaluation: SpanAnnotation
+}
+
+interface Evaluator implements Node {
+  """The Globally Unique ID of this object"""
+  id: ID!
+  name: Identifier!
+  description: String
+  metadata: JSON!
+  kind: EvaluatorKind!
+  createdAt: DateTime!
+  updatedAt: DateTime!
+  inputSchema: JSON
+  isAssignedToDataset(datasetId: ID = null): Boolean!
+  datasetInputMapping(datasetId: ID = null): EvaluatorInputMapping
+}
+
+enum EvaluatorColumn {
+  name
+  kind
+  createdAt
+  updatedAt
+}
+
+"""A connection to a list of items."""
+type EvaluatorConnection {
+  """Pagination data for this connection"""
+  pageInfo: PageInfo!
+
+  """Contains the nodes in this connection"""
+  edges: [EvaluatorEdge!]!
+}
+
+"""An edge in a connection."""
+type EvaluatorEdge {
+  """A cursor for use in pagination"""
+  cursor: String!
+
+  """The item at the end of the edge"""
+  node: Evaluator!
+}
+
+"""The filter key and value for evaluator connections"""
+input EvaluatorFilter {
+  col: EvaluatorFilterColumn!
+  value: String!
+}
+
+enum EvaluatorFilterColumn {
+  name
+}
+
+type EvaluatorInputMapping {
+  literalMapping: JSON!
+  pathMapping: JSON!
+}
+
+input EvaluatorInputMappingInput {
+  literalMapping: JSON! = {}
+  pathMapping: JSON! = {}
+}
+
+enum EvaluatorKind {
+  LLM
+  CODE
+}
+
+type EvaluatorMutationPayload {
+  evaluator: Evaluator!
+  query: Query!
+}
+
+"""The sort key and direction for evaluator connections"""
+input EvaluatorSort {
+  col: EvaluatorColumn!
+  dir: SortDir!
+}
+
 type Event {
   id: ID!
   eventMetadata: EventMetadata!
@@ -1763,6 +1913,30 @@ type JSONInvocationParameter implements InvocationParameterBase {
   defaultValue: JSON
 }
 
+type LLMEvaluator implements Evaluator & Node {
+  """The Globally Unique ID of this object"""
+  id: ID!
+  name: Identifier!
+  description: String
+  metadata: JSON!
+  kind: EvaluatorKind!
+  createdAt: DateTime!
+  updatedAt: DateTime!
+  inputSchema: JSON
+  isAssignedToDataset(datasetId: ID = null): Boolean!
+  datasetInputMapping(datasetId: ID = null): EvaluatorInputMapping
+  outputConfig: CategoricalAnnotationConfig!
+  prompt: Prompt!
+  promptVersionTag: PromptVersionTag
+  user: User
+  promptVersion: PromptVersion!
+}
+
+type LLMEvaluatorMutationPayload {
+  evaluator: LLMEvaluator!
+  query: Query!
+}
+
 type LabelFraction {
   label: String!
   fraction: Float!
@@ -1825,6 +1999,12 @@ type Mutation {
   deleteDatasetSplits(input: DeleteDatasetSplitInput!): DeleteDatasetSplitsMutationPayload!
   setDatasetExampleSplits(input: SetDatasetExampleSplitsInput!): SetDatasetExampleSplitsMutationPayload!
   createDatasetSplitWithExamples(input: CreateDatasetSplitWithExamplesInput!): DatasetSplitMutationPayloadWithExamples!
+  createCodeEvaluator(input: CreateCodeEvaluatorInput!): CodeEvaluatorMutationPayload!
+  createLlmEvaluator(input: CreateLLMEvaluatorInput!): LLMEvaluatorMutationPayload!
+  updateLlmEvaluator(input: UpdateLLMEvaluatorInput!): LLMEvaluatorMutationPayload!
+  deleteEvaluators(input: DeleteEvaluatorsInput!): DeleteEvaluatorsPayload!
+  assignEvaluatorToDataset(input: AssignEvaluatorToDatasetInput!): EvaluatorMutationPayload!
+  unassignEvaluatorFromDataset(input: UnassignEvaluatorFromDatasetInput!): EvaluatorMutationPayload!
   deleteExperiments(input: DeleteExperimentsInput!): ExperimentMutationPayload!
 
   """
@@ -1990,6 +2170,11 @@ type PerformanceTimeSeries implements TimeSeries {
   data: [TimeSeriesDataPoint!]!
 }
 
+input PlaygroundEvaluatorInput {
+  id: ID!
+  inputMapping: EvaluatorInputMappingInput! = {}
+}
+
 type PlaygroundModel implements ModelInterface {
   name: String!
   providerKey: GenerativeProviderKey!
@@ -2497,6 +2682,8 @@ type Query {
   promptLabels(first: Int = 50, last: Int, after: String, before: String): PromptLabelConnection!
   datasetLabels(first: Int = 50, last: Int, after: String, before: String): DatasetLabelConnection!
   datasetSplits(first: Int = 50, last: Int, after: String, before: String): DatasetSplitConnection!
+  builtInEvaluators: [BuiltInEvaluator!]!
+  evaluators(first: Int = 50, last: Int, after: String, before: String, sort: EvaluatorSort, filter: EvaluatorFilter): EvaluatorConnection!
   annotationConfigs(first: Int = 50, last: Int = null, after: String = null, before: String = null): AnnotationConfigConnection!
   clusters(clusters: [ClusterInput!]!): [Cluster!]!
   hdbscanClustering(
@@ -3263,6 +3450,11 @@ type UMAPPoints {
   contextRetrievals: [Retrieval!]!
 }
 
+input UnassignEvaluatorFromDatasetInput {
+  datasetId: ID!
+  evaluatorId: ID!
+}
+
 input UnsetPromptLabelsInput {
   promptId: ID!
   promptLabelIds: [ID!]!
@@ -3289,6 +3481,14 @@ input UpdateAnnotationInput {
   source: AnnotationSource! = APP
 }
 
+input UpdateLLMEvaluatorInput {
+  evaluatorId: ID!
+  name: Identifier!
+  description: String = null
+  promptVersion: ChatPromptVersionInput!
+  outputConfig: CategoricalAnnotationConfigInput!
+}
+
 input UpdateModelMutationInput {
   id: ID!
   name: String!