Metrics API

For a brief overview of GraphQL itself, please consult our introduction.

Average Metric Value

Metrics are associated with models; therefore, we utilize the model node to query for the average metric value.

Built-in Metrics

AveragePerformanceQuery(
  $dataset: ModelDatasetInput, 
  $performanceMetric: PerformanceMetric!, 
  $predictionValueClass: String, 
  $timeZone: String!, 
  $id: ID!) {
  node(id: $id) {
    ... on Model {
      averagePerformanceMetric(
        performanceMetric: $performanceMetric
        positiveClass: $predictionValueClass
        timeZone: $timeZone
        dataset: $dataset
      )
    }
  }
}

Variables

{
  "dataset": {
    "startTime": "2024-06-25T06:00:00.000Z", #target start time
    "endTime": "2024-07-26T05:59:59.999Z", #target end time
    "externalModelVersionIds": [],
    "externalBatchIds": [],
    "environmentName": "production",
    "filters": []
  },
  "performanceMetric": "TargetMetric", 
  "predictionValueClass": "positiveClassValue",
  "timeZone": "time/zone", #The timezone of the request, used to determine the localized offset from UTC.
  "id": "model_id"
}

Custom Metrics

query AveragePerformanceQuery(
 $dataset: ModelDatasetInput, 
 $customMetricConfig: String, 
  $timeZone: String!, 
  $id: ID!) {
  node(id: $id) {
    __typename
    ... on Model {
      averagePerformanceMetric(
        performanceMetric: udf
        timeZone: $timeZone
        dataset: $dataset
        customMetricConfig: $customMetricConfig
      )
    }
  }
}

Variables

{
  "dataset": {
    "startTime": "2024-06-25T06:00:00.000Z", #target start time
    "endTime": "2024-07-26T05:59:59.999Z", #target end time
    "externalModelVersionIds": [],
    "externalBatchIds": [],
    "environmentName": "production",
    "filters": []
  },
  "customMetricConfig": "select count(*) from model", #udf in SQL syntax
  "timeZone": "time/zone", #The timezone of the request, used to determine the localized offset from UTC.
  "id": "model_id"
}

Metric Over Time

To analyze metric values over time or plot reconstruction, use the following query

Built-in Metrics

query AveragePerformanceQuery(
  $performanceMetric: PerformanceMetric!
  $positiveClass:String!
  $startTime: DateTime!
  $endTime: DateTime!
  $timeZone: String!, 
  $id: ID!
  $environmentName: ModelEnvironmentName!
  $timeSeriesDataGranularity:DataGranularity!
) {
  node(id: $id) {
    __typename
    ... on Model {
      performanceMetricOverTime(
        performanceMetric: $performanceMetric
        timeZone: $timeZone,
        startTime: $startTime,
        endTime: $endTime,
        externalModelVersionIds: [],
        externalBatchIds: [],
        environmentName: $environmentName,
        timeSeriesDataGranularity: $timeSeriesDataGranularity
        filters: []
        positiveClass: $positiveClass
      ){
        dataPoints{
          x
          y
        }
      }
    }
    id
  }
}

Variables

{
  "performanceMetric": "metric",
  "positiveClass": "positive_class_value",
  "timeZone": "time/zome", #The timezone of the request, used to determine the localized offset from UTC.
  "id": "model_id",
  "startTime": "2024-06-25T06:00:00.000Z",
  "endTime": "2024-07-26T05:59:59.999Z",
  "environmentName": "production",
  "timeSeriesDataGranularity": "month", #desired granularity 
}

Custom Metrics

query AveragePerformanceQuery(
  $startTime: DateTime!
  $endTime: DateTime!
  $timeZone: String!, 
  $id: ID!
  $environmentName: ModelEnvironmentName!
  $timeSeriesDataGranularity:DataGranularity!
  $customMetricConfig:String
) {
  node(id: $id) {
    __typename
    ... on Model {
      performanceMetricOverTime(
        performanceMetric: udf
        timeZone: $timeZone,
        startTime: $startTime,
        endTime: $endTime,
        externalModelVersionIds: [],
        externalBatchIds: [],
        environmentName: $environmentName,
        timeSeriesDataGranularity: $timeSeriesDataGranularity
        filters: []
        customMetricConfig: $customMetricConfig
      ){
        dataPoints{
          x
          y
        }
      }
    }
    id
  }
}

variables

{
  "timeZone": "time/zone",
  "id": "model_id",
  "startTime": "2024-06-25T06:00:00.000Z", #target start time
  "endTime": "2024-07-26T05:59:59.999Z", #target end time
  "environmentName": "production", #The timezone of the request, used to determine the localized offset from UTC.
  "timeSeriesDataGranularity": "month", #target granularity for points
  "customMetricConfig": "select count(*) from model" #udf in SQL syntax
}

Drift Over Time

This endpoint allows you to query drift data points for a specific model within a given time interval. You can use it to monitor feature drift using metrics like PSI (Population Stability Index), KL (Kullback-Leibler divergence), or JS (Jensen-Shannon divergence). Your model baseline is defaulted as the baseline in this query but you can optionally specify a referenceDataset.

Query with Default Baseline

query GetDriftOverTime {
  node(id: "MODEL ID") {
    ... on Model {
      dimension(
        startTime: "2024-08-20T00:00:00Z" 
        endTime: "2024-08-30T00:00:00Z" 
        category: featureLabel 
        name: "annual_income"
        environmentName: production
      ) {
        name
        driftHistory(
          startTime: "2024-08-20T00:00:00Z"
          endTime: "2024-08-30T00:00:00Z"
          driftMetric: psi 
          timeSeriesDataGranularity: day 
          timeZone: "UTC"
        ) {
          ... on TimeSeriesWithThresholdDataType {
            dataPoints {
              x
              y
            }
          }
        }
      }
    }
  }
}query

Query with Custom Baseline

query GetDriftWithCustomBaseline {
  node(id: "model_id") {
    ... on Model {
      dimension(
        startTime: "2024-08-20T00:00:00Z"
        endTime: "2024-08-30T00:00:00Z"
        category: featureLabel
        name: "state" 
        environmentName: production
      ) {
        name
        driftHistory(
          referenceDataset:{
            startTime: "2024-08-01T00:00:00Z"
            endTime: "2024-08-15T00:00:00Z"
            environmentName:production 
            externalBatchIds: []
            externalModelVersionIds:[]
            filters:[]
          }
        
          startTime: "2024-08-20T00:00:00Z"
          endTime: "2024-08-30T00:00:00Z"
          driftMetric: psi
          timeSeriesDataGranularity: hour
          timeZone: "UTC"
        ) {
          ... on TimeSeriesWithThresholdDataType {
            dataPoints {
              x
              y
            }
          }
        }
      }
    }
  }
}

Parameters

• "MODEL ID": The unique identifier for your model.

• dimension: Specifies the feature or dimension you want to analyze.

  • startTime: The start of the interval for querying data (in UTC).

  • endTime: The end of the interval for querying data (in UTC).

  • category: The type of data you are querying. Options include:

    • featureLabel

    • prediction

    • actuals

    • actualScore

    • actualClass

    • predictionClass

    • predictionScore

    • tag

  • name: The specific feature or dimension name you want to query.

  • environmentName: The environment for which the data is being queried, such as 'production'.

• driftHistory: Retrieves the drift data over the specified time period.

  • driftMetric: The metric used to measure drift. Options include:

    • psi (Population Stability Index)

    • kl (Kullback-Leibler divergence)

    • js (Jensen-Shannon divergence)

  • timeSeriesDataGranularity: The granularity of the data points in the time series. Options include:

    • day

    • hour

    • week

    • month

  • timeZone: The time zone for the data points, e.g., "UTC".

Global Feature Importance

This endpoint allows you to query the global feature importance for all your feautres.

query GloablShapQuery($modelId: ID!,
  $startTimeA: DateTime!,
  $endTimeA: DateTime!,
  $globalFilters: [FilterItemInputType!]!, 
  $externalModelVersionIdsGlobal: [String!]!) {
  node(id: $modelId) {
    id
    __typename
    ... on Model {
      modelExplainability(
        startTime: $startTimeA
        endTime: $endTimeA
        externalModelVersionIds: $externalModelVersionIdsGlobal
        filters: $globalFilters
      ) {
        featureImportanceValues {
          dimension {
            name
          }
          importanceValue
        }
      }
    }
  }
}

Variables

{
  "modelId": "MODEL_ID",
  "startTimeA": "2024-09-07T06:00:00.000Z",
  "endTimeA": "2024-10-08T05:59:59.999Z",
  "globalFilters": [],
  "externalModelVersionIdsGlobal": []
}

Get Model Schema Stats

You can query the model schema to gather key statistics such as cardinality, percent empty, and drift. The query below retrieves data on features and predictions, including percentEmpty, cardinality, and drift metrics.

query GetSchemaStats(
  $id: ID!, 
  $startTime: DateTime!, 
  $endTime: DateTime!,
  $environmentName: ModelEnvironmentName!, 
  ) {
  node(id: $id) {
    __typename
    ... on Model {
      id
      modelSchema(
        startTime: $startTime
        endTime: $endTime
        modelEnvironmentName: $environmentName
        modelVersionEnvironmentMetadataIds: []
        
      ) {
        features(
          first: 10
        ) {
          edges {
            node {
              id
              dimension {
                name
              }
              dimensionStats{
                percentEmpty
                cardinality
              }
               dimensionDriftStats(
                driftMetric: psi, 
                externalModelVersionIds: [], 
                externalBatchIds:[], 
                filters: []
            ) {
            __typename
            ... on DimensionDriftStats {
              driftValue
              threshold
              operator
            }
                ... on DriftCalculationError {
              errorCode: code
            }
              }
              }
            }
        }
        predictions(first:10){
          edges{
            node{
              dimensionStats{
                percentEmpty
                cardinality
              }
               dimensionDriftStats(
                driftMetric: psi, 
                externalModelVersionIds: [], 
                externalBatchIds:[], 
                filters: []) {
            __typename
            ... on DimensionDriftStats {
              driftValue
              threshold
              operator
            }
                ... on DriftCalculationError {
              errorCode: code
            }
              }
              }
            }
        }
            }
          }
        }
      }

Variables

{
  "id": "MODEL_ID",
  "startTime": "2024-09-08T06:00:00.000Z",
  "endTime": "2024-10-09T05:59:59.999Z",
  "environmentName": "production"
}