{
  "title": "API Feature Lab: RED",
  "uid": "api-feature-lab-red",
  "description": "Rate, errors and duration of the API lab (api.djmiskey.com on Vercel and api-cf.djmiskey.com on Cloudflare Workers), from span metrics generated from its traces. Built by obs.djmiskey.com.",
  "tags": [
    "api-feature-lab",
    "red",
    "span-metrics"
  ],
  "timezone": "browser",
  "refresh": "1m",
  "time": {
    "from": "now-6h",
    "to": "now"
  },
  "schemaVersion": 39,
  "editable": true,
  "links": [
    {
      "title": "Observability Lab",
      "url": "https://obs.djmiskey.com/metrics",
      "type": "link",
      "targetBlank": true
    }
  ],
  "templating": {
    "list": [
      {
        "name": "prometheus",
        "label": "Metrics",
        "type": "datasource",
        "query": "prometheus",
        "regex": ".*grafanacloud.*prom.*",
        "current": {}
      },
      {
        "name": "loki",
        "label": "Logs",
        "type": "datasource",
        "query": "loki",
        "regex": ".*grafanacloud.*logs.*",
        "current": {}
      }
    ]
  },
  "panels": [
    {
      "id": 1,
      "type": "row",
      "title": "Per platform",
      "collapsed": false,
      "gridPos": {
        "x": 0,
        "y": 0,
        "w": 24,
        "h": 1
      },
      "panels": []
    },
    {
      "id": 2,
      "title": "Requests per second",
      "type": "timeseries",
      "gridPos": {
        "x": 0,
        "y": 1,
        "w": 12,
        "h": 8
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${prometheus}"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "reqps",
          "custom": {
            "lineWidth": 2,
            "fillOpacity": 0,
            "showPoints": "never"
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "vercel"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "mode": "fixed",
                  "fixedColor": "#2a78d6"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "cloudflare-workers"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "mode": "fixed",
                  "fixedColor": "#eb6834"
                }
              }
            ]
          }
        ]
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      },
      "transformations": [],
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${prometheus}"
          },
          "expr": "sum by (platform) (label_replace(label_replace(rate(traces_spanmetrics_calls_total{span_kind=\"SPAN_KIND_SERVER\", cloud_region!=\"\"}[$__rate_interval]), \"platform\", \"vercel\", \"cloud_region\", \"[a-z]+[0-9]+\"), \"platform\", \"cloudflare-workers\", \"cloud_region\", \"[A-Z]+\"))",
          "legendFormat": "{{platform}}",
          "instant": false,
          "range": true,
          "format": "time_series"
        }
      ]
    },
    {
      "id": 3,
      "title": "p95 response time",
      "description": "95% of requests were faster than this. Workers far from the database (Virginia) pay for distance on every query.",
      "type": "timeseries",
      "gridPos": {
        "x": 12,
        "y": 1,
        "w": 12,
        "h": 8
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${prometheus}"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "custom": {
            "lineWidth": 2,
            "fillOpacity": 0,
            "showPoints": "never"
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "vercel"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "mode": "fixed",
                  "fixedColor": "#2a78d6"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "cloudflare-workers"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "mode": "fixed",
                  "fixedColor": "#eb6834"
                }
              }
            ]
          }
        ]
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      },
      "transformations": [],
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${prometheus}"
          },
          "expr": "histogram_quantile(0.95, sum by (platform) (label_replace(label_replace(rate(traces_spanmetrics_latency{span_kind=\"SPAN_KIND_SERVER\", cloud_region!=\"\"}[$__rate_interval]), \"platform\", \"vercel\", \"cloud_region\", \"[a-z]+[0-9]+\"), \"platform\", \"cloudflare-workers\", \"cloud_region\", \"[A-Z]+\")))",
          "legendFormat": "{{platform}}",
          "instant": false,
          "range": true,
          "format": "time_series"
        }
      ]
    },
    {
      "id": 4,
      "title": "Server errors (5xx share)",
      "type": "timeseries",
      "gridPos": {
        "x": 0,
        "y": 9,
        "w": 12,
        "h": 8
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${prometheus}"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "percentunit",
          "custom": {
            "lineWidth": 2,
            "fillOpacity": 0,
            "showPoints": "never"
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "vercel"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "mode": "fixed",
                  "fixedColor": "#2a78d6"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "cloudflare-workers"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "mode": "fixed",
                  "fixedColor": "#eb6834"
                }
              }
            ]
          }
        ]
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      },
      "transformations": [],
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${prometheus}"
          },
          "expr": "(sum by (platform) (label_replace(label_replace(rate(traces_spanmetrics_calls_total{span_kind=\"SPAN_KIND_SERVER\", cloud_region!=\"\", status_code=\"STATUS_CODE_ERROR\"}[$__rate_interval]), \"platform\", \"vercel\", \"cloud_region\", \"[a-z]+[0-9]+\"), \"platform\", \"cloudflare-workers\", \"cloud_region\", \"[A-Z]+\")) or sum by (platform) (label_replace(label_replace(rate(traces_spanmetrics_calls_total{span_kind=\"SPAN_KIND_SERVER\", cloud_region!=\"\"}[$__rate_interval]), \"platform\", \"vercel\", \"cloud_region\", \"[a-z]+[0-9]+\"), \"platform\", \"cloudflare-workers\", \"cloud_region\", \"[A-Z]+\")) * 0) / sum by (platform) (label_replace(label_replace(rate(traces_spanmetrics_calls_total{span_kind=\"SPAN_KIND_SERVER\", cloud_region!=\"\"}[$__rate_interval]), \"platform\", \"vercel\", \"cloud_region\", \"[a-z]+[0-9]+\"), \"platform\", \"cloudflare-workers\", \"cloud_region\", \"[A-Z]+\"))",
          "legendFormat": "{{platform}}",
          "instant": false,
          "range": true,
          "format": "time_series"
        }
      ]
    },
    {
      "id": 5,
      "title": "Client errors (4xx share)",
      "description": "404s, 422s, 429s: the caller's mistakes, not the service's.",
      "type": "timeseries",
      "gridPos": {
        "x": 12,
        "y": 9,
        "w": 12,
        "h": 8
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${prometheus}"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "percentunit",
          "custom": {
            "lineWidth": 2,
            "fillOpacity": 0,
            "showPoints": "never"
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "vercel"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "mode": "fixed",
                  "fixedColor": "#2a78d6"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "cloudflare-workers"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "mode": "fixed",
                  "fixedColor": "#eb6834"
                }
              }
            ]
          }
        ]
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      },
      "transformations": [],
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${prometheus}"
          },
          "expr": "(sum by (platform) (label_replace(label_replace(rate(traces_spanmetrics_calls_total{span_kind=\"SPAN_KIND_SERVER\", cloud_region!=\"\", http_response_status_code=~\"4..\"}[$__rate_interval]), \"platform\", \"vercel\", \"cloud_region\", \"[a-z]+[0-9]+\"), \"platform\", \"cloudflare-workers\", \"cloud_region\", \"[A-Z]+\")) or sum by (platform) (label_replace(label_replace(rate(traces_spanmetrics_calls_total{span_kind=\"SPAN_KIND_SERVER\", cloud_region!=\"\"}[$__rate_interval]), \"platform\", \"vercel\", \"cloud_region\", \"[a-z]+[0-9]+\"), \"platform\", \"cloudflare-workers\", \"cloud_region\", \"[A-Z]+\")) * 0) / sum by (platform) (label_replace(label_replace(rate(traces_spanmetrics_calls_total{span_kind=\"SPAN_KIND_SERVER\", cloud_region!=\"\"}[$__rate_interval]), \"platform\", \"vercel\", \"cloud_region\", \"[a-z]+[0-9]+\"), \"platform\", \"cloudflare-workers\", \"cloud_region\", \"[A-Z]+\"))",
          "legendFormat": "{{platform}}",
          "instant": false,
          "range": true,
          "format": "time_series"
        }
      ]
    },
    {
      "id": 6,
      "type": "row",
      "title": "Per endpoint",
      "collapsed": false,
      "gridPos": {
        "x": 0,
        "y": 17,
        "w": 24,
        "h": 1
      },
      "panels": []
    },
    {
      "id": 7,
      "title": "Requests per second, by endpoint",
      "type": "timeseries",
      "gridPos": {
        "x": 0,
        "y": 18,
        "w": 12,
        "h": 8
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${prometheus}"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "reqps",
          "custom": {
            "lineWidth": 2,
            "fillOpacity": 0,
            "showPoints": "never"
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      },
      "transformations": [],
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${prometheus}"
          },
          "expr": "sum by (span_name) (label_replace(label_replace(rate(traces_spanmetrics_calls_total{span_kind=\"SPAN_KIND_SERVER\", cloud_region!=\"\"}[$__rate_interval]), \"platform\", \"vercel\", \"cloud_region\", \"[a-z]+[0-9]+\"), \"platform\", \"cloudflare-workers\", \"cloud_region\", \"[A-Z]+\"))",
          "legendFormat": "{{span_name}}",
          "instant": false,
          "range": true,
          "format": "time_series"
        }
      ]
    },
    {
      "id": 8,
      "title": "p95 by endpoint",
      "type": "timeseries",
      "gridPos": {
        "x": 12,
        "y": 18,
        "w": 12,
        "h": 8
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${prometheus}"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "custom": {
            "lineWidth": 2,
            "fillOpacity": 0,
            "showPoints": "never"
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      },
      "transformations": [],
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${prometheus}"
          },
          "expr": "histogram_quantile(0.95, sum by (span_name) (label_replace(label_replace(rate(traces_spanmetrics_latency{span_kind=\"SPAN_KIND_SERVER\", cloud_region!=\"\"}[$__rate_interval]), \"platform\", \"vercel\", \"cloud_region\", \"[a-z]+[0-9]+\"), \"platform\", \"cloudflare-workers\", \"cloud_region\", \"[A-Z]+\")))",
          "legendFormat": "{{span_name}}",
          "instant": false,
          "range": true,
          "format": "time_series"
        }
      ]
    },
    {
      "id": 9,
      "title": "Endpoints over the selected range",
      "description": "p95 per endpoint and platform for the whole time range.",
      "type": "table",
      "gridPos": {
        "x": 0,
        "y": 26,
        "w": 24,
        "h": 9
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${prometheus}"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        },
        "showHeader": true,
        "sortBy": [
          {
            "displayName": "p95",
            "desc": true
          }
        ]
      },
      "transformations": [
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "Time": true
            },
            "indexByName": {
              "span_name": 0,
              "platform": 1,
              "Value": 2
            },
            "renameByName": {
              "span_name": "Endpoint",
              "platform": "Platform",
              "Value": "p95"
            }
          }
        }
      ],
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${prometheus}"
          },
          "expr": "histogram_quantile(0.95, sum by (span_name, platform) (label_replace(label_replace(rate(traces_spanmetrics_latency{span_kind=\"SPAN_KIND_SERVER\", cloud_region!=\"\"}[$__range]), \"platform\", \"vercel\", \"cloud_region\", \"[a-z]+[0-9]+\"), \"platform\", \"cloudflare-workers\", \"cloud_region\", \"[A-Z]+\")))",
          "legendFormat": "__auto",
          "instant": true,
          "range": false,
          "format": "table"
        }
      ]
    },
    {
      "id": 10,
      "type": "row",
      "title": "Database",
      "collapsed": false,
      "gridPos": {
        "x": 0,
        "y": 35,
        "w": 24,
        "h": 1
      },
      "panels": []
    },
    {
      "id": 11,
      "title": "p95 per query",
      "description": "Client spans named after the statement and table, e.g. SELECT scores.",
      "type": "timeseries",
      "gridPos": {
        "x": 0,
        "y": 36,
        "w": 12,
        "h": 8
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${prometheus}"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "custom": {
            "lineWidth": 2,
            "fillOpacity": 0,
            "showPoints": "never"
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      },
      "transformations": [],
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${prometheus}"
          },
          "expr": "histogram_quantile(0.95, sum by (span_name) (rate(traces_spanmetrics_latency{span_kind=\"SPAN_KIND_CLIENT\", span_name=~\"(SELECT|INSERT|UPDATE|DELETE|WITH|CREATE).*\"}[$__rate_interval])))",
          "legendFormat": "{{span_name}}",
          "instant": false,
          "range": true,
          "format": "time_series"
        }
      ]
    },
    {
      "id": 12,
      "title": "Queries per second",
      "type": "timeseries",
      "gridPos": {
        "x": 12,
        "y": 36,
        "w": 12,
        "h": 8
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${prometheus}"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "qps",
          "custom": {
            "lineWidth": 2,
            "fillOpacity": 0,
            "showPoints": "never"
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      },
      "transformations": [],
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${prometheus}"
          },
          "expr": "sum by (span_name) (rate(traces_spanmetrics_calls_total{span_kind=\"SPAN_KIND_CLIENT\", span_name=~\"(SELECT|INSERT|UPDATE|DELETE|WITH|CREATE).*\"}[$__rate_interval]))",
          "legendFormat": "{{span_name}}",
          "instant": false,
          "range": true,
          "format": "time_series"
        }
      ]
    },
    {
      "id": 13,
      "type": "row",
      "title": "Alerting",
      "collapsed": false,
      "gridPos": {
        "x": 0,
        "y": 44,
        "w": 24,
        "h": 1
      },
      "panels": []
    },
    {
      "id": 14,
      "title": "Alert: server error rate",
      "description": "The alert rule's own query: a value appears only while a platform has more than 5% 5xx over 10m with at least 10 requests. To create the rule: this panel's menu → More… → New alert rule (the query arrives filled in), condition \"is above 0\", no data = Normal.",
      "type": "timeseries",
      "gridPos": {
        "x": 0,
        "y": 45,
        "w": 24,
        "h": 8
      },
      "datasource": {
        "type": "prometheus",
        "uid": "${prometheus}"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "percentunit",
          "custom": {
            "lineWidth": 2,
            "fillOpacity": 0,
            "showPoints": "never"
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "vercel"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "mode": "fixed",
                  "fixedColor": "#2a78d6"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "cloudflare-workers"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "mode": "fixed",
                  "fixedColor": "#eb6834"
                }
              }
            ]
          }
        ]
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      },
      "transformations": [],
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "${prometheus}"
          },
          "expr": "((sum by (platform) (label_replace(label_replace(rate(traces_spanmetrics_calls_total{span_kind=\"SPAN_KIND_SERVER\", cloud_region!=\"\", status_code=\"STATUS_CODE_ERROR\"}[10m]), \"platform\", \"vercel\", \"cloud_region\", \"[a-z]+[0-9]+\"), \"platform\", \"cloudflare-workers\", \"cloud_region\", \"[A-Z]+\")) or sum by (platform) (label_replace(label_replace(rate(traces_spanmetrics_calls_total{span_kind=\"SPAN_KIND_SERVER\", cloud_region!=\"\"}[10m]), \"platform\", \"vercel\", \"cloud_region\", \"[a-z]+[0-9]+\"), \"platform\", \"cloudflare-workers\", \"cloud_region\", \"[A-Z]+\")) * 0) / sum by (platform) (label_replace(label_replace(rate(traces_spanmetrics_calls_total{span_kind=\"SPAN_KIND_SERVER\", cloud_region!=\"\"}[10m]), \"platform\", \"vercel\", \"cloud_region\", \"[a-z]+[0-9]+\"), \"platform\", \"cloudflare-workers\", \"cloud_region\", \"[A-Z]+\"))) > 0.05 and (sum by (platform) (label_replace(label_replace(increase(traces_spanmetrics_calls_total{span_kind=\"SPAN_KIND_SERVER\", cloud_region!=\"\"}[10m]), \"platform\", \"vercel\", \"cloud_region\", \"[a-z]+[0-9]+\"), \"platform\", \"cloudflare-workers\", \"cloud_region\", \"[A-Z]+\"))) >= 10",
          "legendFormat": "{{platform}}",
          "instant": false,
          "range": true,
          "format": "time_series"
        }
      ]
    },
    {
      "id": 15,
      "type": "row",
      "title": "Logs",
      "collapsed": false,
      "gridPos": {
        "x": 0,
        "y": 53,
        "w": 24,
        "h": 1
      },
      "panels": []
    },
    {
      "id": 16,
      "title": "Warnings and errors",
      "type": "logs",
      "datasource": {
        "type": "loki",
        "uid": "${loki}"
      },
      "gridPos": {
        "x": 0,
        "y": 54,
        "w": 24,
        "h": 10
      },
      "description": "Every 4xx and 5xx request line, newest first. Each has a requestId: paste it into Tempo to open its trace.",
      "options": {
        "showTime": true,
        "wrapLogMessage": true,
        "sortOrder": "Descending",
        "enableLogDetails": true
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "loki",
            "uid": "${loki}"
          },
          "expr": "{service_name=\"api-feature-lab\"} | json | level=~\"warn|error\""
        }
      ]
    }
  ]
}