Commit 296fb4d9 authored by Tobias Schottdorf's avatar Tobias Schottdorf

localmetrics: add local Grafana timeseries tooling

Add a docker-compose setup that starts a local Grafana backed by a local
Postgres along with a helper that can import timeseries data into the
Postgres instance which the Grafana instance is configured to display.

Consult scripts/localmetrics/README.md for a quickstart.

This isn't a valuable debug tool just yet, but with a bit of elbow
grease, I believe that it will become an invaluable tool to avoid
the many back-and-forth round-trips we have these days with customers
to exchange screenshots of the Admin UI.

To make it truly useful, we need

1. [timeseries in debug.zip](https://github.com/cockroachdb/cockroach/pull/50432)
2. auto-generate dashboards from `./pkg/ts/catalog`.

Both are totally doable, and even without 2) there's already some
utility as it's easy to make ad-hoc panels in Grafana thanks to the
built-in query builder.

Release note: None
parent 9669df8a
......@@ -20,6 +20,7 @@ import (
"github.com/cockroachdb/cockroach/pkg/ts/tspb"
"github.com/cockroachdb/cockroach/pkg/util/log"
"github.com/cockroachdb/cockroach/pkg/util/timeutil"
"github.com/spf13/cobra"
)
......@@ -84,7 +85,7 @@ type csvTSWriter struct {
func (w csvTSWriter) Emit(data *tspb.TimeSeriesData) error {
for _, d := range data.Datapoints {
if err := w.w.Write(
[]string{data.Name, time.Unix(0, d.TimestampNanos).In(time.UTC).Format(time.RFC3339), data.Source, fmt.Sprint(d.Value)},
[]string{data.Name, timeutil.Unix(0, d.TimestampNanos).In(time.UTC).Format(time.RFC3339), data.Source, fmt.Sprint(d.Value)},
); err != nil {
return err
}
......@@ -112,7 +113,7 @@ func (w rawTSWriter) Emit(data *tspb.TimeSeriesData) error {
fmt.Fprintf(w.w, "%s %s\n", data.Name, data.Source)
}
for _, d := range data.Datapoints {
fmt.Fprintf(w.w, "%s %v\n", d.TimestampNanos, d.Value)
fmt.Fprintf(w.w, "%v %v\n", d.TimestampNanos, d.Value)
}
return nil
}
# Local timeseries tooling
## Quick Start
```
docker-compose up -d
./import-csv.sh < (curl https://gist.githubusercontent.com/tbg/98d9814f624629833e6cfb7d25cb8258/raw/70a96d50032361f864b240dbd9f1c36c385b7515/sample.csv)
# User/Pass admin/x
open http://127.0.0.1:3000
```
## Usage:
### Step 1: procure a metrics dump
The source of this could be a `debug zip` (pending [#50432]), or
`./cockroach debug tsdump --format=csv --host=... > dump.csv`.
### Step 2: `docker-compose up -d`
Not much more to be said. Unsurprisingly, this needs Docker to work. Omit the
`-d` if you want to see what's going on behind the scenes. It may take a moment
for the next step to work.
### Step 3: `./import-csv.sh < dump.csv`
This loads the CSV data into your local Postgres instance. Note that it will
truncate any existing data you have imported, so you're not accidentally mixing
metrics from various sources.
If you legitimately want to import multiple csvs at once, use `cat *.csv |
./import-csv` instead.
### Step 3: open [Grafana](http://127.0.0.1:3000)
Log in as user `admin`, password `x`. You should be able to find a reference to
the CockroachDB dashboard on the landing page.
### Step 4: Play
You can edit the provided panel or add panels to plot interesting metrics.
A good starting point for learning how to do that is the [Grafana blog].
Replace ./grafana/dashboards/home.json if you want the changes to persist.
TODO(tbg): auto-generate a better home.json from `pkg/ts/catalog`.
### Step 5: docker-compose stop
To avoid hogging resources on your machine. The postgres database is on your
local file system, so it will remain. If you want to nuke everything, use
`down` instead of `stop` and then `git clean -f .`.
[#50432]: https://github.com/cockroachdb/cockroach/pull/50432
[Grafana blog]: https://grafana.com/blog/2018/10/15/make-time-series-exploration-easier-with-the-postgresql/timescaledb-query-editor/
version: '2.0'
services:
grafana:
build: ./grafana
container_name: grafana
ports:
- 3000:3000
links:
- postgres
# It's already in the container, but by mounting it we can edit these files
# without rebuilding the container.
volumes:
- ./grafana/dashboards:/var/lib/grafana/dashboards
postgres:
image: postgres:12.3-alpine
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: postgres
PGDATA: /postgres-data
ports:
- 5432:5432
volumes:
- ./postgres-data:/postgres-data
# NB: want master to pick up GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH:
# https://github.com/grafana/grafana/pull/25595
# TODO(tbg): pin a release once the above is released to avoid random breakage.
FROM grafana/grafana:master
ENV GF_INSTALL_PLUGINS grafana-clock-panel,briangann-gauge-panel,natel-plotly-panel,grafana-simple-json-datasource
# Set up admin login
ENV GF_SECURITY_ADMIN_PASSWORD x
# Disable anonymous login for now - when we have great auto-generated dashboards
# we can enable it, but as is a shitty dashboard you can't edit isn't a great
# place to land by default.
#ENV GF_AUTH_ANONYMOUS_ENABLED true
#ENV GF_AUTH_ANONYMOUS_ORG_NAME Main Org.
ENV GF_USERS_ALLOW_SIGN_UP false
ENV GF_DASHBOARDS_JSON_ENABLED true
ENV GF_DASHBOARDS_JSON_PATH ./docker-compose.d/grafana
ENV GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH /var/lib/grafana/dashboards/home.json
COPY ./postgres.yml /etc/grafana/provisioning/datasources/postgres.yml
COPY ./dashboards.yml /etc/grafana/provisioning/dashboards/dashboards.yml
# /var/lib/grafana/dashboards/ is mounted in from the outside, to allow editing
# while Grafana is running.
# COPY ./dashboards /var/lib/grafana/dashboards
apiVersion: 1
providers:
- name: 'default'
orgId: 1
folder: ''
type: file
disableDeletion: false
updateIntervalSeconds: 10
options:
path: /var/lib/grafana/dashboards
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"links": [],
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 0
},
"hiddenSeries": false,
"id": 2,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"format": "time_series",
"group": [],
"metricColumn": "concat(name,'.s',source)",
"rawQuery": false,
"rawSql": "SELECT\n \"time\" AS \"time\",\n concat(name,'.s',source) AS metric,\n (CASE WHEN value >= lag(value) OVER (PARTITION BY concat(name,'.s',source) ORDER BY \"time\") THEN value - lag(value) OVER (PARTITION BY concat(name,'.s',source) ORDER BY \"time\") WHEN lag(value) OVER (PARTITION BY concat(name,'.s',source) ORDER BY \"time\") IS NULL THEN NULL ELSE value END)/extract(epoch from \"time\" - lag(\"time\") OVER (PARTITION BY concat(name,'.s',source) ORDER BY \"time\")) AS \"value\"\nFROM metrics\nWHERE\n $__timeFilter(\"time\") AND\n name = 'cr.node.distsender.batches'\nORDER BY 1,2",
"refId": "A",
"select": [
[
{
"params": [
"value"
],
"type": "column"
},
{
"params": [
"rate"
],
"type": "window"
},
{
"params": [
"value"
],
"type": "alias"
}
]
],
"table": "metrics",
"timeColumn": "\"time\"",
"timeColumnType": "timestamp",
"where": [
{
"name": "$__timeFilter",
"params": [],
"type": "macro"
},
{
"datatype": "varchar",
"name": "",
"params": [
"name",
"=",
"'cr.node.distsender.batches'"
],
"type": "expression"
}
]
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Playground Panel",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"refresh": false,
"schemaVersion": 26,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
]
},
"timezone": "UTC",
"title": "CockroachDB",
"uid": "crdb",
"version": 1
}
apiVersion: 1
datasources:
- name: pg
type: postgres
url: postgres:5432
access: proxy
user: postgres
database: postgres
basicAuth: false
isDefault: true
jsonData:
sslmode: disable
secureJsonData:
password: postgres
version: 1
editable: true
#!/bin/sh
set -euo pipefail
psql 'postgresql://postgres:[email protected]?sslmode=disable' \
-c 'create table if not exists metrics(name varchar(255), time timestamp, source varchar(255), value real)' \
-c 'truncate table metrics;' \
-c 'copy metrics(name, time, source, value) from stdin csv;' < /dev/stdin
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment