deployment/docker: add grafana datasource to the docker-compose files (#5363)

https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3920
https://github.com/VictoriaMetrics/grafana-datasource/issues/113

deployment/docker/Makefile

@@ -187,8 +187,20 @@ docker-single-up:
 docker-single-down:
 	$(DOCKER_COMPOSE) -f deployment/docker/docker-compose.yml down -v
 
+docker-single-vm-datasource-up:
+	$(DOCKER_COMPOSE) -f deployment/docker/docker-compose.yml -f deployment/docker/vm-datasource/docker-compose-with-datasource.yml up -d
+
+docker-single-vm-datasource-down:
+	$(DOCKER_COMPOSE) -f deployment/docker/docker-compose.yml -f deployment/docker/vm-datasource/docker-compose-with-datasource.yml down -v
+
 docker-cluster-up:
 	$(DOCKER_COMPOSE) -f deployment/docker/docker-compose-cluster.yml up -d
 
 docker-cluster-down:
 	$(DOCKER_COMPOSE) -f deployment/docker/docker-compose-cluster.yml down -v
+
+docker-cluster-vm-datasource-up:
+	$(DOCKER_COMPOSE) -f deployment/docker/docker-compose-cluster.yml -f deployment/docker/vm-datasource/docker-compose-cluster-with-datasource.yml up -d
+
+docker-cluster-vm-datasource-down:
+	$(DOCKER_COMPOSE) -f deployment/docker/docker-compose-cluster.yml -f deployment/docker/vm-datasource/docker-compose-cluster-with-datasource.yml down -v

deployment/docker/docker-compose-cluster.yml

@@ -17,7 +17,7 @@ services:
 
   grafana:
     container_name: grafana
-    image: grafana/grafana:9.2.7
+    image: grafana/grafana:9.5.15
     depends_on:
       - "vmauth"
     ports:
@@ -25,7 +25,8 @@ services:
     restart: always
     volumes:
       - grafanadata:/var/lib/grafana
-      - ./provisioning/:/etc/grafana/provisioning/
+      - ./provisioning/datasources/prometheus-datasource:/etc/grafana/provisioning/datasources
+      - ./provisioning/dashboards:/etc/grafana/provisioning/dashboards
       - ./../../dashboards/victoriametrics-cluster.json:/var/lib/grafana/dashboards/vm.json
       - ./../../dashboards/vmagent.json:/var/lib/grafana/dashboards/vmagent.json
       - ./../../dashboards/vmalert.json:/var/lib/grafana/dashboards/vmalert.json

deployment/docker/docker-compose.yml

@@ -40,14 +40,15 @@ services:
     restart: always
   grafana:
     container_name: grafana
-    image: grafana/grafana:9.2.7
+    image: grafana/grafana:9.5.15
     depends_on:
       - "victoriametrics"
     ports:
       - 3000:3000
     volumes:
       - grafanadata:/var/lib/grafana
-      - ./provisioning/:/etc/grafana/provisioning/
+      - ./provisioning/datasources/prometheus-datasource:/etc/grafana/provisioning/datasources
+      - ./provisioning/dashboards:/etc/grafana/provisioning/dashboards
       - ./../../dashboards/victoriametrics.json:/var/lib/grafana/dashboards/vm.json
       - ./../../dashboards/victorialogs.json:/var/lib/grafana/dashboards/vl.json
       - ./../../dashboards/vmagent.json:/var/lib/grafana/dashboards/vmagent.json

deployment/docker/vm-datasource/config/victoriametrics-datasource.yml

@@ -0,0 +1,36 @@
+apiVersion: 1
+
+# List of data sources to insert/update depending on what's
+# available in the database.
+datasources:
+  # <string, required> Name of the VictoriaMetrics datasource
+  # displayed in Grafana panels and queries.
+  - name: VictoriaMetrics
+    # <string, required> Sets the data source type.
+    type: victoriametrics-datasource
+      # <string, required> Sets the access mode, either
+      # proxy or direct (Server or Browser in the UI).
+    # Some data sources are incompatible with any setting
+    # but proxy (Server).
+    access: proxy
+    # <string> Sets default URL of the single node version of VictoriaMetrics
+    url: http://victoriametrics:8428
+    # <string> Sets the pre-selected datasource for new panels.
+    # You can set only one default data source per organization.
+    isDefault: true
+
+    # <string, required> Name of the VictoriaMetrics datasource
+    # displayed in Grafana panels and queries.
+  - name: VictoriaMetrics - cluster
+    # <string, required> Sets the data source type.
+    type: victoriametrics-datasource
+    # <string, required> Sets the access mode, either
+    # proxy or direct (Server or Browser in the UI).
+    # Some data sources are incompatible with any setting
+    # but proxy (Server).
+    access: proxy
+    # <string> Sets default URL of the cluster version of VictoriaMetrics
+    url: http://vmauth:8427/select/0/prometheus
+    # <string> Sets the pre-selected datasource for new panels.
+    # You can set only one default data source per organization.
+    isDefault: false

deployment/docker/vm-datasource/docker-compose-cluster-with-datasource.yml

@@ -0,0 +1,24 @@
+version: "3.5"
+services:
+  grafana:
+    container_name: grafana
+    image: grafana/grafana:9.5.15
+    depends_on:
+      - "vmauth"
+    ports:
+      - 3000:3000
+    entrypoint: ["/bin/bash", "-c"]
+    command: ["chmod +x /download-vm-datasource.sh && /download-vm-datasource.sh && /run.sh"]
+    volumes:
+      - grafanadata:/var/lib/grafana
+      - ./vm-datasource/config:/etc/grafana/provisioning/datasources
+      - ./provisioning/dashboards:/etc/grafana/provisioning/dashboards
+      - ./provisioning/plugins/:/var/lib/grafana/plugins
+      - ./../../dashboards/vm/victoriametrics-cluster.json:/var/lib/grafana/dashboards/vm.json
+      - ./../../dashboards/vm/vmagent.json:/var/lib/grafana/dashboards/vmagent.json
+      - ./../../dashboards/vm/vmalert.json:/var/lib/grafana/dashboards/vmalert.json
+      - ./vm-datasource/scripts/download-vm-datasource.sh:/download-vm-datasource.sh
+    environment:
+      - "GF_ALLOW_LOADING_UNSIGNED_PLUGINS=victoriametrics-datasource"
+      - "GF_DEFAULT_APP_MODE=development"
+    restart: always

deployment/docker/vm-datasource/docker-compose-with-datasource.yml

@@ -0,0 +1,26 @@
+version: "3.5"
+services:
+  grafana:
+    container_name: grafana
+    image: grafana/grafana:9.5.15
+    depends_on:
+      - "victoriametrics"
+    ports:
+      - 3000:3000
+    entrypoint: [ "/bin/bash", "-c" ]
+    command: [ "chmod +x /download-vm-datasource.sh && /download-vm-datasource.sh && /run.sh" ]
+    volumes:
+      - grafanadata:/var/lib/grafana
+      - ./vm-datasource/config:/etc/grafana/provisioning/datasources
+      - ./provisioning/dashboards:/etc/grafana/provisioning/dashboards
+      - ./provisioning/plugins/:/var/lib/grafana/plugins
+      - ./../../dashboards/vm/victoriametrics.json:/var/lib/grafana/dashboards/vm.json
+      - ./../../dashboards/vm/vmagent.json:/var/lib/grafana/dashboards/vmagent.json
+      - ./../../dashboards/vm/vmalert.json:/var/lib/grafana/dashboards/vmalert.json
+      - ./vm-datasource/scripts/download-vm-datasource.sh:/download-vm-datasource.sh
+    environment:
+      - "GF_ALLOW_LOADING_UNSIGNED_PLUGINS=victoriametrics-datasource"
+      - "GF_DEFAULT_APP_MODE=development"
+    networks:
+      - vm_net
+    restart: always

deployment/docker/vm-datasource/scripts/download-vm-datasource.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -ex
+
+VM_DS_PATH='/var/lib/grafana/plugins/victoriametrics-datasource'
+PLUGIN_PATH='/var/lib/grafana/plugins'
+
+if [[ -f ${VM_DS_PATH}/plugin.json ]]; then
+    ver=$(cat ${VM_DS_PATH}/plugin.json)
+    if [[ ! -z "$ver" ]]; then
+    exit
+    fi
+fi
+
+echo "Victoriametrics datasource is not installed. Installing datasource..."
+rm -rf ${VM_DS_PATH}/* || true
+mkdir -p ${VM_DS_PATH}
+
+export LATEST_VERSION=$(curl https://api.github.com/repos/VictoriaMetrics/grafana-datasource/releases/latest | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' | head -1); \
+curl -L https://github.com/VictoriaMetrics/grafana-datasource/releases/download/${LATEST_VERSION}/victoriametrics-datasource-${LATEST_VERSION}.tar.gz -o ${PLUGIN_PATH}/plugin.tar.gz && \
+tar -xzf ${PLUGIN_PATH}/plugin.tar.gz -C ${PLUGIN_PATH}
+echo "Victoriametrics datasource has been installed."
+rm ${PLUGIN_PATH}/plugin.tar.gz

docs/CHANGELOG.md

@@ -53,6 +53,7 @@ The sandbox cluster installation is running under the constant load generated by
 * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): rename cmd-line flag `vm-native-disable-retries` to `vm-native-disable-per-metric-migration` to better reflect its meaning.
 * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add `-vm-native-src-insecure-skip-verify` and `-vm-native-dst-insecure-skip-verify` command-line flags for native protocol. It can be used for skipping TLS certificate verification when connecting to the source or destination addresses.
 * FEATURE: [Alerting rules for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts): add `job` label to `DiskRunsOutOfSpace` alerting rule, so it is easier to understand to which installation the triggered instance belongs.
+* FEATURE: add [VictoriaMetrics datasource](https://github.com/VictoriaMetrics/grafana-datasource) to docker compose environment. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5363).
 
 * BUGFIX: properly return errors from [export APIs](https://docs.victoriametrics.com/#how-to-export-time-series). Previously these errors were silently suppressed. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5649).
 * BUGFIX: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): properly return full results when `-search.skipSlowReplicas` command-line flag is passed to `vmselect` and when [vmstorage groups](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#vmstorage-groups-at-vmselect) are in use. Previously partial results could be returned in this case.