From 73afeaf510533b3cd2143d6b57bfd53fb5ecd9be Mon Sep 17 00:00:00 2001 From: Andrii Chubatiuk Date: Wed, 25 Sep 2024 19:33:26 +0300 Subject: [PATCH] deployment: restructure victorialogs examples (#6971) ### Describe Your Changes - Use common compose.yaml file for all victorialogs setups to set version in a single place and override it on demand for each agent and protocol - Replaced multiple victorialogs instances in HA setup with single setup with `deploy.replica` parameter set - Added fluentd setup ### Checklist The following checks are **mandatory**: - [ ] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/). (cherry picked from commit e75ae1b2740d1692f06907ec9aa38e1972e439b6) --- deployment/docker/victorialogs/compose-ha.yml | 4 + .../{telegraf-docker => }/compose.yml | 55 +++++---- .../victorialogs/filebeat-syslog/README.md | 45 -------- .../victorialogs/filebeat-syslog/compose.yml | 23 ---- .../docker/victorialogs/filebeat/README.md | 36 ++---- .../docker/victorialogs/filebeat/compose.yml | 50 +-------- .../filebeat/elasticsearch/compose.yml | 3 + .../filebeat/{ => elasticsearch}/filebeat.yml | 0 .../docker/victorialogs/filebeat/scrape.yml | 16 --- .../victorialogs/filebeat/syslog/compose.yml | 3 + .../syslog}/filebeat.yml | 0 .../fluentbit-ha-single-node/README.md | 88 --------------- .../fluentbit-ha-single-node/auth.yml | 6 - .../fluentbit-ha-single-node/compose.yml | 38 ------- .../docker/victorialogs/fluentbit/README.md | 46 ++------ .../docker/victorialogs/fluentbit/compose.yml | 18 +-- .../fluentbit/jsonline-ha/compose.yml | 5 + .../jsonline-ha}/fluent-bit.conf | 36 +++--- .../fluentbit/jsonline/compose.yml | 3 + .../fluentbit/jsonline/fluent-bit.conf | 30 +++++ .../victorialogs/fluentbit/loki/compose.yml | 3 + .../fluentbit/{ => loki}/fluent-bit.conf | 25 ----- .../docker/victorialogs/fluentd/Dockerfile | 8 ++ .../docker/victorialogs/fluentd/README.md | 35 ++++++ .../docker/victorialogs/fluentd/compose.yml | 11 ++ .../fluentd/elasticsearch/compose.yml | 3 + .../fluentd/elasticsearch/fluent.conf | 13 +++ .../victorialogs/fluentd/jsonline/compose.yml | 3 + .../victorialogs/fluentd/jsonline/fluent.conf | 12 ++ .../victorialogs/fluentd/loki/compose.yml | 3 + .../victorialogs/fluentd/loki/fluent.conf | 17 +++ .../logstash-ha-single-node/README.md | 71 ------------ .../logstash-ha-single-node/auth.yml | 6 - .../logstash-ha-single-node/compose.yml | 41 ------- .../logstash-ha-single-node/logstash.yml | 2 - .../docker/victorialogs/logstash/Dockerfile | 2 +- .../docker/victorialogs/logstash/README.md | 38 ++----- .../docker/victorialogs/logstash/compose.yml | 19 +--- .../logstash/elasticsearch/compose.yml | 3 + .../{ => elasticsearch}/pipeline.conf | 8 -- .../logstash/jsonline-ha/compose.yml | 5 + .../jsonline-ha}/pipeline.conf | 4 +- .../logstash/jsonline/compose.yml | 3 + .../logstash/jsonline/pipeline.conf | 13 +++ .../victorialogs/logstash/loki/compose.yml | 3 + .../victorialogs/logstash/loki/pipeline.conf | 11 ++ .../{elasticsearch => }/README.md | 17 ++- .../opentelemetry-collector/compose.yml | 15 +++ .../elasticsearch-ha/compose.yml | 5 + .../elasticsearch-ha/config.yaml | 15 +++ .../elasticsearch/compose.yml | 51 +-------- .../elasticsearch/config.yaml | 2 +- .../elasticsearch/scrape.yml | 11 -- .../opentelemetry-collector/loki/README.md | 27 ----- .../opentelemetry-collector/loki/compose.yml | 51 +-------- .../opentelemetry-collector/loki/config.yaml | 2 +- .../opentelemetry-collector/loki/scrape.yml | 11 -- .../opentelemetry-collector/otlp/README.md | 27 ----- .../opentelemetry-collector/otlp/compose.yml | 51 +-------- .../opentelemetry-collector/otlp/config.yaml | 2 +- .../opentelemetry-collector/otlp/scrape.yml | 11 -- .../opentelemetry-collector/syslog/README.md | 27 ----- .../syslog/compose.yml | 52 +-------- .../opentelemetry-collector/syslog/scrape.yml | 11 -- .../docker/victorialogs/promtail/compose.yml | 18 +-- .../victorialogs/promtail/loki/compose.yml | 3 + .../promtail/{ => loki}/config.yml | 2 +- .../victorialogs/telegraf-docker/scrape.yml | 11 -- .../{telegraf-docker => telegraf}/README.md | 17 ++- .../docker/victorialogs/telegraf/compose.yml | 20 ++++ .../telegraf/elasticsearch/compose.yml | 3 + .../elasticsearch}/telegraf.conf | 17 --- .../telegraf/jsonline/compose.yml | 3 + .../telegraf/jsonline/telegraf.conf | 43 ++++++++ .../victorialogs/telegraf/loki/compose.yml | 3 + .../victorialogs/telegraf/loki/telegraf.conf | 44 ++++++++ .../victorialogs/telegraf/syslog/compose.yml | 3 + .../telegraf/syslog/telegraf.conf | 41 +++++++ .../vector-ha-single-node/README.md | 104 ------------------ .../vector-ha-single-node/auth.yml | 6 - .../vector-ha-single-node/compose.yml | 48 -------- .../vector-ha-single-node/vector.yaml | 58 ---------- .../docker/victorialogs/vector/README.md | 51 +++------ .../docker/victorialogs/vector/compose.yml | 42 +------ .../vector/elasticsearch/compose.yaml | 3 + .../vector/elasticsearch/vector.yaml | 43 ++++++++ .../vector/jsonline-ha/compose.yml | 5 + .../vector/{ => jsonline-ha}/vector.yaml | 42 +++---- .../victorialogs/vector/jsonline/compose.yml | 3 + .../victorialogs/vector/jsonline/vector.yaml | 43 ++++++++ .../victorialogs/vector/loki/compose.yml | 3 + .../victorialogs/vector/loki/vector.yaml | 38 +++++++ .../docker/victorialogs/vector/scrape.yml | 11 -- docs/VictoriaLogs/data-ingestion/Telegraf.md | 2 +- docs/VictoriaLogs/data-ingestion/Vector.md | 2 +- 95 files changed, 705 insertions(+), 1282 deletions(-) create mode 100644 deployment/docker/victorialogs/compose-ha.yml rename deployment/docker/victorialogs/{telegraf-docker => }/compose.yml (51%) delete mode 100644 deployment/docker/victorialogs/filebeat-syslog/README.md delete mode 100644 deployment/docker/victorialogs/filebeat-syslog/compose.yml create mode 100644 deployment/docker/victorialogs/filebeat/elasticsearch/compose.yml rename deployment/docker/victorialogs/filebeat/{ => elasticsearch}/filebeat.yml (100%) delete mode 100644 deployment/docker/victorialogs/filebeat/scrape.yml create mode 100644 deployment/docker/victorialogs/filebeat/syslog/compose.yml rename deployment/docker/victorialogs/{filebeat-syslog => filebeat/syslog}/filebeat.yml (100%) delete mode 100644 deployment/docker/victorialogs/fluentbit-ha-single-node/README.md delete mode 100644 deployment/docker/victorialogs/fluentbit-ha-single-node/auth.yml delete mode 100644 deployment/docker/victorialogs/fluentbit-ha-single-node/compose.yml create mode 100644 deployment/docker/victorialogs/fluentbit/jsonline-ha/compose.yml rename deployment/docker/victorialogs/{fluentbit-ha-single-node => fluentbit/jsonline-ha}/fluent-bit.conf (77%) create mode 100644 deployment/docker/victorialogs/fluentbit/jsonline/compose.yml create mode 100644 deployment/docker/victorialogs/fluentbit/jsonline/fluent-bit.conf create mode 100644 deployment/docker/victorialogs/fluentbit/loki/compose.yml rename deployment/docker/victorialogs/fluentbit/{ => loki}/fluent-bit.conf (54%) create mode 100644 deployment/docker/victorialogs/fluentd/Dockerfile create mode 100644 deployment/docker/victorialogs/fluentd/README.md create mode 100644 deployment/docker/victorialogs/fluentd/compose.yml create mode 100644 deployment/docker/victorialogs/fluentd/elasticsearch/compose.yml create mode 100644 deployment/docker/victorialogs/fluentd/elasticsearch/fluent.conf create mode 100644 deployment/docker/victorialogs/fluentd/jsonline/compose.yml create mode 100644 deployment/docker/victorialogs/fluentd/jsonline/fluent.conf create mode 100644 deployment/docker/victorialogs/fluentd/loki/compose.yml create mode 100644 deployment/docker/victorialogs/fluentd/loki/fluent.conf delete mode 100644 deployment/docker/victorialogs/logstash-ha-single-node/README.md delete mode 100644 deployment/docker/victorialogs/logstash-ha-single-node/auth.yml delete mode 100644 deployment/docker/victorialogs/logstash-ha-single-node/compose.yml delete mode 100644 deployment/docker/victorialogs/logstash-ha-single-node/logstash.yml create mode 100644 deployment/docker/victorialogs/logstash/elasticsearch/compose.yml rename deployment/docker/victorialogs/logstash/{ => elasticsearch}/pipeline.conf (51%) create mode 100644 deployment/docker/victorialogs/logstash/jsonline-ha/compose.yml rename deployment/docker/victorialogs/{logstash-ha-single-node => logstash/jsonline-ha}/pipeline.conf (80%) create mode 100644 deployment/docker/victorialogs/logstash/jsonline/compose.yml create mode 100644 deployment/docker/victorialogs/logstash/jsonline/pipeline.conf create mode 100644 deployment/docker/victorialogs/logstash/loki/compose.yml create mode 100644 deployment/docker/victorialogs/logstash/loki/pipeline.conf rename deployment/docker/victorialogs/opentelemetry-collector/{elasticsearch => }/README.md (66%) create mode 100644 deployment/docker/victorialogs/opentelemetry-collector/compose.yml create mode 100644 deployment/docker/victorialogs/opentelemetry-collector/elasticsearch-ha/compose.yml create mode 100644 deployment/docker/victorialogs/opentelemetry-collector/elasticsearch-ha/config.yaml delete mode 100644 deployment/docker/victorialogs/opentelemetry-collector/elasticsearch/scrape.yml delete mode 100644 deployment/docker/victorialogs/opentelemetry-collector/loki/README.md delete mode 100644 deployment/docker/victorialogs/opentelemetry-collector/loki/scrape.yml delete mode 100644 deployment/docker/victorialogs/opentelemetry-collector/otlp/README.md delete mode 100644 deployment/docker/victorialogs/opentelemetry-collector/otlp/scrape.yml delete mode 100644 deployment/docker/victorialogs/opentelemetry-collector/syslog/README.md delete mode 100644 deployment/docker/victorialogs/opentelemetry-collector/syslog/scrape.yml create mode 100644 deployment/docker/victorialogs/promtail/loki/compose.yml rename deployment/docker/victorialogs/promtail/{ => loki}/config.yml (90%) delete mode 100644 deployment/docker/victorialogs/telegraf-docker/scrape.yml rename deployment/docker/victorialogs/{telegraf-docker => telegraf}/README.md (56%) create mode 100644 deployment/docker/victorialogs/telegraf/compose.yml create mode 100644 deployment/docker/victorialogs/telegraf/elasticsearch/compose.yml rename deployment/docker/victorialogs/{telegraf-docker => telegraf/elasticsearch}/telegraf.conf (70%) create mode 100644 deployment/docker/victorialogs/telegraf/jsonline/compose.yml create mode 100644 deployment/docker/victorialogs/telegraf/jsonline/telegraf.conf create mode 100644 deployment/docker/victorialogs/telegraf/loki/compose.yml create mode 100644 deployment/docker/victorialogs/telegraf/loki/telegraf.conf create mode 100644 deployment/docker/victorialogs/telegraf/syslog/compose.yml create mode 100644 deployment/docker/victorialogs/telegraf/syslog/telegraf.conf delete mode 100644 deployment/docker/victorialogs/vector-ha-single-node/README.md delete mode 100644 deployment/docker/victorialogs/vector-ha-single-node/auth.yml delete mode 100644 deployment/docker/victorialogs/vector-ha-single-node/compose.yml delete mode 100644 deployment/docker/victorialogs/vector-ha-single-node/vector.yaml create mode 100644 deployment/docker/victorialogs/vector/elasticsearch/compose.yaml create mode 100644 deployment/docker/victorialogs/vector/elasticsearch/vector.yaml create mode 100644 deployment/docker/victorialogs/vector/jsonline-ha/compose.yml rename deployment/docker/victorialogs/vector/{ => jsonline-ha}/vector.yaml (66%) create mode 100644 deployment/docker/victorialogs/vector/jsonline/compose.yml create mode 100644 deployment/docker/victorialogs/vector/jsonline/vector.yaml create mode 100644 deployment/docker/victorialogs/vector/loki/compose.yml create mode 100644 deployment/docker/victorialogs/vector/loki/vector.yaml delete mode 100644 deployment/docker/victorialogs/vector/scrape.yml diff --git a/deployment/docker/victorialogs/compose-ha.yml b/deployment/docker/victorialogs/compose-ha.yml new file mode 100644 index 0000000000..6de4684545 --- /dev/null +++ b/deployment/docker/victorialogs/compose-ha.yml @@ -0,0 +1,4 @@ +services: + victorialogs-2: + deploy: + replicas: 1 diff --git a/deployment/docker/victorialogs/telegraf-docker/compose.yml b/deployment/docker/victorialogs/compose.yml similarity index 51% rename from deployment/docker/victorialogs/telegraf-docker/compose.yml rename to deployment/docker/victorialogs/compose.yml index 953d4fa5d9..f9ef9ed658 100644 --- a/deployment/docker/victorialogs/telegraf-docker/compose.yml +++ b/deployment/docker/victorialogs/compose.yml @@ -1,49 +1,47 @@ services: - telegraf: - image: bitnami/telegraf:1.31.0 - restart: on-failure - volumes: - - type: bind - source: /var/run/docker.sock - target: /var/run/docker.sock - - type: bind - source: /var/lib/docker - target: /var/lib/docker - - ./telegraf.conf:/etc/telegraf/telegraf.conf:ro - command: --config /etc/telegraf/telegraf.conf - depends_on: - victorialogs: - condition: service_healthy - victoriametrics: - condition: service_healthy - - victorialogs: + # meta service will be ignored by compose + .victorialogs: image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs - volumes: - - victorialogs-vector-docker-vl:/vlogs - ports: - - '9428:9428' command: - -storageDataPath=/vlogs - -loggerFormat=json - -syslog.listenAddr.tcp=0.0.0.0:8094 + deploy: + replicas: 0 healthcheck: test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9428/health"] interval: 1s timeout: 1s retries: 10 + victorialogs: + extends: .victorialogs + ports: + - '9428:9428' + volumes: + - victorialogs:/vlogs + deploy: + replicas: 1 + + # second replica is needed for HA setup and its replica count is set to 1 in compose-ha.yaml file + victorialogs-2: + extends: .victorialogs + ports: + - '9429:9428' + volumes: + - victorialogs-2:/vlogs + deploy: + replicas: 0 + victoriametrics: image: victoriametrics/victoria-metrics:latest ports: - '8428:8428' command: - -storageDataPath=/vmsingle - - -promscrape.config=/promscrape.yml - -loggerFormat=json volumes: - - victorialogs-vector-docker-vm:/vmsingle - - ./scrape.yml:/promscrape.yml + - victoriametrics:/vmsingle healthcheck: test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8428/health"] interval: 1s @@ -51,5 +49,6 @@ services: retries: 10 volumes: - victorialogs-vector-docker-vl: - victorialogs-vector-docker-vm: + victorialogs: + victorialogs-2: + victoriametrics: diff --git a/deployment/docker/victorialogs/filebeat-syslog/README.md b/deployment/docker/victorialogs/filebeat-syslog/README.md deleted file mode 100644 index 845b363ca2..0000000000 --- a/deployment/docker/victorialogs/filebeat-syslog/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# Docker compose Filebeat integration with VictoriaLogs for syslog - -The folder contains the example of integration of [filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-overview.html) with Victorialogs - -To spin-up environment run the following command: -``` -docker compose up -d -``` - -To shut down the docker-compose environment run the following command: -``` -docker compose down -docker compose rm -f -``` - -The docker compose file contains the following components: - -* filebeat - fileabeat is configured to accept `syslog` logs in `rfc3164` format on `5140` port, you can find configuration in the `filebeat.yml`. It writes data in VictoriaLogs -* VictoriaLogs - the log database, it accepts the data from `filebeat` by elastic protocol - -Querying the data - -* [vmui](https://docs.victoriametrics.com/victorialogs/querying/#vmui) - a web UI is accessible by `http://localhost:9428/select/vmui` -* for querying the data via command-line please check [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line) - -the example of filebeat configuration(`filebeat.yml`) - -```yaml -filebeat.inputs: - - type: syslog - format: rfc3164 - protocol.tcp: - host: "0.0.0.0:5140" - -output.elasticsearch: - hosts: [ "http://victorialogs:9428/insert/elasticsearch/" ] - worker: 5 - bulk_max_size: 1000 - parameters: - _msg_field: "message" - _time_field: "@timestamp" - _stream_fields: "host.name,process.program,process.pid,container.name" -``` - -Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance. diff --git a/deployment/docker/victorialogs/filebeat-syslog/compose.yml b/deployment/docker/victorialogs/filebeat-syslog/compose.yml deleted file mode 100644 index 94687c3070..0000000000 --- a/deployment/docker/victorialogs/filebeat-syslog/compose.yml +++ /dev/null @@ -1,23 +0,0 @@ -services: - filebeat-vlogs: - image: docker.elastic.co/beats/filebeat:8.8.1 - user: root - command: - - "--strict.perms=false" - volumes: - - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro - depends_on: [victorialogs] - ports: - - "5140:5140" - - victorialogs: - image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs - volumes: - - victorialogs-filebeat-syslog-vl:/vlogs - ports: - - "9428:9428" - command: - - -storageDataPath=/vlogs - -volumes: - victorialogs-filebeat-syslog-vl: diff --git a/deployment/docker/victorialogs/filebeat/README.md b/deployment/docker/victorialogs/filebeat/README.md index 200da23e98..40bb451c09 100644 --- a/deployment/docker/victorialogs/filebeat/README.md +++ b/deployment/docker/victorialogs/filebeat/README.md @@ -1,8 +1,11 @@ -# Docker compose Filebeat integration with VictoriaLogs for docker +# Docker compose Filebeat integration with VictoriaLogs using listed below protocols: + +* [syslog](./syslog) +* [elasticsearch](./elasticsearch) The folder contains the example of integration of [filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-overview.html) with Victorialogs -To spin-up environment run the following command: +To spin-up environment `cd` to any of listed above directories run the following command: ``` docker compose up -d ``` @@ -16,39 +19,16 @@ docker compose rm -f The docker compose file contains the following components: * filebeat - fileabeat is configured to collect logs from the `docker`, you can find configuration in the `filebeat.yml`. It writes data in VictoriaLogs -* filebeat-exporter - it export metrics about the filebeat * VictoriaLogs - the log database, it accepts the data from `filebeat` by elastic protocol * VictoriaMetrics - collects metrics from `filebeat` via `filebeat-exporter`, `VictoriaLogs` and `VictoriaMetrics` -* grafana - it comes with two predefined dashboards for `VictoriaLogs` and `VictoriaMetrics` Querying the data * [vmui](https://docs.victoriametrics.com/victorialogs/querying/#vmui) - a web UI is accessible by `http://localhost:9428/select/vmui` * for querying the data via command-line please check [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line) -the example of filebeat configuration(`filebeat.yml`) - -```yaml -filebeat.autodiscover: - providers: - - type: docker - hints.enabled: true - -processors: - - add_docker_metadata: ~ - -output.elasticsearch: - hosts: [ "http://victorialogs:9428/insert/elasticsearch/" ] - worker: 5 - parameters: - _msg_field: "message" - _time_field: "@timestamp" - _stream_fields: "container.name" - -http: - enabled: true - host: 0.0.0.0 - port: 5066 -``` +Filebeat configuration example can be found below: +- [syslog](./syslog/filebeat.yml) +- [elasticsearch](./elasticsearch/filebeat.yml) Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance. diff --git a/deployment/docker/victorialogs/filebeat/compose.yml b/deployment/docker/victorialogs/filebeat/compose.yml index 0dbd5a69c3..308ff363b6 100644 --- a/deployment/docker/victorialogs/filebeat/compose.yml +++ b/deployment/docker/victorialogs/filebeat/compose.yml @@ -1,6 +1,8 @@ +include: + - ../compose.yml services: filebeat-victorialogs: - image: docker.elastic.co/beats/filebeat:8.8.1 + image: docker.elastic.co/beats/filebeat:8.15.0 restart: on-failure volumes: - type: bind @@ -9,50 +11,8 @@ services: - type: bind source: /var/lib/docker/containers/ target: /var/lib/docker/containers/ - - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro + - ${PWD}/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro user: root command: - "--strict.perms=false" - depends_on: [ victorialogs ] - beat-exporter-victorialogs: - image: trustpilot/beat-exporter:0.4.0 - command: - - -beat.uri=http://filebeat-victorialogs:5066 - - victorialogs: - image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs - volumes: - - victorialogs-filebeat-docker-vl:/vlogs - ports: - - '9428:9428' - command: - - -storageDataPath=/vlogs - - -loggerFormat=json - grafana: - container_name: grafana - image: grafana/grafana:9.2.7 - depends_on: - - "victoriametrics" - ports: - - 3000:3000 - volumes: - - victorialogs-filebeat-docker-grafana:/var/lib/grafana - - ./../../provisioning/:/etc/grafana/provisioning/ - - ./../../../../dashboards/victoriametrics.json:/var/lib/grafana/dashboards/vm.json - - ./../../../../dashboards/victorialogs.json:/var/lib/grafana/dashboards/vl.json - victoriametrics: - image: victoriametrics/victoria-metrics:latest - ports: - - '8428:8428' - command: - - -storageDataPath=/vmsingle - - -promscrape.config=/promscrape.yml - - -loggerFormat=json - volumes: - - victorialogs-filebeat-docker-vm:/vmsingle - - ./scrape.yml:/promscrape.yml - -volumes: - victorialogs-filebeat-docker-vl: - victorialogs-filebeat-docker-vm: - victorialogs-filebeat-docker-grafana: + depends_on: [victorialogs] diff --git a/deployment/docker/victorialogs/filebeat/elasticsearch/compose.yml b/deployment/docker/victorialogs/filebeat/elasticsearch/compose.yml new file mode 100644 index 0000000000..f15e3b2f51 --- /dev/null +++ b/deployment/docker/victorialogs/filebeat/elasticsearch/compose.yml @@ -0,0 +1,3 @@ +include: + - ../compose.yml +name: filebeat-elasticsearch diff --git a/deployment/docker/victorialogs/filebeat/filebeat.yml b/deployment/docker/victorialogs/filebeat/elasticsearch/filebeat.yml similarity index 100% rename from deployment/docker/victorialogs/filebeat/filebeat.yml rename to deployment/docker/victorialogs/filebeat/elasticsearch/filebeat.yml diff --git a/deployment/docker/victorialogs/filebeat/scrape.yml b/deployment/docker/victorialogs/filebeat/scrape.yml deleted file mode 100644 index 5bf5f9ddff..0000000000 --- a/deployment/docker/victorialogs/filebeat/scrape.yml +++ /dev/null @@ -1,16 +0,0 @@ -scrape_configs: - - job_name: "filebeat" - scrape_interval: 30s - static_configs: - - targets: - - beat-exporter-victorialogs:9479 - - job_name: "victoriametrics" - scrape_interval: 30s - static_configs: - - targets: - - victoriametrics:8428 - - job_name: "victorialogs" - scrape_interval: 30s - static_configs: - - targets: - - victorialogs:9428 diff --git a/deployment/docker/victorialogs/filebeat/syslog/compose.yml b/deployment/docker/victorialogs/filebeat/syslog/compose.yml new file mode 100644 index 0000000000..eb908fd0c7 --- /dev/null +++ b/deployment/docker/victorialogs/filebeat/syslog/compose.yml @@ -0,0 +1,3 @@ +include: + - ../compose.yml +name: filebeat-syslog diff --git a/deployment/docker/victorialogs/filebeat-syslog/filebeat.yml b/deployment/docker/victorialogs/filebeat/syslog/filebeat.yml similarity index 100% rename from deployment/docker/victorialogs/filebeat-syslog/filebeat.yml rename to deployment/docker/victorialogs/filebeat/syslog/filebeat.yml diff --git a/deployment/docker/victorialogs/fluentbit-ha-single-node/README.md b/deployment/docker/victorialogs/fluentbit-ha-single-node/README.md deleted file mode 100644 index f845a9d6b1..0000000000 --- a/deployment/docker/victorialogs/fluentbit-ha-single-node/README.md +++ /dev/null @@ -1,88 +0,0 @@ -# Docker compose Fluentbit integration with VictoriaLogs for docker. High-Availability example - -The folder contains the example of integration of [fluentbit](https://docs.fluentbit.io/manual) with VictoriaLogs Single-Nodes(s) and [vmauth](https://docs.victoriametrics.com/vmauth/) for achieving High Availability. - -Check [this documentation](https://docs.victoriametrics.com/victorialogs/#high-availability) with a description of the architecture and components. - -To spin-up environment run the following command: - -```shell -docker compose up -d -``` - -To shut down the docker-compose environment run the following command: - -```shell -docker compose down -docker compose rm -f -``` - -The docker compose file contains the following components: - -* fluentbit - fluentbit is configured to collect logs from the `docker`, you can find configuration in the `fluent-bit.conf`. It writes data in VictoriaLogs -* VictoriaLogs - the two instances of log database, they accept the data from `fluentbit` by json line protocol -* vmauth - load balancer for proxying requests to one of VictoriaLogs - -Querying the data - -* [vmui](https://docs.victoriametrics.com/victorialogs/querying/#vmui) - a web UI is accessible by `http://localhost:8427/select/vmui/` -* for querying the data via command-line please check [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line) - - -the example of fluentbit configuration(`fluent-bit.conf`) - -```text -[INPUT] - name tail - path /var/lib/docker/containers/**/*.log - path_key path - multiline.parser docker, cri - Parser docker - Docker_Mode On - -[INPUT] - Name syslog - Listen 0.0.0.0 - Port 5140 - Parser syslog-rfc3164 - Mode tcp - -[SERVICE] - Flush 1 - Parsers_File parsers.conf - -[OUTPUT] - Name http - Match * - host victorialogs-2 - port 9428 - compress gzip - uri /insert/jsonline?_stream_fields=stream,path&_msg_field=log&_time_field=date - format json_lines - json_date_format iso8601 - header AccountID 0 - header ProjectID 0 - -[OUTPUT] - Name http - Match * - host victorialogs-1 - port 9428 - compress gzip - uri /insert/jsonline?_stream_fields=stream,path&_msg_field=log&_time_field=date - format json_lines - json_date_format iso8601 - header AccountID 0 - header ProjectID 0 -``` - -Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance. - -The example of vmauth configuration (`auth.yml`) - -```yaml -unauthorized_user: - url_prefix: - - http://victorialogs-1:9428 - - http://victorialogs-2:9428 -``` diff --git a/deployment/docker/victorialogs/fluentbit-ha-single-node/auth.yml b/deployment/docker/victorialogs/fluentbit-ha-single-node/auth.yml deleted file mode 100644 index eeafbb852b..0000000000 --- a/deployment/docker/victorialogs/fluentbit-ha-single-node/auth.yml +++ /dev/null @@ -1,6 +0,0 @@ -# balance load among victorialogs instances -# see https://docs.victoriametrics.com/vmauth/#load-balancing -unauthorized_user: - url_prefix: - - http://victorialogs-1:9428 - - http://victorialogs-2:9428 diff --git a/deployment/docker/victorialogs/fluentbit-ha-single-node/compose.yml b/deployment/docker/victorialogs/fluentbit-ha-single-node/compose.yml deleted file mode 100644 index 737be03470..0000000000 --- a/deployment/docker/victorialogs/fluentbit-ha-single-node/compose.yml +++ /dev/null @@ -1,38 +0,0 @@ -services: - fluentbit: - image: cr.fluentbit.io/fluent/fluent-bit:3.0.7 - volumes: - - /var/lib/docker/containers:/var/lib/docker/containers:ro - - ./fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf - depends_on: [victorialogs-1,victorialogs-2] - ports: - - "5140:5140" - - victorialogs-1: - image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs - volumes: - - victorialogs-fluentbit-vl-ha-single-1:/vlogs - command: - - -storageDataPath=/vlogs - victorialogs-2: - image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs - volumes: - - victorialogs-fluentbit-vl-ha-single-2:/vlogs - command: - - -storageDataPath=/vlogs - vmauth: - container_name: vmauth - image: victoriametrics/vmauth:v1.103.0 - depends_on: - - "victorialogs-1" - - "victorialogs-2" - volumes: - - ./auth.yml:/etc/auth.yml - command: - - '--auth.config=/etc/auth.yml' - ports: - - 8427:8427 - restart: always -volumes: - victorialogs-fluentbit-vl-ha-single-1: - victorialogs-fluentbit-vl-ha-single-2: diff --git a/deployment/docker/victorialogs/fluentbit/README.md b/deployment/docker/victorialogs/fluentbit/README.md index 1fce9d455a..5a8f51709d 100644 --- a/deployment/docker/victorialogs/fluentbit/README.md +++ b/deployment/docker/victorialogs/fluentbit/README.md @@ -1,8 +1,12 @@ -# Docker compose Fluentbit integration with VictoriaLogs for docker +# Docker compose Fluentbit integration with VictoriaLogs using given below protocols: + +* [loki](./loki) +* [jsonline single node](./jsonline) +* [jsonline HA setup](./jsonline-ha) The folder contains the example of integration of [fluentbit](https://docs.fluentbit.io/manual) with Victorialogs -To spin-up environment run the following command: +To spin-up environment `cd` to any of listed above directories run the following command: ``` docker compose up -d ``` @@ -23,39 +27,9 @@ Querying the data * [vmui](https://docs.victoriametrics.com/victorialogs/querying/#vmui) - a web UI is accessible by `http://localhost:9428/select/vmui` * for querying the data via command-line please check [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line) - -the example of fluentbit configuration(`fluent-bit.conf`) - -```shell -[INPUT] - name tail - path /var/lib/docker/containers/**/*.log - multiline.parser docker, cri - Parser docker - Docker_Mode On - -[INPUT] - Name syslog - Listen 0.0.0.0 - Port 5140 - Parser syslog-rfc3164 - Mode tcp - -[SERVICE] - Flush 1 - Parsers_File parsers.conf - -[Output] - Name http - Match * - host victorialogs - port 9428 - compress gzip - uri /insert/jsonline?_stream_fields=stream&_msg_field=log&_time_field=date - format json_lines - json_date_format iso8601 - header AccountID 0 - header ProjectID 0 -``` +FluentBit configuration example can be found below: +* [loki](./loki/fluent-bit.conf) +* [jsonline single node](./jsonline/fluent-bit.conf) +* [jsonline HA setup](./jsonline-ha/fluent-bit.conf) Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance. diff --git a/deployment/docker/victorialogs/fluentbit/compose.yml b/deployment/docker/victorialogs/fluentbit/compose.yml index ea9b4a7bdf..2f38c9f206 100644 --- a/deployment/docker/victorialogs/fluentbit/compose.yml +++ b/deployment/docker/victorialogs/fluentbit/compose.yml @@ -1,21 +1,11 @@ +include: + - ../compose.yml services: fluentbit: - image: cr.fluentbit.io/fluent/fluent-bit:3.0.7 + image: cr.fluentbit.io/fluent/fluent-bit:3.1.7 volumes: - /var/lib/docker/containers:/var/lib/docker/containers:ro - - ./fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf + - ${PWD}/fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf depends_on: [victorialogs] ports: - "5140:5140" - - victorialogs: - image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs - volumes: - - victorialogs-fluentbit-vl:/vlogs - ports: - - "9428:9428" - command: - - -storageDataPath=/vlogs - -volumes: - victorialogs-fluentbit-vl: diff --git a/deployment/docker/victorialogs/fluentbit/jsonline-ha/compose.yml b/deployment/docker/victorialogs/fluentbit/jsonline-ha/compose.yml new file mode 100644 index 0000000000..ffe557c614 --- /dev/null +++ b/deployment/docker/victorialogs/fluentbit/jsonline-ha/compose.yml @@ -0,0 +1,5 @@ +include: + - path: + - ../compose.yml + - ../../compose-ha.yml +name: fluentbit-jsonline-ha diff --git a/deployment/docker/victorialogs/fluentbit-ha-single-node/fluent-bit.conf b/deployment/docker/victorialogs/fluentbit/jsonline-ha/fluent-bit.conf similarity index 77% rename from deployment/docker/victorialogs/fluentbit-ha-single-node/fluent-bit.conf rename to deployment/docker/victorialogs/fluentbit/jsonline-ha/fluent-bit.conf index ce6423898f..fb439860c7 100644 --- a/deployment/docker/victorialogs/fluentbit-ha-single-node/fluent-bit.conf +++ b/deployment/docker/victorialogs/fluentbit/jsonline-ha/fluent-bit.conf @@ -1,10 +1,10 @@ [INPUT] - name tail - path /var/lib/docker/containers/**/*.log + name tail + path /var/lib/docker/containers/**/*.log path_key path - multiline.parser docker, cri - Parser docker - Docker_Mode On + multiline.parser docker, cri + Parser docker + Docker_Mode On [INPUT] Name syslog @@ -17,7 +17,19 @@ Flush 1 Parsers_File parsers.conf -[OUTPUT] +[Output] + Name http + Match * + host victorialogs + port 9428 + compress gzip + uri /insert/jsonline?_stream_fields=stream,path&_msg_field=log&_time_field=date + format json_lines + json_date_format iso8601 + header AccountID 0 + header ProjectID 0 + +[Output] Name http Match * host victorialogs-2 @@ -28,15 +40,3 @@ json_date_format iso8601 header AccountID 0 header ProjectID 0 - -[OUTPUT] - Name http - Match * - host victorialogs-1 - port 9428 - compress gzip - uri /insert/jsonline?_stream_fields=stream,path&_msg_field=log&_time_field=date - format json_lines - json_date_format iso8601 - header AccountID 0 - header ProjectID 0 diff --git a/deployment/docker/victorialogs/fluentbit/jsonline/compose.yml b/deployment/docker/victorialogs/fluentbit/jsonline/compose.yml new file mode 100644 index 0000000000..4edd0c18b7 --- /dev/null +++ b/deployment/docker/victorialogs/fluentbit/jsonline/compose.yml @@ -0,0 +1,3 @@ +include: + - ../compose.yml +name: fluentbit-jsonline diff --git a/deployment/docker/victorialogs/fluentbit/jsonline/fluent-bit.conf b/deployment/docker/victorialogs/fluentbit/jsonline/fluent-bit.conf new file mode 100644 index 0000000000..42d349338b --- /dev/null +++ b/deployment/docker/victorialogs/fluentbit/jsonline/fluent-bit.conf @@ -0,0 +1,30 @@ +[INPUT] + name tail + path /var/lib/docker/containers/**/*.log + path_key path + multiline.parser docker, cri + Parser docker + Docker_Mode On + +[INPUT] + Name syslog + Listen 0.0.0.0 + Port 5140 + Parser syslog-rfc3164 + Mode tcp + +[SERVICE] + Flush 1 + Parsers_File parsers.conf + +[Output] + Name http + Match * + host victorialogs + port 9428 + compress gzip + uri /insert/jsonline?_stream_fields=stream,path&_msg_field=log&_time_field=date + format json_lines + json_date_format iso8601 + header AccountID 0 + header ProjectID 0 diff --git a/deployment/docker/victorialogs/fluentbit/loki/compose.yml b/deployment/docker/victorialogs/fluentbit/loki/compose.yml new file mode 100644 index 0000000000..1627ba0435 --- /dev/null +++ b/deployment/docker/victorialogs/fluentbit/loki/compose.yml @@ -0,0 +1,3 @@ +include: + - ../compose.yml +name: fluentbit-loki diff --git a/deployment/docker/victorialogs/fluentbit/fluent-bit.conf b/deployment/docker/victorialogs/fluentbit/loki/fluent-bit.conf similarity index 54% rename from deployment/docker/victorialogs/fluentbit/fluent-bit.conf rename to deployment/docker/victorialogs/fluentbit/loki/fluent-bit.conf index 0c78d93646..6c7906d0db 100644 --- a/deployment/docker/victorialogs/fluentbit/fluent-bit.conf +++ b/deployment/docker/victorialogs/fluentbit/loki/fluent-bit.conf @@ -17,31 +17,6 @@ Flush 1 Parsers_File parsers.conf -[OUTPUT] - Name http - Match * - host victorialogs - port 9428 - compress gzip - uri /insert/jsonline?_stream_fields=stream,path&_msg_field=log&_time_field=date - format json_lines - json_date_format iso8601 - header AccountID 0 - header ProjectID 0 - -[OUTPUT] - Name es - Match * - host victorialogs - port 9428 - compress gzip - path /insert/elasticsearch - header AccountID 0 - header ProjectID 0 - header VL-Stream-Fields path - header VL-Msg-Field log - header VL-Time-Field @timestamp - [OUTPUT] name loki match * diff --git a/deployment/docker/victorialogs/fluentd/Dockerfile b/deployment/docker/victorialogs/fluentd/Dockerfile new file mode 100644 index 0000000000..b3bba2e0f6 --- /dev/null +++ b/deployment/docker/victorialogs/fluentd/Dockerfile @@ -0,0 +1,8 @@ +FROM fluent/fluentd:v1.17 +USER root +RUN \ + gem install \ + fluent-plugin-datadog \ + fluent-plugin-grafana-loki \ + fluent-plugin-elasticsearch +USER fluent diff --git a/deployment/docker/victorialogs/fluentd/README.md b/deployment/docker/victorialogs/fluentd/README.md new file mode 100644 index 0000000000..0d7e422982 --- /dev/null +++ b/deployment/docker/victorialogs/fluentd/README.md @@ -0,0 +1,35 @@ +# Docker compose Fluentd integration with VictoriaLogs using given below protocols: + +* [loki](./loki) +* [jsonline](./jsonline) +* [elasticsearch](./elasticsearch) + +The folder contains the example of integration of [fluentd](https://www.fluentd.org/) with Victorialogs + +To spin-up environment `cd` to any of listed above directories run the following command: +``` +docker compose up -d +``` + +To shut down the docker-compose environment run the following command: +``` +docker compose down +docker compose rm -f +``` + +The docker compose file contains the following components: + +* fluentd - fluentd is configured to collect logs from the `docker`, you can find configuration in the `fluent-bit.conf`. It writes data in VictoriaLogs +* VictoriaLogs - the log database, it accepts the data from `fluentd` by json line protocol + +Querying the data + +* [vmui](https://docs.victoriametrics.com/victorialogs/querying/#vmui) - a web UI is accessible by `http://localhost:9428/select/vmui` +* for querying the data via command-line please check [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line) + +Fluentd configuration example can be found below: +* [loki](./loki/fluent.conf) +* [jsonline](./jsonline/fluent.conf) +* [elasticsearch](./elasticsearch/fluent.conf) + +Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance. diff --git a/deployment/docker/victorialogs/fluentd/compose.yml b/deployment/docker/victorialogs/fluentd/compose.yml new file mode 100644 index 0000000000..6a3e33d35a --- /dev/null +++ b/deployment/docker/victorialogs/fluentd/compose.yml @@ -0,0 +1,11 @@ +include: + - ../compose.yml +services: + fluentd: + build: . + volumes: + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - ${PWD}/fluent.conf:/fluentd/etc/fluent.conf + depends_on: [victorialogs] + ports: + - "5140:5140" diff --git a/deployment/docker/victorialogs/fluentd/elasticsearch/compose.yml b/deployment/docker/victorialogs/fluentd/elasticsearch/compose.yml new file mode 100644 index 0000000000..9c65b063e7 --- /dev/null +++ b/deployment/docker/victorialogs/fluentd/elasticsearch/compose.yml @@ -0,0 +1,3 @@ +include: + - ../compose.yml +name: fluentd-elasticsearch diff --git a/deployment/docker/victorialogs/fluentd/elasticsearch/fluent.conf b/deployment/docker/victorialogs/fluentd/elasticsearch/fluent.conf new file mode 100644 index 0000000000..145c4d32d3 --- /dev/null +++ b/deployment/docker/victorialogs/fluentd/elasticsearch/fluent.conf @@ -0,0 +1,13 @@ + + @type tail + format none + tag docker.testlog + path /var/lib/docker/containers/**/*.log + + + + @type elasticsearch + host victorialogs + path /insert/elasticsearch + port 9428 + diff --git a/deployment/docker/victorialogs/fluentd/jsonline/compose.yml b/deployment/docker/victorialogs/fluentd/jsonline/compose.yml new file mode 100644 index 0000000000..efe3cb8256 --- /dev/null +++ b/deployment/docker/victorialogs/fluentd/jsonline/compose.yml @@ -0,0 +1,3 @@ +include: + - ../compose.yml +name: fluentd-jsonline diff --git a/deployment/docker/victorialogs/fluentd/jsonline/fluent.conf b/deployment/docker/victorialogs/fluentd/jsonline/fluent.conf new file mode 100644 index 0000000000..cac841e00b --- /dev/null +++ b/deployment/docker/victorialogs/fluentd/jsonline/fluent.conf @@ -0,0 +1,12 @@ + + @type tail + format none + tag docker.testlog + path /var/lib/docker/containers/**/*.log + + + + @type http + endpoint "http://victorialogs:9428/insert/jsonline" + headers {"VL-Msg-Field": "log", "VL-Time-Field": "time", "VL-Stream-Fields": "path"} + diff --git a/deployment/docker/victorialogs/fluentd/loki/compose.yml b/deployment/docker/victorialogs/fluentd/loki/compose.yml new file mode 100644 index 0000000000..1627ba0435 --- /dev/null +++ b/deployment/docker/victorialogs/fluentd/loki/compose.yml @@ -0,0 +1,3 @@ +include: + - ../compose.yml +name: fluentbit-loki diff --git a/deployment/docker/victorialogs/fluentd/loki/fluent.conf b/deployment/docker/victorialogs/fluentd/loki/fluent.conf new file mode 100644 index 0000000000..4604376737 --- /dev/null +++ b/deployment/docker/victorialogs/fluentd/loki/fluent.conf @@ -0,0 +1,17 @@ + + @type tail + format none + tag docker.testlog + path /var/lib/docker/containers/**/*.log + + + + @type loki + url "http://victorialogs:9428/insert" + + flush_interval 10s + flush_at_shutdown true + + custom_headers {"VL-Msg-Field": "log", "VL-Time-Field": "time", "VL-Stream-Fields": "path"} + buffer_chunk_limit 1m + diff --git a/deployment/docker/victorialogs/logstash-ha-single-node/README.md b/deployment/docker/victorialogs/logstash-ha-single-node/README.md deleted file mode 100644 index 7e5805a214..0000000000 --- a/deployment/docker/victorialogs/logstash-ha-single-node/README.md +++ /dev/null @@ -1,71 +0,0 @@ -# Docker compose Logstash integration with VictoriaLogs for docker. High-Availability example - -The folder contains the example of integration of [logstash](https://www.elastic.co/logstash) with VictoriaLogs Single-Node(s) and [vmauth](https://docs.victoriametrics.com/vmauth/) for achieving High Availability. - -Check [this documentation](https://docs.victoriametrics.com/victorialogs/#high-availability) with a description of the architecture and components. - -To spin-up environment run the following command: - -```shell -docker compose up -d -``` - -To shut down the docker-compose environment run the following command: - -```shell -docker compose down -docker compose rm -f -``` - -The docker compose file contains the following components: - -* logstash - logstash is configured to read docker log files, you can find configuration in the `pipeline.conf`. It writes data in two instances of VictoriaLogs -* VictoriaLogs - the two instances of log database, they accept the data from `fluentbit` by json line protocol -* vmauth - load balancer for proxying requests to one of VictoriaLogs - -Querying the data - -* [vmui](https://docs.victoriametrics.com/victorialogs/querying/#vmui) - a web UI is accessible by `http://localhost:8427/select/vmui/` -* for querying the data via command-line please check [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line) - - -Here is an example of logstash configuration(`pipeline.conf`): - -```text -input { - file { - path => "/var/lib/docker/containers/*/*.log" - start_position => "beginning" - type => "docker" - sincedb_path => "/dev/null" - codec => "json" - add_field => { - "path" => "%{[@metadata][path]}" - } - } -} - -output { - http { - url => "http://victorialogs-1:9428/insert/jsonline?_stream_fields=host.name,stream&_msg_field=log&_time_field=time" - format => "json" - http_method => "post" - } - http { - url => "http://victorialogs-2:9428/insert/jsonline?_stream_fields=host.name,stream&_msg_field=log&_time_field=time" - format => "json" - http_method => "post" - } -} -``` - -Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance. - -The example of vmauth configuration (`auth.yml`) - -```yaml -unauthorized_user: - url_prefix: - - http://victorialogs-1:9428 - - http://victorialogs-2:9428 -``` \ No newline at end of file diff --git a/deployment/docker/victorialogs/logstash-ha-single-node/auth.yml b/deployment/docker/victorialogs/logstash-ha-single-node/auth.yml deleted file mode 100644 index eeafbb852b..0000000000 --- a/deployment/docker/victorialogs/logstash-ha-single-node/auth.yml +++ /dev/null @@ -1,6 +0,0 @@ -# balance load among victorialogs instances -# see https://docs.victoriametrics.com/vmauth/#load-balancing -unauthorized_user: - url_prefix: - - http://victorialogs-1:9428 - - http://victorialogs-2:9428 diff --git a/deployment/docker/victorialogs/logstash-ha-single-node/compose.yml b/deployment/docker/victorialogs/logstash-ha-single-node/compose.yml deleted file mode 100644 index 6b8153b716..0000000000 --- a/deployment/docker/victorialogs/logstash-ha-single-node/compose.yml +++ /dev/null @@ -1,41 +0,0 @@ -services: - logstash: - image: docker.elastic.co/logstash/logstash:8.8.1 - user: root - volumes: - - ./pipeline.conf:/usr/share/logstash/pipeline/logstash.conf:ro - - ./logstash.yml:/usr/share/logstash/config/logstash.yml:ro - - /var/lib/docker/containers:/var/lib/docker/containers:ro - depends_on: [victorialogs-1,victorialogs-2] - ports: - - "5140:5140" - - victorialogs-1: - image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs - volumes: - - victorialogs-logstash-vl-ha-single-1:/vlogs - command: - - -storageDataPath=/vlogs - victorialogs-2: - image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs - volumes: - - victorialogs-logstash-vl-ha-single-2:/vlogs - command: - - -storageDataPath=/vlogs - vmauth: - container_name: vmauth - image: victoriametrics/vmauth:v1.103.0 - depends_on: - - "victorialogs-1" - - "victorialogs-2" - volumes: - - ./auth.yml:/etc/auth.yml - command: - - '--auth.config=/etc/auth.yml' - ports: - - 8427:8427 - restart: always - -volumes: - victorialogs-logstash-vl-ha-single-1: - victorialogs-logstash-vl-ha-single-2: diff --git a/deployment/docker/victorialogs/logstash-ha-single-node/logstash.yml b/deployment/docker/victorialogs/logstash-ha-single-node/logstash.yml deleted file mode 100644 index c223de3763..0000000000 --- a/deployment/docker/victorialogs/logstash-ha-single-node/logstash.yml +++ /dev/null @@ -1,2 +0,0 @@ -http.host: 0.0.0.0 -xpack.monitoring.enabled: false \ No newline at end of file diff --git a/deployment/docker/victorialogs/logstash/Dockerfile b/deployment/docker/victorialogs/logstash/Dockerfile index a6e809fd16..5b92d20733 100644 --- a/deployment/docker/victorialogs/logstash/Dockerfile +++ b/deployment/docker/victorialogs/logstash/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.elastic.co/logstash/logstash:8.8.1 +FROM docker.elastic.co/logstash/logstash:8.15.0 RUN bin/logstash-plugin install \ logstash-output-opensearch \ diff --git a/deployment/docker/victorialogs/logstash/README.md b/deployment/docker/victorialogs/logstash/README.md index b1e9cb5d81..de39545fa5 100644 --- a/deployment/docker/victorialogs/logstash/README.md +++ b/deployment/docker/victorialogs/logstash/README.md @@ -1,4 +1,9 @@ -# Docker compose Logstash integration with VictoriaLogs for syslog +# Docker compose Logstash integration with VictoriaLogs for given below protocols: + +* [loki](./loki) +* [jsonline single node](./jsonline) +* [jsonline HA setup](./jsonline-ha) +* [elasticsearch](./elasticsearch) It is required to use [OpenSearch plugin](https://github.com/opensearch-project/logstash-output-opensearch) for output configuration. Plugin can be installed by using the following command: @@ -7,7 +12,7 @@ bin/logstash-plugin install logstash-output-opensearch ``` OpenSearch plugin is required because elasticsearch output plugin performs various checks for Elasticsearch version and license which are not applicable for VictoriaLogs. -To spin-up environment run the following command: +To spin-up environment `cd` to any of listed above directories run the following command: ``` docker compose up -d ``` @@ -28,29 +33,10 @@ Querying the data * [vmui](https://docs.victoriametrics.com/victorialogs/querying/#vmui) - a web UI is accessible by `http://localhost:9428/select/vmui` * for querying the data via command-line please check [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line) - -Here is an example of logstash configuration(`pipeline.conf`): - -``` -input { - syslog { - port => 5140 - } -} -output { - opensearch { - hosts => ["http://victorialogs:9428/insert/elasticsearch"] - custom_headers => { - "AccountID" => "0" - "ProjectID" => "0" - } - parameters => { - "_stream_fields" => "host.ip,process.name" - "_msg_field" => "message" - "_time_field" => "@timestamp" - } - } -} -``` +Logstash configuration example can be found below: +* [loki](./loki/pipeline.conf) +* [jsonline single node](./jsonline/pipeline.conf) +* [jsonline HA setup](./jsonline-ha/pipeline.conf) +* [elasticsearch](./elasticsearch/pipeline.conf) Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance. diff --git a/deployment/docker/victorialogs/logstash/compose.yml b/deployment/docker/victorialogs/logstash/compose.yml index 2523140f36..4263a392cd 100644 --- a/deployment/docker/victorialogs/logstash/compose.yml +++ b/deployment/docker/victorialogs/logstash/compose.yml @@ -1,24 +1,15 @@ +include: + - ../compose.yml services: logstash: build: context: . - dockerfile: Dockerfile + user: root restart: on-failure volumes: - - ./pipeline.conf:/usr/share/logstash/pipeline/logstash.conf:ro + - /var/lib/docker/containers:/var/lib/docker/containers + - ${PWD}/pipeline.conf:/usr/share/logstash/pipeline/logstash.conf:ro - ./logstash.yml:/usr/share/logstash/config/logstash.yml:ro depends_on: [victorialogs] ports: - "5140:5140" - - victorialogs: - image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs - volumes: - - victorialogs-logstash-vl:/vlogs - ports: - - "9428:9428" - command: - - -storageDataPath=/vlogs - -volumes: - victorialogs-logstash-vl: diff --git a/deployment/docker/victorialogs/logstash/elasticsearch/compose.yml b/deployment/docker/victorialogs/logstash/elasticsearch/compose.yml new file mode 100644 index 0000000000..79399ecc2c --- /dev/null +++ b/deployment/docker/victorialogs/logstash/elasticsearch/compose.yml @@ -0,0 +1,3 @@ +include: + - ../compose.yml +name: logstash-elasticsearch diff --git a/deployment/docker/victorialogs/logstash/pipeline.conf b/deployment/docker/victorialogs/logstash/elasticsearch/pipeline.conf similarity index 51% rename from deployment/docker/victorialogs/logstash/pipeline.conf rename to deployment/docker/victorialogs/logstash/elasticsearch/pipeline.conf index ab6cd901de..f9b3f18c4a 100644 --- a/deployment/docker/victorialogs/logstash/pipeline.conf +++ b/deployment/docker/victorialogs/logstash/elasticsearch/pipeline.conf @@ -17,12 +17,4 @@ output { "_time_field" => "@timestamp" } } - http { - url => "http://victorialogs:9428/insert/jsonline?_stream_fields=host.ip,process.name&_msg_field=message&_time_field=@timestamp" - format => "json" - http_method => "post" - } - loki { - url => "http://victorialogs:9428/insert/loki/api/v1/push?_stream_fields=host.ip,process.name&_msg_field=message&_time_field=@timestamp" - } } diff --git a/deployment/docker/victorialogs/logstash/jsonline-ha/compose.yml b/deployment/docker/victorialogs/logstash/jsonline-ha/compose.yml new file mode 100644 index 0000000000..d63fa78c08 --- /dev/null +++ b/deployment/docker/victorialogs/logstash/jsonline-ha/compose.yml @@ -0,0 +1,5 @@ +include: + - path: + - ../compose.yml + - ../../compose-ha.yml +name: logstash-jsonline-ha diff --git a/deployment/docker/victorialogs/logstash-ha-single-node/pipeline.conf b/deployment/docker/victorialogs/logstash/jsonline-ha/pipeline.conf similarity index 80% rename from deployment/docker/victorialogs/logstash-ha-single-node/pipeline.conf rename to deployment/docker/victorialogs/logstash/jsonline-ha/pipeline.conf index 7dacc1d8aa..1d731325c0 100644 --- a/deployment/docker/victorialogs/logstash-ha-single-node/pipeline.conf +++ b/deployment/docker/victorialogs/logstash/jsonline-ha/pipeline.conf @@ -13,7 +13,7 @@ input { output { http { - url => "http://victorialogs-1:9428/insert/jsonline?_stream_fields=host.name,stream&_msg_field=log&_time_field=time" + url => "http://victorialogs:9428/insert/jsonline?_stream_fields=host.name,stream&_msg_field=log&_time_field=time" format => "json" http_method => "post" } @@ -22,4 +22,4 @@ output { format => "json" http_method => "post" } -} \ No newline at end of file +} diff --git a/deployment/docker/victorialogs/logstash/jsonline/compose.yml b/deployment/docker/victorialogs/logstash/jsonline/compose.yml new file mode 100644 index 0000000000..11d4a05487 --- /dev/null +++ b/deployment/docker/victorialogs/logstash/jsonline/compose.yml @@ -0,0 +1,3 @@ +include: + - ../compose.yml +name: logstash-jsonline diff --git a/deployment/docker/victorialogs/logstash/jsonline/pipeline.conf b/deployment/docker/victorialogs/logstash/jsonline/pipeline.conf new file mode 100644 index 0000000000..b399a6fb22 --- /dev/null +++ b/deployment/docker/victorialogs/logstash/jsonline/pipeline.conf @@ -0,0 +1,13 @@ +input { + syslog { + port => 5140 + } +} + +output { + http { + url => "http://victorialogs:9428/insert/jsonline?_stream_fields=host.ip,process.name&_msg_field=message&_time_field=@timestamp" + format => "json" + http_method => "post" + } +} diff --git a/deployment/docker/victorialogs/logstash/loki/compose.yml b/deployment/docker/victorialogs/logstash/loki/compose.yml new file mode 100644 index 0000000000..9e3249883d --- /dev/null +++ b/deployment/docker/victorialogs/logstash/loki/compose.yml @@ -0,0 +1,3 @@ +include: + - ../compose.yml +name: logstash-loki diff --git a/deployment/docker/victorialogs/logstash/loki/pipeline.conf b/deployment/docker/victorialogs/logstash/loki/pipeline.conf new file mode 100644 index 0000000000..c5dafc41ea --- /dev/null +++ b/deployment/docker/victorialogs/logstash/loki/pipeline.conf @@ -0,0 +1,11 @@ +input { + syslog { + port => 5140 + } +} + +output { + loki { + url => "http://victorialogs:9428/insert/loki/api/v1/push?_stream_fields=host.ip,process.name&_msg_field=message&_time_field=@timestamp" + } +} diff --git a/deployment/docker/victorialogs/opentelemetry-collector/elasticsearch/README.md b/deployment/docker/victorialogs/opentelemetry-collector/README.md similarity index 66% rename from deployment/docker/victorialogs/opentelemetry-collector/elasticsearch/README.md rename to deployment/docker/victorialogs/opentelemetry-collector/README.md index 30cfc3f2c0..6dab25daf2 100644 --- a/deployment/docker/victorialogs/opentelemetry-collector/elasticsearch/README.md +++ b/deployment/docker/victorialogs/opentelemetry-collector/README.md @@ -1,8 +1,14 @@ -# Docker compose OpenTelemetry Elasticsearch integration with VictoriaLogs for docker +# Docker compose OpenTelemetry integration with VictoriaLogs using protocols: + +* [loki](./loki) +* [otlp](./otlp) +* [syslog](./syslog) +* [elasticsearch single node](./elasticsearch) +* [elasticsearch HA mode](./elasticsearch-ha/) The folder contains the example of integration of [OpenTelemetry collector](https://opentelemetry.io/docs/collector/) with Victorialogs -To spin-up environment run the following command: +To spin-up environment `cd` to any of listed above directories run the following command: ``` docker compose up -d ``` @@ -24,4 +30,11 @@ Querying the data * [vmui](https://docs.victoriametrics.com/victorialogs/querying/#vmui) - a web UI is accessible by `http://localhost:9428/select/vmui` * for querying the data via command-line please check [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line) +OpenTelemetry collector configuration example can be found below: +* [loki](./loki/config.yaml) +* [otlp](./otlp/config.yaml) +* [syslog](./syslog/config.yaml) +* [elasticsearch single node](./elasticsearch/config.yaml) +* [elasticsearch HA mode](./elasticsearch-ha/config.yaml) + Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance. diff --git a/deployment/docker/victorialogs/opentelemetry-collector/compose.yml b/deployment/docker/victorialogs/opentelemetry-collector/compose.yml new file mode 100644 index 0000000000..794f2382bb --- /dev/null +++ b/deployment/docker/victorialogs/opentelemetry-collector/compose.yml @@ -0,0 +1,15 @@ +include: + - ../compose.yml +services: + collector: + image: docker.io/otel/opentelemetry-collector-contrib:0.108.0 + restart: on-failure + user: 0:0 + volumes: + - ${PWD}/config.yaml:/etc/otelcol-contrib/config.yaml + - /var/lib/docker/containers:/var/lib/docker/containers + depends_on: + victorialogs: + condition: service_healthy + victoriametrics: + condition: service_healthy diff --git a/deployment/docker/victorialogs/opentelemetry-collector/elasticsearch-ha/compose.yml b/deployment/docker/victorialogs/opentelemetry-collector/elasticsearch-ha/compose.yml new file mode 100644 index 0000000000..6fc7eafd61 --- /dev/null +++ b/deployment/docker/victorialogs/opentelemetry-collector/elasticsearch-ha/compose.yml @@ -0,0 +1,5 @@ +include: + - path: + - ../compose.yml + - ../../compose-ha.yml +name: collector-elasticsearch-ha diff --git a/deployment/docker/victorialogs/opentelemetry-collector/elasticsearch-ha/config.yaml b/deployment/docker/victorialogs/opentelemetry-collector/elasticsearch-ha/config.yaml new file mode 100644 index 0000000000..46b6073a0d --- /dev/null +++ b/deployment/docker/victorialogs/opentelemetry-collector/elasticsearch-ha/config.yaml @@ -0,0 +1,15 @@ +exporters: + elasticsearch: + endpoints: + - http://victorialogs:9428/insert/elasticsearch + - http://victorialogs-2:9428/insert/elasticsearch +receivers: + filelog: + include: [/var/lib/docker/containers/**/*.log] + resource: + region: us-east-1 +service: + pipelines: + logs: + receivers: [filelog] + exporters: [elasticsearch] diff --git a/deployment/docker/victorialogs/opentelemetry-collector/elasticsearch/compose.yml b/deployment/docker/victorialogs/opentelemetry-collector/elasticsearch/compose.yml index 5cb5040a60..6d8c4be2df 100644 --- a/deployment/docker/victorialogs/opentelemetry-collector/elasticsearch/compose.yml +++ b/deployment/docker/victorialogs/opentelemetry-collector/elasticsearch/compose.yml @@ -1,48 +1,3 @@ -services: - collector: - image: docker.io/otel/opentelemetry-collector-contrib:0.102.1 - restart: on-failure - volumes: - - $PWD/logs:/tmp/logs - - $PWD/config.yaml:/etc/otelcol-contrib/config.yaml - depends_on: - victorialogs: - condition: service_healthy - victoriametrics: - condition: service_healthy - - victorialogs: - image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs - volumes: - - victorialogs-vector-docker-vl:/vlogs - ports: - - '9428:9428' - command: - - -storageDataPath=/vlogs - - -loggerFormat=json - healthcheck: - test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9428/health"] - interval: 1s - timeout: 1s - retries: 10 - - victoriametrics: - image: victoriametrics/victoria-metrics:latest - ports: - - '8428:8428' - command: - - -storageDataPath=/vmsingle - - -promscrape.config=/promscrape.yml - - -loggerFormat=json - volumes: - - victorialogs-vector-docker-vm:/vmsingle - - ./scrape.yml:/promscrape.yml - healthcheck: - test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8428/health"] - interval: 1s - timeout: 1s - retries: 10 - -volumes: - victorialogs-vector-docker-vl: - victorialogs-vector-docker-vm: +include: + - ../compose.yml +name: collector-elasticsearch diff --git a/deployment/docker/victorialogs/opentelemetry-collector/elasticsearch/config.yaml b/deployment/docker/victorialogs/opentelemetry-collector/elasticsearch/config.yaml index 84f7deea14..0725251758 100644 --- a/deployment/docker/victorialogs/opentelemetry-collector/elasticsearch/config.yaml +++ b/deployment/docker/victorialogs/opentelemetry-collector/elasticsearch/config.yaml @@ -4,7 +4,7 @@ exporters: - http://victorialogs:9428/insert/elasticsearch receivers: filelog: - include: [/tmp/logs/*.log] + include: [/var/lib/docker/containers/**/*.log] resource: region: us-east-1 service: diff --git a/deployment/docker/victorialogs/opentelemetry-collector/elasticsearch/scrape.yml b/deployment/docker/victorialogs/opentelemetry-collector/elasticsearch/scrape.yml deleted file mode 100644 index 8257db8ef0..0000000000 --- a/deployment/docker/victorialogs/opentelemetry-collector/elasticsearch/scrape.yml +++ /dev/null @@ -1,11 +0,0 @@ -scrape_configs: - - job_name: "victoriametrics" - scrape_interval: 30s - static_configs: - - targets: - - victoriametrics:8428 - - job_name: "victorialogs" - scrape_interval: 30s - static_configs: - - targets: - - victorialogs:9428 \ No newline at end of file diff --git a/deployment/docker/victorialogs/opentelemetry-collector/loki/README.md b/deployment/docker/victorialogs/opentelemetry-collector/loki/README.md deleted file mode 100644 index eabf498e4d..0000000000 --- a/deployment/docker/victorialogs/opentelemetry-collector/loki/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# Docker compose OpenTelemetry Loki integration with VictoriaLogs for docker - -The folder contains the example of integration of [OpenTelemetry collector](https://opentelemetry.io/docs/collector/) with Victorialogs - -To spin-up environment run the following command: -``` -docker compose up -d -``` - -To shut down the docker-compose environment run the following command: -``` -docker compose down -docker compose rm -f -``` - -The docker compose file contains the following components: - -* collector - vector is configured to collect logs from the `docker`, you can find configuration in the `config.yaml`. It writes data in VictoriaLogs. It pushes metrics to VictoriaMetrics. -* VictoriaLogs - the log database, it accepts the data from `collector` by Loki protocol -* VictoriaMetrics - collects metrics from `VictoriaLogs` and `VictoriaMetrics` - -Querying the data - -* [vmui](https://docs.victoriametrics.com/victorialogs/querying/#vmui) - a web UI is accessible by `http://localhost:9428/select/vmui` -* for querying the data via command-line please check [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line) - -Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance. diff --git a/deployment/docker/victorialogs/opentelemetry-collector/loki/compose.yml b/deployment/docker/victorialogs/opentelemetry-collector/loki/compose.yml index d02b5f9ff2..865979e9b9 100644 --- a/deployment/docker/victorialogs/opentelemetry-collector/loki/compose.yml +++ b/deployment/docker/victorialogs/opentelemetry-collector/loki/compose.yml @@ -1,48 +1,3 @@ -services: - collector: - image: docker.io/otel/opentelemetry-collector-contrib:0.102.1 - restart: on-failure - volumes: - - $PWD/logs:/tmp/logs - - $PWD/config.yaml:/etc/otelcol-contrib/config.yaml - depends_on: - victorialogs: - condition: service_healthy - victoriametrics: - condition: service_healthy - - victorialogs: - image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs - volumes: - - victorialogs-vector-docker-vl:/loki - ports: - - '9428:9428' - command: - - -storageDataPath=/loki - - -loggerFormat=json - healthcheck: - test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9428/health"] - interval: 1s - timeout: 1s - retries: 10 - - victoriametrics: - image: victoriametrics/victoria-metrics:latest - ports: - - '8428:8428' - command: - - -storageDataPath=/vmsingle - - -promscrape.config=/promscrape.yml - - -loggerFormat=json - volumes: - - victorialogs-vector-docker-vm:/vmsingle - - ./scrape.yml:/promscrape.yml - healthcheck: - test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8428/health"] - interval: 1s - timeout: 1s - retries: 10 - -volumes: - victorialogs-vector-docker-vl: - victorialogs-vector-docker-vm: +include: + - ../compose.yml +name: collector-loki diff --git a/deployment/docker/victorialogs/opentelemetry-collector/loki/config.yaml b/deployment/docker/victorialogs/opentelemetry-collector/loki/config.yaml index 63df2e614d..1891bd531a 100644 --- a/deployment/docker/victorialogs/opentelemetry-collector/loki/config.yaml +++ b/deployment/docker/victorialogs/opentelemetry-collector/loki/config.yaml @@ -3,7 +3,7 @@ exporters: endpoint: http://victorialogs:9428/insert/loki/api/v1/push receivers: filelog: - include: [/tmp/logs/*.log] + include: [/var/lib/docker/containers/**/*.log] resource: region: us-east-1 service: diff --git a/deployment/docker/victorialogs/opentelemetry-collector/loki/scrape.yml b/deployment/docker/victorialogs/opentelemetry-collector/loki/scrape.yml deleted file mode 100644 index 8257db8ef0..0000000000 --- a/deployment/docker/victorialogs/opentelemetry-collector/loki/scrape.yml +++ /dev/null @@ -1,11 +0,0 @@ -scrape_configs: - - job_name: "victoriametrics" - scrape_interval: 30s - static_configs: - - targets: - - victoriametrics:8428 - - job_name: "victorialogs" - scrape_interval: 30s - static_configs: - - targets: - - victorialogs:9428 \ No newline at end of file diff --git a/deployment/docker/victorialogs/opentelemetry-collector/otlp/README.md b/deployment/docker/victorialogs/opentelemetry-collector/otlp/README.md deleted file mode 100644 index fc0e993bdc..0000000000 --- a/deployment/docker/victorialogs/opentelemetry-collector/otlp/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# Docker compose OpenTelemetry OTLP integration with VictoriaLogs for docker - -The folder contains the example of integration of [OpenTelemetry collector](https://opentelemetry.io/docs/collector/) with Victorialogs - -To spin-up environment run the following command: -``` -docker compose up -d -``` - -To shut down the docker-compose environment run the following command: -``` -docker compose down -docker compose rm -f -``` - -The docker compose file contains the following components: - -* collector - vector is configured to collect logs from the `docker`, you can find configuration in the `config.yaml`. It writes data in VictoriaLogs. It pushes metrics to VictoriaMetrics. -* VictoriaLogs - the log database, it accepts the data from `collector` by otlp protocol -* VictoriaMetrics - collects metrics from `VictoriaLogs` and `VictoriaMetrics` - -Querying the data - -* [vmui](https://docs.victoriametrics.com/victorialogs/querying/#vmui) - a web UI is accessible by `http://localhost:9428/select/vmui` -* for querying the data via command-line please check [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line) - -Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance. diff --git a/deployment/docker/victorialogs/opentelemetry-collector/otlp/compose.yml b/deployment/docker/victorialogs/opentelemetry-collector/otlp/compose.yml index ca17988827..9dcaed9be2 100644 --- a/deployment/docker/victorialogs/opentelemetry-collector/otlp/compose.yml +++ b/deployment/docker/victorialogs/opentelemetry-collector/otlp/compose.yml @@ -1,48 +1,3 @@ -services: - collector: - image: docker.io/otel/opentelemetry-collector-contrib:0.102.1 - restart: on-failure - volumes: - - $PWD/logs:/tmp/logs - - $PWD/config.yaml:/etc/otelcol-contrib/config.yaml - depends_on: - victorialogs: - condition: service_healthy - victoriametrics: - condition: service_healthy - - victorialogs: - image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs - volumes: - - victorialogs-vector-docker-vl:/otlp - ports: - - '9428:9428' - command: - - -storageDataPath=/otlp - - -loggerFormat=json - healthcheck: - test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9428/health"] - interval: 1s - timeout: 1s - retries: 10 - - victoriametrics: - image: victoriametrics/victoria-metrics:latest - ports: - - '8428:8428' - command: - - -storageDataPath=/vmsingle - - -promscrape.config=/promscrape.yml - - -loggerFormat=json - volumes: - - victorialogs-vector-docker-vm:/vmsingle - - ./scrape.yml:/promscrape.yml - healthcheck: - test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8428/health"] - interval: 1s - timeout: 1s - retries: 10 - -volumes: - victorialogs-vector-docker-vl: - victorialogs-vector-docker-vm: +include: + - ../compose.yml +name: collector-otlp diff --git a/deployment/docker/victorialogs/opentelemetry-collector/otlp/config.yaml b/deployment/docker/victorialogs/opentelemetry-collector/otlp/config.yaml index 1b56a034f7..b06ac851df 100644 --- a/deployment/docker/victorialogs/opentelemetry-collector/otlp/config.yaml +++ b/deployment/docker/victorialogs/opentelemetry-collector/otlp/config.yaml @@ -5,7 +5,7 @@ exporters: verbosity: detailed receivers: filelog: - include: [/tmp/logs/*.log] + include: [/var/lib/docker/containers/**/*.log] resource: region: us-east-1 service: diff --git a/deployment/docker/victorialogs/opentelemetry-collector/otlp/scrape.yml b/deployment/docker/victorialogs/opentelemetry-collector/otlp/scrape.yml deleted file mode 100644 index 8257db8ef0..0000000000 --- a/deployment/docker/victorialogs/opentelemetry-collector/otlp/scrape.yml +++ /dev/null @@ -1,11 +0,0 @@ -scrape_configs: - - job_name: "victoriametrics" - scrape_interval: 30s - static_configs: - - targets: - - victoriametrics:8428 - - job_name: "victorialogs" - scrape_interval: 30s - static_configs: - - targets: - - victorialogs:9428 \ No newline at end of file diff --git a/deployment/docker/victorialogs/opentelemetry-collector/syslog/README.md b/deployment/docker/victorialogs/opentelemetry-collector/syslog/README.md deleted file mode 100644 index 12a7290ae1..0000000000 --- a/deployment/docker/victorialogs/opentelemetry-collector/syslog/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# Docker compose OpenTelemetry Syslog integration with VictoriaLogs for docker - -The folder contains the example of integration of [OpenTelemetry collector](https://opentelemetry.io/docs/collector/) with Victorialogs - -To spin-up environment run the following command: -``` -docker compose up -d -``` - -To shut down the docker-compose environment run the following command: -``` -docker compose down -docker compose rm -f -``` - -The docker compose file contains the following components: - -* collector - vector is configured to collect logs from the `docker`, you can find configuration in the `config.yaml`. It writes data in VictoriaLogs. It pushes metrics to VictoriaMetrics. -* VictoriaLogs - the log database, it accepts the data from `collector` by syslog protocol -* VictoriaMetrics - collects metrics from `VictoriaLogs` and `VictoriaMetrics` - -Querying the data - -* [vmui](https://docs.victoriametrics.com/victorialogs/querying/#vmui) - a web UI is accessible by `http://localhost:9428/select/vmui` -* for querying the data via command-line please check [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line) - -Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance. diff --git a/deployment/docker/victorialogs/opentelemetry-collector/syslog/compose.yml b/deployment/docker/victorialogs/opentelemetry-collector/syslog/compose.yml index 4ac680a477..b87ea641e8 100644 --- a/deployment/docker/victorialogs/opentelemetry-collector/syslog/compose.yml +++ b/deployment/docker/victorialogs/opentelemetry-collector/syslog/compose.yml @@ -1,49 +1,3 @@ -services: - collector: - image: docker.io/otel/opentelemetry-collector-contrib:0.107.0 - restart: on-failure - volumes: - - $PWD/logs:/tmp/logs - - $PWD/config.yaml:/etc/otelcol-contrib/config.yaml - depends_on: - victorialogs: - condition: service_healthy - victoriametrics: - condition: service_healthy - - victorialogs: - image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs - volumes: - - victorialogs-vector-docker-vl:/syslog - ports: - - '9428:9428' - command: - - -storageDataPath=/syslog - - -syslog.listenAddr.tcp=:5410 - - -syslog.useLocalTimestamp.tcp - healthcheck: - test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9428/health"] - interval: 1s - timeout: 1s - retries: 10 - - victoriametrics: - image: victoriametrics/victoria-metrics:latest - ports: - - '8428:8428' - command: - - -storageDataPath=/vmsingle - - -promscrape.config=/promscrape.yml - - -loggerFormat=json - volumes: - - victorialogs-vector-docker-vm:/vmsingle - - ./scrape.yml:/promscrape.yml - healthcheck: - test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8428/health"] - interval: 1s - timeout: 1s - retries: 10 - -volumes: - victorialogs-vector-docker-vl: - victorialogs-vector-docker-vm: +include: + - ../compose.yml +name: collector-syslog diff --git a/deployment/docker/victorialogs/opentelemetry-collector/syslog/scrape.yml b/deployment/docker/victorialogs/opentelemetry-collector/syslog/scrape.yml deleted file mode 100644 index 8257db8ef0..0000000000 --- a/deployment/docker/victorialogs/opentelemetry-collector/syslog/scrape.yml +++ /dev/null @@ -1,11 +0,0 @@ -scrape_configs: - - job_name: "victoriametrics" - scrape_interval: 30s - static_configs: - - targets: - - victoriametrics:8428 - - job_name: "victorialogs" - scrape_interval: 30s - static_configs: - - targets: - - victorialogs:9428 \ No newline at end of file diff --git a/deployment/docker/victorialogs/promtail/compose.yml b/deployment/docker/victorialogs/promtail/compose.yml index 256bd54be3..01d8e01d4a 100644 --- a/deployment/docker/victorialogs/promtail/compose.yml +++ b/deployment/docker/victorialogs/promtail/compose.yml @@ -1,22 +1,12 @@ +include: + - ../compose.yml services: promtail: - image: grafana/promtail:2.8.2 + image: grafana/promtail:3.0.1 volumes: - /var/lib/docker/containers:/var/lib/docker/containers:ro - /var/log:/var/log:ro - - ./config.yml:/etc/promtail/docker-config.yml:ro + - ${PWD}/config.yml:/etc/promtail/docker-config.yml:ro command: -config.file=/etc/promtail/docker-config.yml ports: - "5140:5140" - - vlogs: - image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs - volumes: - - victorialogs-promtail-docker:/vlogs - ports: - - '9428:9428' - command: - - -storageDataPath=/vlogs - -volumes: - victorialogs-promtail-docker: diff --git a/deployment/docker/victorialogs/promtail/loki/compose.yml b/deployment/docker/victorialogs/promtail/loki/compose.yml new file mode 100644 index 0000000000..5ee00fa085 --- /dev/null +++ b/deployment/docker/victorialogs/promtail/loki/compose.yml @@ -0,0 +1,3 @@ +include: + - ../compose.yml +name: promtail-elasticsearch diff --git a/deployment/docker/victorialogs/promtail/config.yml b/deployment/docker/victorialogs/promtail/loki/config.yml similarity index 90% rename from deployment/docker/victorialogs/promtail/config.yml rename to deployment/docker/victorialogs/promtail/loki/config.yml index 98a78e3c97..85b304f8b7 100644 --- a/deployment/docker/victorialogs/promtail/config.yml +++ b/deployment/docker/victorialogs/promtail/loki/config.yml @@ -6,7 +6,7 @@ positions: filename: /tmp/positions.yaml clients: - - url: http://vlogs:9428/insert/loki/api/v1/push?_stream_fields=instance,job,host,app + - url: http://victorialogs:9428/insert/loki/api/v1/push?_stream_fields=instance,job,host,app tenant_id: "0:0" scrape_configs: diff --git a/deployment/docker/victorialogs/telegraf-docker/scrape.yml b/deployment/docker/victorialogs/telegraf-docker/scrape.yml deleted file mode 100644 index 8257db8ef0..0000000000 --- a/deployment/docker/victorialogs/telegraf-docker/scrape.yml +++ /dev/null @@ -1,11 +0,0 @@ -scrape_configs: - - job_name: "victoriametrics" - scrape_interval: 30s - static_configs: - - targets: - - victoriametrics:8428 - - job_name: "victorialogs" - scrape_interval: 30s - static_configs: - - targets: - - victorialogs:9428 \ No newline at end of file diff --git a/deployment/docker/victorialogs/telegraf-docker/README.md b/deployment/docker/victorialogs/telegraf/README.md similarity index 56% rename from deployment/docker/victorialogs/telegraf-docker/README.md rename to deployment/docker/victorialogs/telegraf/README.md index d460ba9fcd..6c08ef8a36 100644 --- a/deployment/docker/victorialogs/telegraf-docker/README.md +++ b/deployment/docker/victorialogs/telegraf/README.md @@ -1,8 +1,13 @@ # Docker compose Telegraf integration with VictoriaLogs for docker -The folder contains the example of integration of [telegraf](https://www.influxdata.com/time-series-platform/telegraf/) with VictoriaLogs +The folder contains the examples of integration of [telegraf](https://www.influxdata.com/time-series-platform/telegraf/) with VictoriaLogs using: -To spin-up environment run the following command: +* [elasticsearch](./elasticsearch) +* [loki](./loki) +* [jsonline](./jsonline) +* [syslog](./syslog) + +To spin-up environment `cd` to any of listed above directories run the following command: ``` docker compose up -d ``` @@ -23,3 +28,11 @@ Querying the data * [vmui](https://docs.victoriametrics.com/victorialogs/querying/#vmui) - a web UI is accessible by `http://localhost:9428/select/vmui` * for querying the data via command-line please check [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line) + +Telegraf configuration example can be found below: +* [elasticsearch](./elasticsearch/telegraf.conf) +* [loki](./loki/telegraf.conf) +* [jsonline](./jsonline/telegraf.conf) +* [syslog](./syslog/telegraf.conf) + +Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance. diff --git a/deployment/docker/victorialogs/telegraf/compose.yml b/deployment/docker/victorialogs/telegraf/compose.yml new file mode 100644 index 0000000000..3132712ed1 --- /dev/null +++ b/deployment/docker/victorialogs/telegraf/compose.yml @@ -0,0 +1,20 @@ +include: + - ../compose.yml +services: + telegraf: + image: bitnami/telegraf:1.31.3 + restart: on-failure + volumes: + - type: bind + source: /var/run/docker.sock + target: /var/run/docker.sock + - type: bind + source: /var/lib/docker + target: /var/lib/docker + - ${PWD}/telegraf.conf:/etc/telegraf/telegraf.conf:ro + command: --config /etc/telegraf/telegraf.conf + depends_on: + victorialogs: + condition: service_healthy + victoriametrics: + condition: service_healthy diff --git a/deployment/docker/victorialogs/telegraf/elasticsearch/compose.yml b/deployment/docker/victorialogs/telegraf/elasticsearch/compose.yml new file mode 100644 index 0000000000..e9285fa039 --- /dev/null +++ b/deployment/docker/victorialogs/telegraf/elasticsearch/compose.yml @@ -0,0 +1,3 @@ +include: + - ../compose.yml +name: telegraf-elasticsearch diff --git a/deployment/docker/victorialogs/telegraf-docker/telegraf.conf b/deployment/docker/victorialogs/telegraf/elasticsearch/telegraf.conf similarity index 70% rename from deployment/docker/victorialogs/telegraf-docker/telegraf.conf rename to deployment/docker/victorialogs/telegraf/elasticsearch/telegraf.conf index 3eafb51b58..ab08dab80a 100644 --- a/deployment/docker/victorialogs/telegraf-docker/telegraf.conf +++ b/deployment/docker/victorialogs/telegraf/elasticsearch/telegraf.conf @@ -16,23 +16,6 @@ [[inputs.cpu]] -[[outputs.http]] - url = "http://victorialogs:9428/insert/jsonline?_msg_field=fields.msg&_stream_fields=tags.log_source,tags.metric_type" - data_format = "json" - namepass = ["docker_log"] - use_batch_format = false - -[[outputs.loki]] - domain = "http://victorialogs:9428" - endpoint = "/insert/loki/api/v1/push?_msg_field=docker_log.msg&_time_field=@timestamp&_stream_fields=log_source,metric_type" - namepass = ["docker_log"] - gzip_request = true - sanitize_label_names = true - -[[outputs.syslog]] - address = "tcp://victorialogs:8094" - namepass = ["docker_log"] - [[outputs.elasticsearch]] urls = ["http://victorialogs:9428/insert/elasticsearch"] timeout = "1m" diff --git a/deployment/docker/victorialogs/telegraf/jsonline/compose.yml b/deployment/docker/victorialogs/telegraf/jsonline/compose.yml new file mode 100644 index 0000000000..68d7844a0e --- /dev/null +++ b/deployment/docker/victorialogs/telegraf/jsonline/compose.yml @@ -0,0 +1,3 @@ +include: + - ../compose.yml +name: telegraf-jsonline diff --git a/deployment/docker/victorialogs/telegraf/jsonline/telegraf.conf b/deployment/docker/victorialogs/telegraf/jsonline/telegraf.conf new file mode 100644 index 0000000000..c47978a903 --- /dev/null +++ b/deployment/docker/victorialogs/telegraf/jsonline/telegraf.conf @@ -0,0 +1,43 @@ +[agent] + interval = "10s" + round_interval = true + metric_batch_size = 1000 + metric_buffer_limit = 100000 + collection_jitter = "0s" + flush_interval = "10s" + flush_jitter = "0s" + precision = "" + debug = false + quiet = false + logtarget = "file" + logfile = "/dev/null" + hostname = "pop-os" + omit_hostname = false + +[[inputs.cpu]] + +[[outputs.http]] + url = "http://victorialogs:9428/insert/jsonline?_msg_field=fields.msg&_stream_fields=tags.log_source,tags.metric_type" + data_format = "json" + namepass = ["docker_log"] + use_batch_format = false + +[[outputs.http]] + url = "http://victoriametrics:8428/api/v1/write" + data_format = "prometheusremotewrite" + namepass = ["cpu"] + [outputs.http.headers] + Content-Type = "application/x-protobuf" + Content-Encoding = "snappy" + X-Prometheus-Remote-Write-Version = "0.1.0" + +[[inputs.docker_log]] + [inputs.docker_log.tags] + metric_type = "logs" + log_source = "telegraf" + +[[processors.rename]] + namepass = ["docker_log"] + [[processors.rename.replace]] + field = "message" + dest = "msg" diff --git a/deployment/docker/victorialogs/telegraf/loki/compose.yml b/deployment/docker/victorialogs/telegraf/loki/compose.yml new file mode 100644 index 0000000000..470a80a0a3 --- /dev/null +++ b/deployment/docker/victorialogs/telegraf/loki/compose.yml @@ -0,0 +1,3 @@ +include: + - ../compose.yml +name: telegraf-loki diff --git a/deployment/docker/victorialogs/telegraf/loki/telegraf.conf b/deployment/docker/victorialogs/telegraf/loki/telegraf.conf new file mode 100644 index 0000000000..be53392c9c --- /dev/null +++ b/deployment/docker/victorialogs/telegraf/loki/telegraf.conf @@ -0,0 +1,44 @@ +[agent] + interval = "10s" + round_interval = true + metric_batch_size = 1000 + metric_buffer_limit = 100000 + collection_jitter = "0s" + flush_interval = "10s" + flush_jitter = "0s" + precision = "" + debug = false + quiet = false + logtarget = "file" + logfile = "/dev/null" + hostname = "pop-os" + omit_hostname = false + +[[inputs.cpu]] + +[[outputs.loki]] + domain = "http://victorialogs:9428" + endpoint = "/insert/loki/api/v1/push?_msg_field=docker_log.msg&_time_field=@timestamp&_stream_fields=log_source,metric_type" + namepass = ["docker_log"] + gzip_request = true + sanitize_label_names = true + +[[outputs.http]] + url = "http://victoriametrics:8428/api/v1/write" + data_format = "prometheusremotewrite" + namepass = ["cpu"] + [outputs.http.headers] + Content-Type = "application/x-protobuf" + Content-Encoding = "snappy" + X-Prometheus-Remote-Write-Version = "0.1.0" + +[[inputs.docker_log]] + [inputs.docker_log.tags] + metric_type = "logs" + log_source = "telegraf" + +[[processors.rename]] + namepass = ["docker_log"] + [[processors.rename.replace]] + field = "message" + dest = "msg" diff --git a/deployment/docker/victorialogs/telegraf/syslog/compose.yml b/deployment/docker/victorialogs/telegraf/syslog/compose.yml new file mode 100644 index 0000000000..698adb2a9e --- /dev/null +++ b/deployment/docker/victorialogs/telegraf/syslog/compose.yml @@ -0,0 +1,3 @@ +include: + - ../compose.yml +name: telegraf-syslog diff --git a/deployment/docker/victorialogs/telegraf/syslog/telegraf.conf b/deployment/docker/victorialogs/telegraf/syslog/telegraf.conf new file mode 100644 index 0000000000..ff44f832c0 --- /dev/null +++ b/deployment/docker/victorialogs/telegraf/syslog/telegraf.conf @@ -0,0 +1,41 @@ +[agent] + interval = "10s" + round_interval = true + metric_batch_size = 1000 + metric_buffer_limit = 100000 + collection_jitter = "0s" + flush_interval = "10s" + flush_jitter = "0s" + precision = "" + debug = false + quiet = false + logtarget = "file" + logfile = "/dev/null" + hostname = "pop-os" + omit_hostname = false + +[[inputs.cpu]] + +[[outputs.syslog]] + address = "tcp://victorialogs:8094" + namepass = ["docker_log"] + +[[outputs.http]] + url = "http://victoriametrics:8428/api/v1/write" + data_format = "prometheusremotewrite" + namepass = ["cpu"] + [outputs.http.headers] + Content-Type = "application/x-protobuf" + Content-Encoding = "snappy" + X-Prometheus-Remote-Write-Version = "0.1.0" + +[[inputs.docker_log]] + [inputs.docker_log.tags] + metric_type = "logs" + log_source = "telegraf" + +[[processors.rename]] + namepass = ["docker_log"] + [[processors.rename.replace]] + field = "message" + dest = "msg" diff --git a/deployment/docker/victorialogs/vector-ha-single-node/README.md b/deployment/docker/victorialogs/vector-ha-single-node/README.md deleted file mode 100644 index 2a6fd630e1..0000000000 --- a/deployment/docker/victorialogs/vector-ha-single-node/README.md +++ /dev/null @@ -1,104 +0,0 @@ -# Docker compose Vector integration with VictoriaLogs for docker. High-Availability example - -The folder contains the example of integration of [vector](https://vector.dev/docs/) with VictoriaLogs Single-Node(s) and [vmauth](https://docs.victoriametrics.com/vmauth/) for achieving High Availability. - -Check [this documentation](https://docs.victoriametrics.com/victorialogs/#high-availability) with a description of the architecture and components. - - -To spin-up environment run the following command: - -```shell -docker compose up -d -``` - -To shut down the docker-compose environment run the following command: - -```shell -docker compose down -docker compose rm -f -``` - -The docker compose file contains the following components: - -* vector - vector is configured to collect logs from the `docker`, you can find configuration in the `vector.yaml`. It writes data in two instances of VictoriaLogs -* VictoriaLogs - the two instances of log database, they accept the data from `vector` by json line protocol -* vmauth - load balancer for proxying requests to one of VictoriaLogs - -Querying the data - -* [vmui](https://docs.victoriametrics.com/victorialogs/querying/#vmui) - a web UI is accessible by `http://localhost:8427/select/vmui/` -* for querying the data via command-line please check [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line) - - -the example of vector configuration(`vector.yaml`) - -```yaml -api: - enabled: true - address: 0.0.0.0:8686 -sources: - docker: - type: docker_logs -transforms: - msg_parser: - type: remap - inputs: - - docker - source: | - if exists(.message) { - .log, err = parse_json(.message) - if err == null { - del(.message) - } - } -sinks: - console_out: - type: console - inputs: - - msg_parser - encoding: - codec: json - vlogs_http_1: - type: http - inputs: - - msg_parser - uri: http://victorialogs-1:9428/insert/jsonline?_stream_fields=source_type,host,container_name&_msg_field=log.msg&_time_field=timestamp - encoding: - codec: json - framing: - method: newline_delimited - compression: gzip - healthcheck: - enabled: false - request: - headers: - AccountID: '0' - ProjectID: '0' - vlogs_http_2: - type: http - inputs: - - msg_parser - uri: http://victorialogs-2:9428/insert/jsonline?_stream_fields=source_type,host,container_name&_msg_field=log.msg&_time_field=timestamp - encoding: - codec: json - framing: - method: newline_delimited - compression: gzip - healthcheck: - enabled: false - request: - headers: - AccountID: '0' - ProjectID: '0' -``` - -Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance. - -The example of vmauth configuration (`auth.yml`) - -```yaml -unauthorized_user: - url_prefix: - - http://victorialogs-1:9428 - - http://victorialogs-2:9428 -``` \ No newline at end of file diff --git a/deployment/docker/victorialogs/vector-ha-single-node/auth.yml b/deployment/docker/victorialogs/vector-ha-single-node/auth.yml deleted file mode 100644 index eeafbb852b..0000000000 --- a/deployment/docker/victorialogs/vector-ha-single-node/auth.yml +++ /dev/null @@ -1,6 +0,0 @@ -# balance load among victorialogs instances -# see https://docs.victoriametrics.com/vmauth/#load-balancing -unauthorized_user: - url_prefix: - - http://victorialogs-1:9428 - - http://victorialogs-2:9428 diff --git a/deployment/docker/victorialogs/vector-ha-single-node/compose.yml b/deployment/docker/victorialogs/vector-ha-single-node/compose.yml deleted file mode 100644 index ae339f83a9..0000000000 --- a/deployment/docker/victorialogs/vector-ha-single-node/compose.yml +++ /dev/null @@ -1,48 +0,0 @@ -services: - vector: - image: docker.io/timberio/vector:0.40.1-distroless-static - restart: on-failure - volumes: - - type: bind - source: /var/run/docker.sock - target: /var/run/docker.sock - - type: bind - source: /var/lib/docker - target: /var/lib/docker - - ./vector.yaml:/etc/vector/vector.yaml:ro - user: root - ports: - - '8686:8686' - depends_on: [victorialogs-1,victorialogs-2] - - victorialogs-1: - image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs - volumes: - - victorialogs-vector-docker-vl-ha-single-1:/vlogs - command: - - -storageDataPath=/vlogs - - -loggerFormat=json - victorialogs-2: - image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs - volumes: - - victorialogs-vector-docker-vl-ha-single-2:/vlogs - command: - - -storageDataPath=/vlogs - - -loggerFormat=json - vmauth: - container_name: vmauth - image: victoriametrics/vmauth:v1.103.0 - depends_on: - - "victorialogs-1" - - "victorialogs-2" - volumes: - - ./auth.yml:/etc/auth.yml - command: - - '--auth.config=/etc/auth.yml' - ports: - - 8427:8427 - restart: always - -volumes: - victorialogs-vector-docker-vl-ha-single-1: - victorialogs-vector-docker-vl-ha-single-2: diff --git a/deployment/docker/victorialogs/vector-ha-single-node/vector.yaml b/deployment/docker/victorialogs/vector-ha-single-node/vector.yaml deleted file mode 100644 index e67320b974..0000000000 --- a/deployment/docker/victorialogs/vector-ha-single-node/vector.yaml +++ /dev/null @@ -1,58 +0,0 @@ -api: - enabled: true - address: 0.0.0.0:8686 -sources: - docker: - type: docker_logs -transforms: - msg_parser: - type: remap - inputs: - - docker - source: | - if exists(.message) { - .log, err = parse_json(.message) - if err == null { - del(.message) - } - } -sinks: - console_out: - type: console - inputs: - - msg_parser - encoding: - codec: json - vlogs_http_1: - type: http - inputs: - - msg_parser - uri: http://victorialogs-1:9428/insert/jsonline?_stream_fields=source_type,host,container_name&_msg_field=log.msg&_time_field=timestamp - encoding: - codec: json - framing: - method: newline_delimited - compression: gzip - healthcheck: - enabled: false - request: - headers: - AccountID: '0' - ProjectID: '0' - vlogs_http_2: - type: http - inputs: - - msg_parser - uri: http://victorialogs-2:9428/insert/jsonline?_stream_fields=source_type,host,container_name&_msg_field=log.msg&_time_field=timestamp - encoding: - codec: json - framing: - method: newline_delimited - compression: gzip - healthcheck: - enabled: false - request: - headers: - AccountID: '0' - ProjectID: '0' - diff --git a/deployment/docker/victorialogs/vector/README.md b/deployment/docker/victorialogs/vector/README.md index 4acd31d361..5022982223 100644 --- a/deployment/docker/victorialogs/vector/README.md +++ b/deployment/docker/victorialogs/vector/README.md @@ -1,8 +1,13 @@ -# Docker compose Vector integration with VictoriaLogs for docker +# Docker compose Vector integration with VictoriaLogs using given below protocols: + +* [elasticsearch](./elasticsearch) +* [loki](./loki) +* [jsonline single node](./jsonline) +* [jsonline HA setup](./jsonline-ha) The folder contains the example of integration of [vector](https://vector.dev/docs/) with Victorialogs -To spin-up environment run the following command: +To spin-up environment `cd` to any of listed above directories run the following command: ``` docker compose up -d ``` @@ -16,7 +21,7 @@ docker compose rm -f The docker compose file contains the following components: * vector - vector is configured to collect logs from the `docker`, you can find configuration in the `vector.yaml`. It writes data in VictoriaLogs. It pushes metrics to VictoriaMetrics. -* VictoriaLogs - the log database, it accepts the data from `vector` by elastic protocol +* VictoriaLogs - the log database, it accepts the data from `vector` by DataDog protocol * VictoriaMetrics - collects metrics from `VictoriaLogs` and `VictoriaMetrics` Querying the data @@ -24,40 +29,10 @@ Querying the data * [vmui](https://docs.victoriametrics.com/victorialogs/querying/#vmui) - a web UI is accessible by `http://localhost:9428/select/vmui` * for querying the data via command-line please check [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line) - -the example of vector configuration(`vector.yaml`) - -``` -sources: - docker: - type: docker_logs -transforms: - msg_parser: - type: remap - inputs: - - docker - source: | - .log = parse_json!(.message) - del(.message) -sinks: - vlogs_es: - type: elasticsearch - inputs: - - msg_parser - endpoints: - - http://victorialogs:9428/insert/elasticsearch/ - mode: bulk - api_version: v8 - compression: gzip - healthcheck.enabled: false - query: - _msg_field: log.msg - _time_field: timestamp - _stream_fields: source_type,host,container_name - request: - headers: - AccountID: "0" - ProjectID: "0" -``` +Vector configuration example can be found below: +* [elasticsearch](./elasticsearch/vector.yaml) +* [loki](./loki/vector.yaml) +* [jsonline single node](./jsonline/vector.yaml) +* [jsonline HA setup](./jsonline-ha/vector.yaml) Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance. diff --git a/deployment/docker/victorialogs/vector/compose.yml b/deployment/docker/victorialogs/vector/compose.yml index 9a444d5511..9c9c117d7b 100644 --- a/deployment/docker/victorialogs/vector/compose.yml +++ b/deployment/docker/victorialogs/vector/compose.yml @@ -1,6 +1,8 @@ +include: + - ../compose.yml services: vector: - image: docker.io/timberio/vector:0.38.0-distroless-static + image: docker.io/timberio/vector:0.40.0-distroless-static restart: on-failure volumes: - type: bind @@ -9,7 +11,7 @@ services: - type: bind source: /var/lib/docker target: /var/lib/docker - - ./vector.yaml:/etc/vector/vector.yaml:ro + - ${PWD}/vector.yaml:/etc/vector/vector.yaml:ro user: root ports: - '8686:8686' @@ -18,39 +20,3 @@ services: condition: service_healthy victoriametrics: condition: service_healthy - - victorialogs: - image: docker.io/victoriametrics/victoria-logs:v0.29.0-victorialogs - volumes: - - victorialogs-vector-docker-vl:/vlogs - ports: - - '9428:9428' - command: - - -storageDataPath=/vlogs - - -loggerFormat=json - healthcheck: - test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9428/health"] - interval: 1s - timeout: 1s - retries: 10 - - victoriametrics: - image: victoriametrics/victoria-metrics:latest - ports: - - '8428:8428' - command: - - -storageDataPath=/vmsingle - - -promscrape.config=/promscrape.yml - - -loggerFormat=json - volumes: - - victorialogs-vector-docker-vm:/vmsingle - - ./scrape.yml:/promscrape.yml - healthcheck: - test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8428/health"] - interval: 1s - timeout: 1s - retries: 10 - -volumes: - victorialogs-vector-docker-vl: - victorialogs-vector-docker-vm: diff --git a/deployment/docker/victorialogs/vector/elasticsearch/compose.yaml b/deployment/docker/victorialogs/vector/elasticsearch/compose.yaml new file mode 100644 index 0000000000..bbb51e727f --- /dev/null +++ b/deployment/docker/victorialogs/vector/elasticsearch/compose.yaml @@ -0,0 +1,3 @@ +include: + - ../compose.yml +name: vector-elasticsearch diff --git a/deployment/docker/victorialogs/vector/elasticsearch/vector.yaml b/deployment/docker/victorialogs/vector/elasticsearch/vector.yaml new file mode 100644 index 0000000000..520a7a9bd7 --- /dev/null +++ b/deployment/docker/victorialogs/vector/elasticsearch/vector.yaml @@ -0,0 +1,43 @@ +api: + enabled: true + address: 0.0.0.0:8686 +sources: + docker: + type: docker_logs + demo: + type: demo_logs + format: apache_common + interval: 10 + vector_metrics: + type: internal_metrics +transforms: + msg_parser: + type: remap + inputs: [docker] + source: | + .log = parse_json!(.message) + del(.message) +sinks: + elasticsearch: + type: elasticsearch + inputs: [demo] + endpoints: [http://victorialogs:9428/insert/elasticsearch/] + mode: bulk + api_version: v8 + compression: gzip + healthcheck: + enabled: false + query: + _msg_field: message + _time_field: timestamp + _stream_fields: source_type + request: + headers: + AccountID: "0" + ProjectID: "0" + victoriametrics: + type: prometheus_remote_write + endpoint: http://victoriametrics:8428/api/v1/write + inputs: [vector_metrics] + healthcheck: + enabled: false diff --git a/deployment/docker/victorialogs/vector/jsonline-ha/compose.yml b/deployment/docker/victorialogs/vector/jsonline-ha/compose.yml new file mode 100644 index 0000000000..7d873452b4 --- /dev/null +++ b/deployment/docker/victorialogs/vector/jsonline-ha/compose.yml @@ -0,0 +1,5 @@ +include: + - path: + - ../compose.yml + - ../../compose-ha.yml +name: vector-jsonline-ha diff --git a/deployment/docker/victorialogs/vector/vector.yaml b/deployment/docker/victorialogs/vector/jsonline-ha/vector.yaml similarity index 66% rename from deployment/docker/victorialogs/vector/vector.yaml rename to deployment/docker/victorialogs/vector/jsonline-ha/vector.yaml index b409823473..331a4f05a9 100644 --- a/deployment/docker/victorialogs/vector/vector.yaml +++ b/deployment/docker/victorialogs/vector/jsonline-ha/vector.yaml @@ -18,7 +18,7 @@ transforms: .log = parse_json!(.message) del(.message) sinks: - vlogs_http: + vlogs-1: type: http inputs: - msg_parser @@ -34,40 +34,26 @@ sinks: headers: AccountID: '0' ProjectID: '0' - vlogs_loki: - type: loki + vlogs-2: + type: http inputs: - - demo - endpoint: http://victorialogs:9428/insert/loki/ - compression: gzip - path: /api/v1/push?_msg_field=message.message&_time_field=timestamp&_stream_fields=source + - msg_parser + uri: http://victorialogs-2:9428/insert/jsonline?_stream_fields=source_type,host,container_name&_msg_field=log.msg&_time_field=timestamp encoding: codec: json - labels: - source: vector - victoriametrics: - type: prometheus_remote_write - endpoint: http://victoriametrics:8428/api/v1/write - inputs: - - metrics - healthcheck: - enabled: false - vlogs_es: - type: elasticsearch - inputs: - - demo - endpoints: - - http://victorialogs:9428/insert/elasticsearch/ - mode: bulk - api_version: v8 + framing: + method: newline_delimited compression: gzip healthcheck: enabled: false - query: - _msg_field: message - _time_field: timestamp - _stream_fields: source_type request: headers: AccountID: '0' ProjectID: '0' + victoriametrics: + type: prometheus_remote_write + endpoint: http://victoriametrics:8428/api/v1/write + inputs: + - metrics + healthcheck: + enabled: false diff --git a/deployment/docker/victorialogs/vector/jsonline/compose.yml b/deployment/docker/victorialogs/vector/jsonline/compose.yml new file mode 100644 index 0000000000..10a44b1404 --- /dev/null +++ b/deployment/docker/victorialogs/vector/jsonline/compose.yml @@ -0,0 +1,3 @@ +include: + - ../compose.yml +name: vector-jsonline diff --git a/deployment/docker/victorialogs/vector/jsonline/vector.yaml b/deployment/docker/victorialogs/vector/jsonline/vector.yaml new file mode 100644 index 0000000000..3c09a818f9 --- /dev/null +++ b/deployment/docker/victorialogs/vector/jsonline/vector.yaml @@ -0,0 +1,43 @@ +api: + enabled: true + address: 0.0.0.0:8686 +sources: + docker: + type: docker_logs + demo: + type: demo_logs + format: json + metrics: + type: internal_metrics +transforms: + msg_parser: + type: remap + inputs: + - docker + source: | + .log = parse_json!(.message) + del(.message) +sinks: + http: + type: http + inputs: + - msg_parser + uri: http://victorialogs:9428/insert/jsonline?_stream_fields=source_type,host,container_name&_msg_field=log.msg&_time_field=timestamp + encoding: + codec: json + framing: + method: newline_delimited + compression: gzip + healthcheck: + enabled: false + request: + headers: + AccountID: '0' + ProjectID: '0' + victoriametrics: + type: prometheus_remote_write + endpoint: http://victoriametrics:8428/api/v1/write + inputs: + - metrics + healthcheck: + enabled: false diff --git a/deployment/docker/victorialogs/vector/loki/compose.yml b/deployment/docker/victorialogs/vector/loki/compose.yml new file mode 100644 index 0000000000..433c2bf9a5 --- /dev/null +++ b/deployment/docker/victorialogs/vector/loki/compose.yml @@ -0,0 +1,3 @@ +include: + - ../compose.yml +name: vector-loki diff --git a/deployment/docker/victorialogs/vector/loki/vector.yaml b/deployment/docker/victorialogs/vector/loki/vector.yaml new file mode 100644 index 0000000000..3c8911386a --- /dev/null +++ b/deployment/docker/victorialogs/vector/loki/vector.yaml @@ -0,0 +1,38 @@ +api: + enabled: true + address: 0.0.0.0:8686 +sources: + docker: + type: docker_logs + demo: + type: demo_logs + format: json + metrics: + type: internal_metrics +transforms: + msg_parser: + type: remap + inputs: + - docker + source: | + .log = parse_json!(.message) + del(.message) +sinks: + loki: + type: loki + inputs: + - demo + endpoint: http://victorialogs:9428/insert/loki/ + compression: gzip + path: /api/v1/push?_msg_field=message.message&_time_field=timestamp&_stream_fields=source + encoding: + codec: json + labels: + source: vector + victoriametrics: + type: prometheus_remote_write + endpoint: http://victoriametrics:8428/api/v1/write + inputs: + - metrics + healthcheck: + enabled: false diff --git a/deployment/docker/victorialogs/vector/scrape.yml b/deployment/docker/victorialogs/vector/scrape.yml deleted file mode 100644 index 8257db8ef0..0000000000 --- a/deployment/docker/victorialogs/vector/scrape.yml +++ /dev/null @@ -1,11 +0,0 @@ -scrape_configs: - - job_name: "victoriametrics" - scrape_interval: 30s - static_configs: - - targets: - - victoriametrics:8428 - - job_name: "victorialogs" - scrape_interval: 30s - static_configs: - - targets: - - victorialogs:9428 \ No newline at end of file diff --git a/docs/VictoriaLogs/data-ingestion/Telegraf.md b/docs/VictoriaLogs/data-ingestion/Telegraf.md index 1e8fe76f9b..a31b86d957 100644 --- a/docs/VictoriaLogs/data-ingestion/Telegraf.md +++ b/docs/VictoriaLogs/data-ingestion/Telegraf.md @@ -120,4 +120,4 @@ See also: - [Data ingestion troubleshooting](https://docs.victoriametrics.com/victorialogs/data-ingestion/#troubleshooting). - [How to query VictoriaLogs](https://docs.victoriametrics.com/victorialogs/querying/). - [Elasticsearch output docs for Telegraf](https://github.com/influxdata/telegraf/tree/master/plugins/outputs/elasticsearch). -- [Docker-compose demo for Telegraf integration with VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/telegraf-docker). +- [Docker-compose demo for Telegraf integration with VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/telegraf). diff --git a/docs/VictoriaLogs/data-ingestion/Vector.md b/docs/VictoriaLogs/data-ingestion/Vector.md index 5726314790..85ac76749c 100644 --- a/docs/VictoriaLogs/data-ingestion/Vector.md +++ b/docs/VictoriaLogs/data-ingestion/Vector.md @@ -213,4 +213,4 @@ See also: - [Data ingestion troubleshooting](https://docs.victoriametrics.com/victorialogs/data-ingestion/#troubleshooting). - [How to query VictoriaLogs](https://docs.victoriametrics.com/victorialogs/querying/). - [Elasticsearch output docs for Vector](https://vector.dev/docs/reference/configuration/sinks/elasticsearch/). -- [Docker-compose demo for Filebeat integration with VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/vector). +- [Docker-compose demo for Vector integration with VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/vector).