From e498fa69600767e24de88fc371a6aedcffc0886c Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 17 Jun 2024 23:16:34 +0200 Subject: [PATCH] app/vlinsert/syslog: allow accepting syslog messages with different configs at different ports --- app/vlinsert/loki/loki.go | 2 +- app/vlinsert/syslog/syslog.go | 123 +++++++++++-------- app/vlinsert/syslog/syslog_test.go | 4 +- docs/VictoriaLogs/CHANGELOG.md | 2 + docs/VictoriaLogs/README.md | 134 ++++++++++++++++----- docs/VictoriaLogs/data-ingestion/syslog.md | 24 +++- lib/logstorage/tenant_id.go | 19 +-- lib/logstorage/tenant_id_test.go | 4 +- 8 files changed, 216 insertions(+), 96 deletions(-) diff --git a/app/vlinsert/loki/loki.go b/app/vlinsert/loki/loki.go index 95c5fd5e6..5900f2495 100644 --- a/app/vlinsert/loki/loki.go +++ b/app/vlinsert/loki/loki.go @@ -46,7 +46,7 @@ func getCommonParams(r *http.Request) (*insertutils.CommonParams, error) { if cp.TenantID.AccountID == 0 && cp.TenantID.ProjectID == 0 { org := r.Header.Get("X-Scope-OrgID") if org != "" { - tenantID, err := logstorage.GetTenantIDFromString(org) + tenantID, err := logstorage.ParseTenantID(org) if err != nil { return nil, err } diff --git a/app/vlinsert/syslog/syslog.go b/app/vlinsert/syslog/syslog.go index 6a9a5ef7b..a6176673b 100644 --- a/app/vlinsert/syslog/syslog.go +++ b/app/vlinsert/syslog/syslog.go @@ -32,20 +32,27 @@ import ( ) var ( - syslogTenantID = flag.String("syslog.tenantID", "0:0", "TenantID for logs ingested via Syslog protocol. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/") syslogTimezone = flag.String("syslog.timezone", "Local", "Timezone to use when parsing timestamps in RFC3164 syslog messages. Timezone must be a valid IANA Time Zone. "+ "For example: America/New_York, Europe/Berlin, Etc/GMT+3 . See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/") - listenAddrTCP = flag.String("syslog.listenAddr.tcp", "", "Optional TCP address to listen to for Syslog messages. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/") - listenAddrUDP = flag.String("syslog.listenAddr.udp", "", "Optional UDP address to listen to for Syslog messages. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/") + syslogTenantIDTCP = flagutil.NewArrayString("syslog.tenantID.tcp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.tcp. "+ + "See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/") + syslogTenantIDUDP = flagutil.NewArrayString("syslog.tenantID.udp", "TenantID for logs ingested via the corresponding -syslog.listenAddr.udp. "+ + "See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/") - tlsEnable = flag.Bool("syslog.tls", false, "Whether to use TLS for receiving syslog messages at -syslog.listenAddr.tcp. -syslog.tlsCertFile and -syslog.tlsKeyFile must be set "+ - "if -syslog.tls is set. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/") - tlsCertFile = flag.String("syslog.tlsCertFile", "", "Path to file with TLS certificate for -syslog.listenAddr.tcp if -syslog.tls is set. "+ + listenAddrTCP = flagutil.NewArrayString("syslog.listenAddr.tcp", "Comma-separated list of TCP addresses to listen to for Syslog messages. "+ + "See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/") + listenAddrUDP = flagutil.NewArrayString("syslog.listenAddr.udp", "Comma-separated list of UDP address to listen to for Syslog messages. "+ + "See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/") + + tlsEnable = flagutil.NewArrayBool("syslog.tls", "Whether to enable TLS for receiving syslog messages at the corresponding -syslog.listenAddr.tcp. "+ + "The corresponding -syslog.tlsCertFile and -syslog.tlsKeyFile must be set if -syslog.tls is set. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/") + tlsCertFile = flagutil.NewArrayString("syslog.tlsCertFile", "Path to file with TLS certificate for the corresponding -syslog.listenAddr.tcp if the corresponding -syslog.tls is set. "+ "Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated. "+ "See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/") - tlsKeyFile = flag.String("syslog.tlsKeyFile", "", "Path to file with TLS key for -syslog.listenAddr.tcp if -syslog.tls is set. "+ - "The provided key file is automatically re-read every second, so it can be dynamically updated") + tlsKeyFile = flagutil.NewArrayString("syslog.tlsKeyFile", "Path to file with TLS key for the corresponding -syslog.listenAddr.tcp if the corresponding -syslog.tls is set. "+ + "The provided key file is automatically re-read every second, so it can be dynamically updated. "+ + "See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/") tlsCipherSuites = flagutil.NewArrayString("syslog.tlsCipherSuites", "Optional list of TLS cipher suites for -syslog.listenAddr.tcp if -syslog.tls is set. "+ "See the list of supported cipher suites at https://pkg.go.dev/crypto/tls#pkg-constants . "+ "See also https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/") @@ -53,7 +60,9 @@ var ( "Supported values: TLS10, TLS11, TLS12, TLS13. "+ "See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/") - compressMethod = flag.String("syslog.compressMethod", "", "Compression method for syslog messages received at -syslog.listenAddr.tcp and -syslog.listenAddr.udp. "+ + compressMethodTCP = flagutil.NewArrayString("syslog.compressMethod.tcp", "Compression method for syslog messages received at the corresponding -syslog.listenAddr.tcp. "+ + "Supported values: none, gzip, deflate. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/") + compressMethodUDP = flagutil.NewArrayString("syslog.compressMethod.udp", "Compression method for syslog messages received at the corresponding -syslog.listenAddr.udp. "+ "Supported values: none, gzip, deflate. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/") ) @@ -68,32 +77,20 @@ func MustInit() { } workersStopCh = make(chan struct{}) - tenantID, err := logstorage.GetTenantIDFromString(*syslogTenantID) - if err != nil { - logger.Fatalf("cannot parse -syslog.tenantID=%q: %s", *syslogTenantID, err) - } - globalTenantID = tenantID - - switch *compressMethod { - case "", "none", "gzip", "deflate": - default: - logger.Fatalf("unexpected -syslog.compressLevel=%q; supported values: none, gzip, deflate", *compressMethod) - } - - if *listenAddrTCP != "" { + for argIdx, addr := range *listenAddrTCP { workersWG.Add(1) - go func() { - runTCPListener(*listenAddrTCP) + go func(addr string, argIdx int) { + runTCPListener(addr, argIdx) workersWG.Done() - }() + }(addr, argIdx) } - if *listenAddrUDP != "" { + for argIdx, addr := range *listenAddrUDP { workersWG.Add(1) - go func() { - runUDPListener(*listenAddrUDP) + go func(addr string, argIdx int) { + runUDPListener(addr, argIdx) workersWG.Done() - }() + }(addr, argIdx) } currentYear := time.Now().Year() @@ -126,7 +123,6 @@ func MustInit() { } var ( - globalTenantID logstorage.TenantID globalCurrentYear atomic.Int64 globalTimezone *time.Location ) @@ -143,15 +139,24 @@ func MustStop() { workersStopCh = nil } -func runUDPListener(addr string) { +func runUDPListener(addr string, argIdx int) { ln, err := net.ListenPacket(netutil.GetUDPNetwork(), addr) if err != nil { logger.Fatalf("cannot start UDP syslog server at %q: %s", addr, err) } + tenantIDStr := syslogTenantIDUDP.GetOptionalArg(argIdx) + tenantID, err := logstorage.ParseTenantID(tenantIDStr) + if err != nil { + logger.Fatalf("cannot parse -syslog.tenantID.udp=%q for -syslog.listenAddr.udp=%q: %s", tenantIDStr, addr, err) + } + + compressMethod := compressMethodUDP.GetOptionalArg(argIdx) + checkCompressMethod(compressMethod, addr, "udp") + doneCh := make(chan struct{}) go func() { - serveUDP(ln) + serveUDP(ln, tenantID, compressMethod) close(doneCh) }() @@ -162,13 +167,15 @@ func runUDPListener(addr string) { <-doneCh } -func runTCPListener(addr string) { +func runTCPListener(addr string, argIdx int) { var tlsConfig *tls.Config - if *tlsEnable { - tc, err := netutil.GetServerTLSConfig(*tlsCertFile, *tlsKeyFile, *tlsMinVersion, *tlsCipherSuites) + if tlsEnable.GetOptionalArg(argIdx) { + certFile := tlsCertFile.GetOptionalArg(argIdx) + keyFile := tlsKeyFile.GetOptionalArg(argIdx) + tc, err := netutil.GetServerTLSConfig(certFile, keyFile, *tlsMinVersion, *tlsCipherSuites) if err != nil { logger.Fatalf("cannot load TLS cert from -syslog.tlsCertFile=%q, -syslog.tlsKeyFile=%q, -syslog.tlsMinVersion=%q, -syslog.tlsCipherSuites=%q: %s", - *tlsCertFile, *tlsKeyFile, *tlsMinVersion, *tlsCipherSuites, err) + certFile, keyFile, *tlsMinVersion, *tlsCipherSuites, err) } tlsConfig = tc } @@ -177,9 +184,18 @@ func runTCPListener(addr string) { logger.Fatalf("syslog: cannot start TCP listener at %s: %s", addr, err) } + tenantIDStr := syslogTenantIDTCP.GetOptionalArg(argIdx) + tenantID, err := logstorage.ParseTenantID(tenantIDStr) + if err != nil { + logger.Fatalf("cannot parse -syslog.tenantID.tcp=%q for -syslog.listenAddr.tcp=%q: %s", tenantIDStr, addr, err) + } + + compressMethod := compressMethodTCP.GetOptionalArg(argIdx) + checkCompressMethod(compressMethod, addr, "tcp") + doneCh := make(chan struct{}) go func() { - serveTCP(ln) + serveTCP(ln, tenantID, compressMethod) close(doneCh) }() @@ -190,7 +206,16 @@ func runTCPListener(addr string) { <-doneCh } -func serveUDP(ln net.PacketConn) { +func checkCompressMethod(compressMethod, addr, protocol string) { + switch compressMethod { + case "", "none", "gzip", "deflate": + return + default: + logger.Fatalf("unsupported -syslog.compressMethod.%s=%q for -syslog.listenAddr.%s=%q; supported values: 'none', 'gzip', 'deflate'", protocol, compressMethod, protocol, addr) + } +} + +func serveUDP(ln net.PacketConn, tenantID logstorage.TenantID, compressMethod string) { gomaxprocs := cgroup.AvailableCPUs() var wg sync.WaitGroup localAddr := ln.LocalAddr() @@ -198,7 +223,7 @@ func serveUDP(ln net.PacketConn) { wg.Add(1) go func() { defer wg.Done() - cp := insertutils.GetCommonParamsForSyslog(globalTenantID) + cp := insertutils.GetCommonParamsForSyslog(tenantID) var bb bytesutil.ByteBuffer bb.B = bytesutil.ResizeNoCopyNoOverallocate(bb.B, 64*1024) for { @@ -223,7 +248,7 @@ func serveUDP(ln net.PacketConn) { } bb.B = bb.B[:n] udpRequestsTotal.Inc() - if err := processStream(bb.NewReader(), cp); err != nil { + if err := processStream(bb.NewReader(), compressMethod, cp); err != nil { logger.Errorf("syslog: cannot process UDP data from %s at %s: %s", remoteAddr, localAddr, err) } } @@ -232,7 +257,7 @@ func serveUDP(ln net.PacketConn) { wg.Wait() } -func serveTCP(ln net.Listener) { +func serveTCP(ln net.Listener, tenantID logstorage.TenantID, compressMethod string) { var cm ingestserver.ConnsMap cm.Init("syslog") @@ -262,8 +287,8 @@ func serveTCP(ln net.Listener) { wg.Add(1) go func() { - cp := insertutils.GetCommonParamsForSyslog(globalTenantID) - if err := processStream(c, cp); err != nil { + cp := insertutils.GetCommonParamsForSyslog(tenantID) + if err := processStream(c, compressMethod, cp); err != nil { logger.Errorf("syslog: cannot process TCP data at %q: %s", addr, err) } @@ -278,20 +303,20 @@ func serveTCP(ln net.Listener) { } // processStream parses a stream of syslog messages from r and ingests them into vlstorage. -func processStream(r io.Reader, cp *insertutils.CommonParams) error { +func processStream(r io.Reader, compressMethod string, cp *insertutils.CommonParams) error { if err := vlstorage.CanWriteData(); err != nil { return err } lmp := cp.NewLogMessageProcessor() - err := processStreamInternal(r, lmp) + err := processStreamInternal(r, compressMethod, lmp) lmp.MustClose() return err } -func processStreamInternal(r io.Reader, lmp insertutils.LogMessageProcessor) error { - switch *compressMethod { +func processStreamInternal(r io.Reader, compressMethod string, lmp insertutils.LogMessageProcessor) error { + switch compressMethod { case "", "none": case "gzip": zr, err := common.GetGzipReader(r) @@ -306,12 +331,12 @@ func processStreamInternal(r io.Reader, lmp insertutils.LogMessageProcessor) err } r = zr default: - logger.Panicf("BUG: compressLevel=%q; supported values: none, gzip, deflate", *compressMethod) + logger.Panicf("BUG: unsupported compressMethod=%q; supported values: none, gzip, deflate", compressMethod) } err := processUncompressedStream(r, lmp) - switch *compressMethod { + switch compressMethod { case "gzip": zr := r.(*gzip.Reader) common.PutGzipReader(zr) diff --git a/app/vlinsert/syslog/syslog_test.go b/app/vlinsert/syslog/syslog_test.go index c99e4707f..b0069e9ec 100644 --- a/app/vlinsert/syslog/syslog_test.go +++ b/app/vlinsert/syslog/syslog_test.go @@ -77,7 +77,7 @@ func TestProcessStreamInternal_Success(t *testing.T) { tlp := &insertutils.TestLogMessageProcessor{} r := bytes.NewBufferString(data) - if err := processStreamInternal(r, tlp); err != nil { + if err := processStreamInternal(r, "", tlp); err != nil { t.Fatalf("unexpected error: %s", err) } if err := tlp.Verify(rowsExpected, timestampsExpected, resultExpected); err != nil { @@ -107,7 +107,7 @@ func TestProcessStreamInternal_Failure(t *testing.T) { tlp := &insertutils.TestLogMessageProcessor{} r := bytes.NewBufferString(data) - if err := processStreamInternal(r, tlp); err == nil { + if err := processStreamInternal(r, "", tlp); err == nil { t.Fatalf("expecting non-nil error") } } diff --git a/docs/VictoriaLogs/CHANGELOG.md b/docs/VictoriaLogs/CHANGELOG.md index 47a366df2..967bb478b 100644 --- a/docs/VictoriaLogs/CHANGELOG.md +++ b/docs/VictoriaLogs/CHANGELOG.md @@ -19,6 +19,8 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta ## tip +* FEATURE: allow configuring multiple receivers with distinct configs for syslog messages. See [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/#multiple-configs). + * BUGFIX: properly read syslog messages over TCP and TLS connections according to [RFC5425](https://datatracker.ietf.org/doc/html/rfc5425) when [data ingestion for syslog protocol](https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/) is enabled. ## [v0.20.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.20.0-victorialogs) diff --git a/docs/VictoriaLogs/README.md b/docs/VictoriaLogs/README.md index 4737c30a6..44f2d6c9b 100644 --- a/docs/VictoriaLogs/README.md +++ b/docs/VictoriaLogs/README.md @@ -140,34 +140,38 @@ via [VictoriaMetrics community channels](https://docs.victoriametrics.com/#commu Pass `-help` to VictoriaLogs in order to see the list of supported command-line flags with their description: ``` + -blockcache.missesBeforeCaching int + The number of cache misses before putting the block into cache. Higher values may reduce indexdb/dataBlocks cache size at the cost of higher CPU and disk read usage (default 2) -cacheExpireDuration duration Items are removed from in-memory caches after they aren't accessed for this duration. Lower values may reduce memory usage at the cost of higher CPU usage. See also -prevCacheRemovalPercent (default 30m0s) + -elasticsearch.version string + Elasticsearch version to report to client (default "8.9.0") -enableTCP6 Whether to enable IPv6 for listening and dialing. By default, only IPv4 TCP and UDP are used -envflag.enable Whether to enable reading flags from environment variables in addition to the command line. Command line flag values have priority over values from environment vars. Flags are read only from the command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details -envflag.prefix string Prefix for environment variables if -envflag.enable is set + -filestream.disableFadvise + Whether to disable fadvise() syscall when reading large data files. The fadvise() syscall prevents from eviction of recently accessed data from OS page cache during background merges and backups. In some rare cases it is better to disable the syscall if it uses too much CPU -flagsAuthKey value Auth key for /flags endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings - Flag value can be read from the given file when using -flagsAuthKey=file:///abs/path/to/file or -flagsAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -flagsAuthKey=http://host/path or -flagsAuthKey=https://host/path + Flag value can be read from the given file when using -flagsAuthKey=file:///abs/path/to/file or -flagsAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -flagsAuthKey=http://host/path or -flagsAuthKey=https://host/path -fs.disableMmap Whether to use pread() instead of mmap() for reading data files. By default, mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread() -futureRetention value Log entries with timestamps bigger than now+futureRetention are rejected during data ingestion; see https://docs.victoriametrics.com/victorialogs/#retention - The following optional suffixes are supported: h (hour), d (day), w (week), y (year). If suffix isn't set, then the duration is counted in months (default 2d) - -gogc int - GOGC to use. See https://tip.golang.org/doc/gc-guide (default 100) + The following optional suffixes are supported: s (second), m (minute), h (hour), d (day), w (week), y (year). If suffix isn't set, then the duration is counted in months (default 2d) -http.connTimeout duration - Incoming connections to -httpListenAddr are closed after the configured timeout. This may help evenly spreading load among a cluster of services behind TCP-level load balancer. Zero value disables closing of incoming connections (default 2m0s) + Incoming connections to -httpListenAddr are closed after the configured timeout. This may help evenly spreading load among a cluster of services behind TCP-level load balancer. Zero value disables closing of incoming connections (default 2m0s) -http.disableResponseCompression Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth -http.header.csp string - Value for 'Content-Security-Policy' header + Value for 'Content-Security-Policy' header, recommended: "default-src 'self'" -http.header.frameOptions string - Value for 'X-Frame-Options' header + Value for 'X-Frame-Options' header -http.header.hsts string - Value for 'Strict-Transport-Security' header + Value for 'Strict-Transport-Security' header, recommended: 'max-age=31536000; includeSubDomains' -http.idleConnTimeout duration Timeout for incoming idle http connections (default 1m0s) -http.maxGracefulShutdownDuration duration @@ -178,13 +182,17 @@ Pass `-help` to VictoriaLogs in order to see the list of supported command-line Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers -httpAuth.password value Password for HTTP server's Basic Auth. The authentication is disabled if -httpAuth.username is empty - Flag value can be read from the given file when using -httpAuth.password=file:///abs/path/to/file or -httpAuth.password=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -httpAuth.password=http://host/path or -httpAuth.password=https://host/path + Flag value can be read from the given file when using -httpAuth.password=file:///abs/path/to/file or -httpAuth.password=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -httpAuth.password=http://host/path or -httpAuth.password=https://host/path -httpAuth.username string Username for HTTP server's Basic Auth. The authentication is disabled if empty. See also -httpAuth.password - -httpListenAddr string - TCP address to listen for http connections. See also -httpListenAddr.useProxyProtocol (default ":9428") - -httpListenAddr.useProxyProtocol - Whether to use proxy protocol for connections accepted at -httpListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing + -httpListenAddr array + TCP address to listen for incoming http requests. See also -httpListenAddr.useProxyProtocol + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -httpListenAddr.useProxyProtocol array + Whether to use proxy protocol for connections accepted at the given -httpListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing + Supports array of values separated by comma or specified via multiple flags. + Empty values are set to false. -inmemoryDataFlushInterval duration The interval for guaranteed saving of in-memory data to disk. The saved data survives unclean shutdowns such as OOM crash, hardware reset, SIGKILL, etc. Bigger intervals may help increase the lifetime of flash storage with limited write cycles (e.g. Raspberry PI). Smaller intervals increase disk IO load. Minimum supported value is 1s (default 5s) -insert.maxFieldsPerLine int @@ -214,6 +222,8 @@ Pass `-help` to VictoriaLogs in order to see the list of supported command-line Allows renaming fields in JSON formatted logs. Example: "ts:timestamp,msg:message" renames "ts" to "timestamp" and "msg" to "message". Supported fields: ts, level, caller, msg -loggerLevel string Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "INFO") + -loggerMaxArgLen int + The maximum length of a single logged argument. Longer arguments are replaced with 'arg_start..arg_end', where 'arg_start' and 'arg_end' is prefix and suffix of the arg with the length not exceeding -loggerMaxArgLen / 2 (default 1000) -loggerOutput string Output for the logs. Supported values: stderr, stdout (default "stderr") -loggerTimezone string @@ -221,52 +231,116 @@ Pass `-help` to VictoriaLogs in order to see the list of supported command-line -loggerWarnsPerSecondLimit int Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit -maxConcurrentInserts int - The maximum number of concurrent insert requests. The default value should work for most cases, since it minimizes memory usage. The default value can be increased when clients send data over slow networks. See also -insert.maxQueueDuration. + The maximum number of concurrent insert requests. Set higher value when clients send data over slow networks. Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage. See also -insert.maxQueueDuration (default 32) -memory.allowedBytes size Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0) -memory.allowedPercent float Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache which will result in higher disk IO usage (default 60) + -metrics.exposeMetadata + Whether to expose TYPE and HELP metadata at the /metrics page, which is exposed at -httpListenAddr . The metadata may be needed when the /metrics page is consumed by systems, which require this information. For example, Managed Prometheus in Google Cloud - https://cloud.google.com/stackdriver/docs/managed-prometheus/troubleshooting#missing-metric-type -metricsAuthKey value Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings - Flag value can be read from the given file when using -metricsAuthKey=file:///abs/path/to/file or -metricsAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -metricsAuthKey=http://host/path or -metricsAuthKey=https://host/path + Flag value can be read from the given file when using -metricsAuthKey=file:///abs/path/to/file or -metricsAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -metricsAuthKey=http://host/path or -metricsAuthKey=https://host/path -pprofAuthKey value Auth key for /debug/pprof/* endpoints. It must be passed via authKey query arg. It overrides httpAuth.* settings - Flag value can be read from the given file when using -pprofAuthKey=file:///abs/path/to/file or -pprofAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -pprofAuthKey=http://host/path or -pprofAuthKey=https://host/path + Flag value can be read from the given file when using -pprofAuthKey=file:///abs/path/to/file or -pprofAuthKey=file://./relative/path/to/file . Flag value can be read from the given http/https url when using -pprofAuthKey=http://host/path or -pprofAuthKey=https://host/path -prevCacheRemovalPercent float Items in the previous caches are removed when the percent of requests it serves becomes lower than this value. Higher values reduce memory usage at the cost of higher CPU usage. See also -cacheExpireDuration (default 0.1) + -pushmetrics.disableCompression + Whether to disable request body compression when pushing metrics to every -pushmetrics.url -pushmetrics.extraLabel array - Optional labels to add to metrics pushed to -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to -pushmetrics.url + Optional labels to add to metrics pushed to every -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to every -pushmetrics.url Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -pushmetrics.header array + Optional HTTP request header to send to every -pushmetrics.url . For example, -pushmetrics.header='Authorization: Basic foobar' adds 'Authorization: Basic foobar' header to every request to every -pushmetrics.url + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -pushmetrics.interval duration - Interval for pushing metrics to -pushmetrics.url (default 10s) + Interval for pushing metrics to every -pushmetrics.url (default 10s) -pushmetrics.url array Optional URL to push metrics exposed at /metrics page. See https://docs.victoriametrics.com/#push-metrics . By default, metrics exposed at /metrics page aren't pushed to any remote storage Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -retentionPeriod value Log entries with timestamps older than now-retentionPeriod are automatically deleted; log entries with timestamps outside the retention are also rejected during data ingestion; the minimum supported retention is 1d (one day); see https://docs.victoriametrics.com/victorialogs/#retention - The following optional suffixes are supported: h (hour), d (day), w (week), y (year). If suffix isn't set, then the duration is counted in months (default 7d) + The following optional suffixes are supported: s (second), m (minute), h (hour), d (day), w (week), y (year). If suffix isn't set, then the duration is counted in months (default 7d) -search.maxConcurrentRequests int - The maximum number of concurrent search requests. It shouldn't be high, since a single request can saturate all the CPU cores, while many concurrently executed requests may require high amounts of memory. See also -search.maxQueueDuration (default 6) + The maximum number of concurrent search requests. It shouldn't be high, since a single request can saturate all the CPU cores, while many concurrently executed requests may require high amounts of memory. See also -search.maxQueueDuration (default 16) -search.maxQueryDuration duration The maximum duration for query execution (default 30s) -search.maxQueueDuration duration The maximum time the search request waits for execution when -search.maxConcurrentRequests limit is reached; see also -search.maxQueryDuration (default 10s) - -storageDataPath string - Path to directory with the VictoriaLogs data; see https://docs.victoriametrics.com/victorialogs/#storage (default "victoria-logs-data") -storage.minFreeDiskSpaceBytes size The minimum free disk space at -storageDataPath after which the storage stops accepting new data Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 10000000) - -tls - Whether to enable TLS for incoming HTTP requests at -httpListenAddr (aka https). -tlsCertFile and -tlsKeyFile must be set if -tls is set - -tlsCertFile string - Path to file with TLS certificate if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated + -storageDataPath string + Path to directory with the VictoriaLogs data; see https://docs.victoriametrics.com/victorialogs/#storage (default "victoria-logs-data") + -syslog.compressMethod.tcp array + Compression method for syslog messages received at the corresponding -syslog.listenAddr.tcp. Supported values: none, gzip, deflate. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/ + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -syslog.compressMethod.udp array + Compression method for syslog messages received at the corresponding -syslog.listenAddr.udp. Supported values: none, gzip, deflate. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/ + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -syslog.listenAddr.tcp array + Comma-separated list of TCP addresses to listen to for Syslog messages. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/ + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -syslog.listenAddr.udp array + Comma-separated list of UDP address to listen to for Syslog messages. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/ + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -syslog.tenantID.tcp array + TenantID for logs ingested via the corresponding -syslog.listenAddr.tcp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/ + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -syslog.tenantID.udp array + TenantID for logs ingested via the corresponding -syslog.listenAddr.udp. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/ + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -syslog.timezone string + Timezone to use when parsing timestamps in RFC3164 syslog messages. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 . See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/ (default "Local") + -syslog.tls array + Whether to enable TLS for receiving syslog messages at the corresponding -syslog.listenAddr.tcp. The corresponding -syslog.tlsCertFile and -syslog.tlsKeyFile must be set if -syslog.tls is set. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/ + Supports array of values separated by comma or specified via multiple flags. + Empty values are set to false. + -syslog.tlsCertFile array + Path to file with TLS certificate for the corresponding -syslog.listenAddr.tcp if the corresponding -syslog.tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/ + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -syslog.tlsCipherSuites array + Optional list of TLS cipher suites for -syslog.listenAddr.tcp if -syslog.tls is set. See the list of supported cipher suites at https://pkg.go.dev/crypto/tls#pkg-constants . See also https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/ + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -syslog.tlsKeyFile array + Path to file with TLS key for the corresponding -syslog.listenAddr.tcp if the corresponding -syslog.tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/ + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -syslog.tlsMinVersion string + The minimum TLS version to use for -syslog.listenAddr.tcp if -syslog.tls is set. Supported values: TLS10, TLS11, TLS12, TLS13. See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/ (default "TLS13") + -tls array + Whether to enable TLS for incoming HTTP requests at the given -httpListenAddr (aka https). -tlsCertFile and -tlsKeyFile must be set if -tls is set. See also -mtls + Supports array of values separated by comma or specified via multiple flags. + Empty values are set to false. + -tlsCertFile array + Path to file with TLS certificate for the corresponding -httpListenAddr if -tls is set. Prefer ECDSA certs instead of RSA certs as RSA certs are slower. The provided certificate file is automatically re-read every second, so it can be dynamically updated. See also -tlsAutocertHosts + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -tlsCipherSuites array Optional list of TLS cipher suites for incoming requests over HTTPS if -tls is set. See the list of supported cipher suites at https://pkg.go.dev/crypto/tls#pkg-constants Supports an array of values separated by comma or specified via multiple flags. - -tlsKeyFile string - Path to file with TLS key if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated - -tlsMinVersion string - Optional minimum TLS version to use for incoming requests over HTTPS if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13 + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -tlsKeyFile array + Path to file with TLS key for the corresponding -httpListenAddr if -tls is set. The provided key file is automatically re-read every second, so it can be dynamically updated. See also -tlsAutocertHosts + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. + -tlsMinVersion array + Optional minimum TLS version to use for the corresponding -httpListenAddr if -tls is set. Supported values: TLS10, TLS11, TLS12, TLS13 + Supports an array of values separated by comma or specified via multiple flags. + Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces. -version + Show VictoriaMetrics version ``` diff --git a/docs/VictoriaLogs/data-ingestion/syslog.md b/docs/VictoriaLogs/data-ingestion/syslog.md index 762856589..86f4de3b0 100644 --- a/docs/VictoriaLogs/data-ingestion/syslog.md +++ b/docs/VictoriaLogs/data-ingestion/syslog.md @@ -80,8 +80,8 @@ starts VictoriaLogs, which accepts TLS-encrypted syslog messages at TCP port 514 ## Compression By default VictoriaLogs accepts uncompressed log messages in Syslog format at `-syslog.listenAddr.tcp` and `-syslog.listenAddr.udp` addresses. -It is possible configuring VictoriaLogs to accept compressed log messages via `-syslog.compressMethod` command-line flag. The following -compression methods are supported: +It is possible configuring VictoriaLogs to accept compressed log messages via `-syslog.compressMethod.tcp` and `-syslog.compressMethod.udp` command-line flags. +The following compression methods are supported: - `none` - no compression - `gzip` - [gzip compression](https://en.wikipedia.org/wiki/Gzip) @@ -90,17 +90,31 @@ compression methods are supported: For example, the following command starts VictoriaLogs, which accepts gzip-compressed syslog messages at TCP port 514: ```sh -./victoria-logs -syslog.listenAddr.tcp=:514 -syslog.compressMethod=gzip +./victoria-logs -syslog.listenAddr.tcp=:514 -syslog.compressMethod.tcp=gzip ``` ## Multitenancy By default, the ingested logs are stored in the `(AccountID=0, ProjectID=0)` [tenant](https://docs.victoriametrics.com/victorialogs/#multitenancy). -If you need storing logs in other tenant, then specify the needed tenant via `-syslog.tenantID` command-line flag. +If you need storing logs in other tenant, then specify the needed tenant via `-syslog.tenantID.tcp` or `-syslog.tenantID.udp` command-line flags +depending on whether TCP or UDP ports are listened for syslog messages. For example, the following command starts VictoriaLogs, which writes syslog messages received at TCP port 514, to `(AccountID=12, ProjectID=34)` tenant: ```sh -./victoria-logs -syslog.listenAddr.tcp=:514 -syslog.tenantID=12:34 +./victoria-logs -syslog.listenAddr.tcp=:514 -syslog.tenantID.tcp=12:34 +``` + +## Multiple configs + +VictoriaLogs can accept syslog messages via multiple TCP and UDP ports with individual configurations for [compression](#compression), [security](#security) +and [multitenancy](#multitenancy). Specify multiple command-line flags for this. For example, the following command starts VictoriaLogs, +which accepts gzip-compressed syslog messages via TCP port 514 at localhost interface and stores them to [tenant](https://docs.victoriametrics.com/victorialogs/#multitenancy) `123:0`, +plus it accepts TLS-encrypted syslog messages via TCP port 6514 and stores them to [tenant](https://docs.victoriametrics.com/victorialogs/#multitenancy) `567:0`: + +```sh +./victoria-logs \ + -syslog.listenAddr.tcp=localhost:514 -syslog.tenantID.tcp=123:0 -syslog.compressMethod.tcp=gzip -syslog.tls=false -syslog.tlsKeyFile='' -syslog.tlsCertFile='' \ + -syslog.listenAddr.tcp=:6514 -syslog.tenantID.tcp=567:0 -syslog.compressMethod.tcp=none -syslog.tls=true -syslog.tlsKeyFile=/path/to/tls/key -syslog.tlsCertFile=/path/to/tls/cert ``` ## Rsyslog diff --git a/lib/logstorage/tenant_id.go b/lib/logstorage/tenant_id.go index 1d406b875..101f53bd7 100644 --- a/lib/logstorage/tenant_id.go +++ b/lib/logstorage/tenant_id.go @@ -79,12 +79,17 @@ func GetTenantIDFromRequest(r *http.Request) (TenantID, error) { return tenantID, nil } -// GetTenantIDFromString returns tenantID from s. -// String is expected in the form of accountID:projectID -func GetTenantIDFromString(s string) (TenantID, error) { +// ParseTenantID returns tenantID from s. +// +// s is expected in the form of accountID:projectID. If s is empty, then zero tenantID is returned. +func ParseTenantID(s string) (TenantID, error) { var tenantID TenantID - colon := strings.Index(s, ":") - if colon < 0 { + if s == "" { + return tenantID, nil + } + + n := strings.Index(s, ":") + if n < 0 { account, err := getUint32FromString(s) if err != nil { return tenantID, fmt.Errorf("cannot parse accountID from %q: %w", s, err) @@ -94,13 +99,13 @@ func GetTenantIDFromString(s string) (TenantID, error) { return tenantID, nil } - account, err := getUint32FromString(s[:colon]) + account, err := getUint32FromString(s[:n]) if err != nil { return tenantID, fmt.Errorf("cannot parse accountID part from %q: %w", s, err) } tenantID.AccountID = account - project, err := getUint32FromString(s[colon+1:]) + project, err := getUint32FromString(s[n+1:]) if err != nil { return tenantID, fmt.Errorf("cannot parse projectID part from %q: %w", s, err) } diff --git a/lib/logstorage/tenant_id_test.go b/lib/logstorage/tenant_id_test.go index 6969a5490..f440e5d8e 100644 --- a/lib/logstorage/tenant_id_test.go +++ b/lib/logstorage/tenant_id_test.go @@ -123,11 +123,11 @@ func TestTenantIDLessEqual(t *testing.T) { } } -func Test_GetTenantIDFromString(t *testing.T) { +func TestParseTenantID(t *testing.T) { f := func(tenant string, expected TenantID) { t.Helper() - got, err := GetTenantIDFromString(tenant) + got, err := ParseTenantID(tenant) if err != nil { t.Errorf("unexpected error: %s", err) return