app/vlogscli: add ability to display query results in logfmt, single-line and multi-line json modes

app/vlogscli/json_prettifier.go

@@ -1,33 +1,70 @@
 package main
 
 import (
+	"bufio"
 	"encoding/json"
 	"fmt"
 	"io"
 	"sort"
 	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )
 
+type outputMode int
+
+const (
+	outputModeJSONMultiline  = outputMode(0)
+	outputModeJSONSingleline = outputMode(1)
+	outputModeLogfmt         = outputMode(2)
+)
+
+func getOutputFormatter(outputMode outputMode) func(w io.Writer, fields []logstorage.Field) error {
+	switch outputMode {
+	case outputModeJSONMultiline:
+		return func(w io.Writer, fields []logstorage.Field) error {
+			return writeJSONObject(w, fields, true)
+		}
+	case outputModeJSONSingleline:
+		return func(w io.Writer, fields []logstorage.Field) error {
+			return writeJSONObject(w, fields, false)
+		}
+	case outputModeLogfmt:
+		return writeLogfmtObject
+	default:
+		panic(fmt.Errorf("BUG: unexpected outputMode=%d", outputMode))
+	}
+}
+
 type jsonPrettifier struct {
-	rOriginal io.ReadCloser
+	r         io.ReadCloser
+	formatter func(w io.Writer, fields []logstorage.Field) error
 
 	d *json.Decoder
 
 	pr *io.PipeReader
 	pw *io.PipeWriter
+	bw *bufio.Writer
 
 	wg sync.WaitGroup
 }
 
-func newJSONPrettifier(r io.ReadCloser) *jsonPrettifier {
+func newJSONPrettifier(r io.ReadCloser, outputMode outputMode) *jsonPrettifier {
 	d := json.NewDecoder(r)
 	pr, pw := io.Pipe()
+	bw := bufio.NewWriter(pw)
+
+	formatter := getOutputFormatter(outputMode)
 
 	jp := &jsonPrettifier{
-		rOriginal: r,
-		d:         d,
-		pr:        pr,
-		pw:        pw,
+		r:         r,
+		formatter: formatter,
+
+		d: d,
+
+		pr: pr,
+		pw: pw,
+		bw: bw,
 	}
 
 	jp.wg.Add(1)
@@ -47,20 +84,23 @@ func (jp *jsonPrettifier) closePipesWithError(err error) {
 
 func (jp *jsonPrettifier) prettifyJSONLines() error {
 	for jp.d.More() {
-		kvs, err := readNextJSONObject(jp.d)
+		fields, err := readNextJSONObject(jp.d)
 		if err != nil {
 			return err
 		}
-		if err := writeJSONObject(jp.pw, kvs); err != nil {
+		sort.Slice(fields, func(i, j int) bool {
+			return fields[i].Name < fields[j].Name
+		})
+		if err := jp.formatter(jp.bw, fields); err != nil {
 			return err
 		}
 	}
-	return nil
+	return jp.bw.Flush()
 }
 
 func (jp *jsonPrettifier) Close() error {
 	jp.closePipesWithError(io.ErrUnexpectedEOF)
-	err := jp.rOriginal.Close()
+	err := jp.r.Close()
 	jp.wg.Wait()
 	return err
 }
@@ -69,7 +109,7 @@ func (jp *jsonPrettifier) Read(p []byte) (int, error) {
 	return jp.pr.Read(p)
 }
 
-func readNextJSONObject(d *json.Decoder) ([]kv, error) {
+func readNextJSONObject(d *json.Decoder) ([]logstorage.Field, error) {
 	t, err := d.Token()
 	if err != nil {
 		return nil, fmt.Errorf("cannot read '{': %w", err)
@@ -79,7 +119,7 @@ func readNextJSONObject(d *json.Decoder) ([]kv, error) {
 		return nil, fmt.Errorf("unexpected token read; got %q; want '{'", delim)
 	}
 
-	var kvs []kv
+	var fields []logstorage.Field
 	for {
 		// Read object key
 		t, err := d.Token()
@@ -89,7 +129,7 @@ func readNextJSONObject(d *json.Decoder) ([]kv, error) {
 		delim, ok := t.(json.Delim)
 		if ok {
 			if delim.String() == "}" {
-				return kvs, nil
+				return fields, nil
 			}
 			return nil, fmt.Errorf("unexpected delimiter read; got %q; want '}'", delim)
 		}
@@ -108,41 +148,56 @@ func readNextJSONObject(d *json.Decoder) ([]kv, error) {
 			return nil, fmt.Errorf("unexpected token read for oject value: %v; want string", t)
 		}
 
-		kvs = append(kvs, kv{
-			key:   key,
-			value: value,
+		fields = append(fields, logstorage.Field{
+			Name:  key,
+			Value: value,
 		})
 	}
 }
 
-func writeJSONObject(w io.Writer, kvs []kv) error {
-	if len(kvs) == 0 {
+func writeLogfmtObject(w io.Writer, fields []logstorage.Field) error {
+	data := logstorage.MarshalFieldsToLogfmt(nil, fields)
+	_, err := fmt.Fprintf(w, "%s\n", data)
+	return err
+}
+
+func writeJSONObject(w io.Writer, fields []logstorage.Field, isMultiline bool) error {
+	if len(fields) == 0 {
 		fmt.Fprintf(w, "{}\n")
 		return nil
 	}
 
-	sort.Slice(kvs, func(i, j int) bool {
-		return kvs[i].key < kvs[j].key
-	})
-
-	fmt.Fprintf(w, "{\n")
-	if err := writeJSONObjectKeyValue(w, kvs[0]); err != nil {
+	fmt.Fprintf(w, "{")
+	writeNewlineIfNeeded(w, isMultiline)
+	if err := writeJSONObjectKeyValue(w, fields[0], isMultiline); err != nil {
 		return err
 	}
-	for _, kv := range kvs[1:] {
-		fmt.Fprintf(w, ",\n")
-		if err := writeJSONObjectKeyValue(w, kv); err != nil {
+	for _, f := range fields[1:] {
+		fmt.Fprintf(w, ",")
+		writeNewlineIfNeeded(w, isMultiline)
+		if err := writeJSONObjectKeyValue(w, f, isMultiline); err != nil {
 			return err
 		}
 	}
-	fmt.Fprintf(w, "\n}\n")
+	writeNewlineIfNeeded(w, isMultiline)
+	fmt.Fprintf(w, "}\n")
 	return nil
 }
 
-func writeJSONObjectKeyValue(w io.Writer, kv kv) error {
-	key := getJSONString(kv.key)
-	value := getJSONString(kv.value)
-	_, err := fmt.Fprintf(w, "  %s: %s", key, value)
+func writeNewlineIfNeeded(w io.Writer, isMultiline bool) {
+	if isMultiline {
+		fmt.Fprintf(w, "\n")
+	}
+}
+
+func writeJSONObjectKeyValue(w io.Writer, f logstorage.Field, isMultiline bool) error {
+	key := getJSONString(f.Name)
+	value := getJSONString(f.Value)
+	if isMultiline {
+		_, err := fmt.Fprintf(w, "  %s: %s", key, value)
+		return err
+	}
+	_, err := fmt.Fprintf(w, "%s:%s", key, value)
 	return err
 }
 
@@ -153,8 +208,3 @@ func getJSONString(s string) string {
 	}
 	return string(data)
 }
-
-type kv struct {
-	key   string
-	value string
-}

app/vlogscli/main.go

@@ -86,6 +86,7 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
 		}
 	}
 
+	outputMode := outputModeJSONMultiline
 	s := ""
 	for {
 		line, err := rl.ReadLine()
@@ -94,7 +95,7 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
 			case io.EOF:
 				if s != "" {
 					// This is non-interactive query execution.
-					if err := executeQuery(context.Background(), rl, s); err != nil {
+					if err := executeQuery(context.Background(), rl, s, outputMode); err != nil {
 						fmt.Fprintf(rl, "%s\n", err)
 					}
 				}
@@ -116,14 +117,43 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
 		}
 
 		s += line
-		if isQuitCommand(s) {
-			fmt.Fprintf(rl, "bye!\n")
-			return
-		}
 		if s == "" {
 			// Skip empty lines
 			continue
 		}
+
+		if isQuitCommand(s) {
+			fmt.Fprintf(rl, "bye!\n")
+			_ = pushToHistory(rl, historyLines, s)
+			return
+		}
+		if isHelpCommand(s) {
+			printCommandsHelp(rl)
+			historyLines = pushToHistory(rl, historyLines, s)
+			s = ""
+			continue
+		}
+		if s == `\s` {
+			fmt.Fprintf(rl, "singleline json output mode\n")
+			outputMode = outputModeJSONSingleline
+			historyLines = pushToHistory(rl, historyLines, s)
+			s = ""
+			continue
+		}
+		if s == `\m` {
+			fmt.Fprintf(rl, "multiline json output mode\n")
+			outputMode = outputModeJSONMultiline
+			historyLines = pushToHistory(rl, historyLines, s)
+			s = ""
+			continue
+		}
+		if s == `\logfmt` {
+			fmt.Fprintf(rl, "logfmt output mode\n")
+			outputMode = outputModeLogfmt
+			historyLines = pushToHistory(rl, historyLines, s)
+			s = ""
+			continue
+		}
 		if line != "" && !strings.HasSuffix(line, ";") {
 			// Assume the query is incomplete and allow the user finishing the query on the next line
 			s += "\n"
@@ -133,7 +163,7 @@ func runReadlineLoop(rl *readline.Instance, incompleteLine *string) {
 
 		// Execute the query
 		ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
-		err = executeQuery(ctx, rl, s)
+		err = executeQuery(ctx, rl, s, outputMode)
 		cancel()
 
 		if err != nil {
@@ -205,14 +235,33 @@ func saveToHistory(filePath string, lines []string) error {
 
 func isQuitCommand(s string) bool {
 	switch s {
-	case "q", "quit", "exit", "\\q":
+	case `\q`, "q", "quit", "exit":
 		return true
 	default:
 		return false
 	}
 }
 
-func executeQuery(ctx context.Context, output io.Writer, s string) error {
+func isHelpCommand(s string) bool {
+	switch s {
+	case `\h`, "h", "help", "?":
+		return true
+	default:
+		return false
+	}
+}
+
+func printCommandsHelp(w io.Writer) {
+	fmt.Fprintf(w, "%s", `List of available commands:
+\q - quit
+\h - show this help
+\s - singleline json output mode
+\m - multiline json output mode
+\logfmt - logfmt output mode
+`)
+}
+
+func executeQuery(ctx context.Context, output io.Writer, s string, outputMode outputMode) error {
 	// Parse the query and convert it to canonical view.
 	s = strings.TrimSuffix(s, ";")
 	q, err := logstorage.ParseQuery(s)
@@ -257,7 +306,7 @@ func executeQuery(ctx context.Context, output io.Writer, s string) error {
 	}
 
 	// Prettify the response and stream it to 'less'.
-	jp := newJSONPrettifier(resp.Body)
+	jp := newJSONPrettifier(resp.Body, outputMode)
 	defer func() {
 		_ = jp.Close()
 	}()

docs/VictoriaLogs/CHANGELOG.md

@@ -15,6 +15,7 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta
 
 ## tip
 
+* FEATURE: [vlogscli](https://docs.victoriametrics.com/victorialogs/querying/vlogscli/): add ability to display results in `logfmt` mode, single-line and multi-line JSON modes according [these docs](https://docs.victoriametrics.com/victorialogs/querying/vlogscli/#output-modes).
 * FEATURE: [vlogscli](https://docs.victoriametrics.com/victorialogs/querying/vlogscli/): preserve `less` output after the exit from scrolling mode. This should help re-using previous query results in subsequent queries.
 * FEATURE: add [`len` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#len-pipe) for calculating the length for the given [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) value in bytes.
 

docs/VictoriaLogs/querying/vlogscli.md

@@ -57,7 +57,8 @@ duration: 0.688s
 
 Query execution can be interrupted at any time by pressing `Ctrl+C`.
 
-Type `q`, `quit` or `exit` and then press `Enter` for exit from `vlogsql`.
+Type `q` and then press `Enter` for exit from `vlogsql` (if you want to search for `q` [word](https://docs.victoriametrics.com/victorialogs/logsql/#word),
+then just wrap it into quotes: `"q"` or `'q'`).
 
 If the query response exceeds vertical screen space, `vlogsql` pipes query response to `less` utility,
 so you can scroll the response as needed. This allows executing queries, which potentially
@@ -88,3 +89,14 @@ Press `Ctrl+R` multiple times for searching other matching queries in the histor
 Press `Enter` when the needed query is found in order to execute it.
 Press `Ctrl+C` for exit from the `search history` mode.
 See also [other available shortcuts](https://github.com/chzyer/readline/blob/f533ef1caae91a1fcc90875ff9a5a030f0237c6a/doc/shortcut.md).
+
+## Output modes
+
+By default `vlogscli` displays query results as prettified JSON object with every field on a separate line.
+Fields in every JSON object are sorted in alphabetical order. This simplifies locating the needed fields.
+
+`vlogscli` supports the following output modes:
+
+* A single JSON line per every result. Type `\s` and press `enter` for this mode.
+* Multline JSON per every result. Type `\m` and press `enter` for this mode.
+* [Logfmt output](https://brandur.org/logfmt). Type `\logfmt` and press `enter` for this mode.