vmui/logs: improve log display for group view (#6419)

### Describe Your Changes 1) Set the default limit to `50`. #6408 2) Configure the default search to cover the `last 5 minutes` and include all messages (`*`). #6405 3) In the header, display only streams and group by stream. #6406 4) Add log processing, without the fields `msg`, `time`, and `stream`. 5) When clicking on logs, display a list of all fields. #6407 <img width="400" alt="image" src="https://github.com/VictoriaMetrics/VictoriaMetrics/assets/29711459/666dcaa3-20fb-4828-b77b-1d849dd9a8ed"> ### Checklist The following checks are **mandatory**: - [ ] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/).
2024-12-12 12:46:23 +01:00 · 2024-06-06 12:14:06 +02:00 · 2024-06-06 12:14:06 +02:00 · 8cf417e1c7
commit 8cf417e1c7
parent 362ee240cd
11 changed files with 372 additions and 130 deletions
--- a/app/vmui/packages/vmui/src/assets/MetricsQL.md
+++ b/app/vmui/packages/vmui/src/assets/MetricsQL.md
@ -328,6 +328,7 @@ See also [increases_over_time](#increases_over_time).

 `default_rollup(series_selector[d])` is a [rollup function](#rollup-functions), which returns the last [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 value on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
+Compared to [last_over_time](#last_over_time) it accounts for [staleness markers](https://docs.victoriametrics.com/vmagent/#prometheus-staleness-markers) to detect stale series.

 If the lookbehind window is skipped in square brackets, then it is automatically calculated as `max(step, scrape_interval)`, where `step` is the query arg value
 passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyconcepts/#range-query) or [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query),
--- a/app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogs.tsx
+++ b/app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogs.tsx
@ -14,7 +14,7 @@ import { useTimeState } from "../../state/time/TimeStateContext";
 import { getFromStorage, saveToStorage } from "../../utils/storage";

 const storageLimit = Number(getFromStorage("LOGS_LIMIT"));
-const defaultLimit = isNaN(storageLimit) ? 1000 : storageLimit;
+const defaultLimit = isNaN(storageLimit) ? 50 : storageLimit;

 const ExploreLogs: FC = () => {
  const { serverUrl } = useAppState();
@ -22,7 +22,7 @@ const ExploreLogs: FC = () => {
  const { setSearchParamsFromKeys } = useSearchParamsFromObject();

  const [limit, setLimit] = useStateSearchParams(defaultLimit, "limit");
-  const [query, setQuery] = useStateSearchParams("", "query");
+  const [query, setQuery] = useStateSearchParams("*", "query");
  const { logs, isLoading, error, fetchLogs } = useFetchLogs(serverUrl, query, limit);
  const [queryError, setQueryError] = useState<ErrorTypes | string>("");
  const [loaded, isLoaded] = useState(false);
--- a/app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/ExploreLogsBody.tsx
+++ b/app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/ExploreLogsBody.tsx
@ -13,7 +13,8 @@ import useSearchParamsFromObject from "../../../hooks/useSearchParamsFromObject"
 import TableSettings from "../../../components/Table/TableSettings/TableSettings";
 import useBoolean from "../../../hooks/useBoolean";
 import TableLogs from "./TableLogs";
-import GroupLogs from "./GroupLogs";
+import GroupLogs from "../GroupLogs/GroupLogs";
+import { DATE_TIME_FORMAT } from "../../../constants/date";

 export interface ExploreLogBodyProps {
  data: Logs[];
@ -42,14 +43,14 @@ const ExploreLogsBody: FC<ExploreLogBodyProps> = ({ data, loaded }) => {
  const { value: tableCompact, toggle: toggleTableCompact } = useBoolean(false);

  const logs = useMemo(() => data.map((item) => ({
-    time: dayjs(item._time).tz().format("MMM DD, YYYY \nHH:mm:ss.SSS"),
-    data: JSON.stringify(item, null, 2),
    ...item,
+    _vmui_time: item._time ? dayjs(item._time).tz().format(`${DATE_TIME_FORMAT}.SSS`) : "",
+    _vmui_data: JSON.stringify(item, null, 2),
  })) as Logs[], [data, timezone]);

  const columns = useMemo(() => {
    if (!logs?.length) return [];
-    const hideColumns = ["data", "_time"];
+    const hideColumns = ["_vmui_data", "_vmui_time"];
    const keys = new Set<string>();
    for (const item of logs) {
      for (const key in item) {
--- a/app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/GroupLogs.tsx
+++ b/app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/GroupLogs.tsx
@ -1,65 +0,0 @@
-import React, { FC, useMemo } from "preact/compat";
-import "./style.scss";
-import { Logs } from "../../../api/types";
-import Accordion from "../../../components/Main/Accordion/Accordion";
-import { groupByMultipleKeys } from "../../../utils/array";
-
-interface TableLogsProps {
-  logs: Logs[];
-  columns: string[];
-}
-
-const GroupLogs: FC<TableLogsProps> = ({ logs, columns }) => {
-
-  const groupData = useMemo(() => {
-    const excludeColumns = ["_msg", "time", "data", "_time"];
-    const keys = columns.filter((c) => !excludeColumns.includes(c as string));
-    return groupByMultipleKeys(logs, keys);
-  }, [logs]);
-
-  return (
-    <div className="vm-explore-logs-body-content">
-      {groupData.map((item) => (
-        <div
-          className="vm-explore-logs-body-content-group"
-          key={item.keys.join("")}
-        >
-          <Accordion
-            defaultExpanded={true}
-            title={(
-              <div className="vm-explore-logs-body-content-group-keys">
-                <span className="vm-explore-logs-body-content-group-keys__title">Group by:</span>
-                {item.keys.map((key) => (
-                  <div
-                    className="vm-explore-logs-body-content-group-keys__key"
-                    key={key}
-                  >
-                    {key}
-                  </div>
-                ))}
-              </div>
-            )}
-          >
-            <div className="vm-explore-logs-body-content-group-rows">
-              {item.values.map((value) => (
-                <div
-                  className="vm-explore-logs-body-content-group-rows-item"
-                  key={`${value._msg}${value._time}`}
-                >
-                  <div className="vm-explore-logs-body-content-group-rows-item__time">
-                    {value.time}
-                  </div>
-                  <div className="vm-explore-logs-body-content-group-rows-item__msg">
-                    {value._msg}
-                  </div>
-                </div>
-              ))}
-            </div>
-          </Accordion>
-        </div>
-      ))}
-    </div>
-  );
-};
-
-export default GroupLogs;
--- a/app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/TableLogs.tsx
+++ b/app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/TableLogs.tsx
@ -13,9 +13,9 @@ interface TableLogsProps {
 const TableLogs: FC<TableLogsProps> = ({ logs, displayColumns, tableCompact, columns }) => {
  const getColumnClass = (key: string) => {
    switch (key) {
-      case "time":
+      case "_time":
        return "vm-table-cell_logs-time";
-      case "data":
+      case "_vmui_data":
        return "vm-table-cell_logs vm-table-cell_pre";
      default:
        return "vm-table-cell_logs";
@ -25,9 +25,9 @@ const TableLogs: FC<TableLogsProps> = ({ logs, displayColumns, tableCompact, col
  const tableColumns = useMemo(() => {
    if (tableCompact) {
      return [{
-        key: "data",
+        key: "_vmui_data",
        title: "Data",
-        className: getColumnClass("data")
+        className: getColumnClass("_vmui_data")
      }];
    }
    return columns.map((key) => ({
@ -48,8 +48,8 @@ const TableLogs: FC<TableLogsProps> = ({ logs, displayColumns, tableCompact, col
      <Table
        rows={logs}
        columns={filteredColumns}
-        defaultOrderBy={"time"}
-        copyToClipboard={"data"}
+        defaultOrderBy={"_vmui_time"}
+        copyToClipboard={"_vmui_data"}
        paginationOffset={{ startIndex: 0, endIndex: Infinity }}
      />
    </>
--- a/app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/style.scss
+++ b/app/vmui/packages/vmui/src/pages/ExploreLogs/ExploreLogsBody/style.scss
@ -41,54 +41,4 @@
      min-width: 700px;
    }
  }
-
-  &-content {
-    &-group {
-
-      &-keys {
-        display: flex;
-        align-items: center;
-        flex-wrap: wrap;
-        gap: $padding-small;
-        border-bottom: $border-divider;
-        padding: $padding-global 0;
-
-        &__title {
-          font-weight: bold;
-        }
-
-        &__key {
-          padding: 4px 12px;
-          background-color: $color-primary;
-          color: $color-primary-text;
-          border-radius: $border-radius-small;
-        }
-      }
-
-      &-rows {
-        display: grid;
-
-        &-item {
-          display: grid;
-          grid-template-columns: 107px 1fr;
-          gap: $padding-small;
-          padding: $padding-global 0;
-          border-bottom: $border-divider;
-
-          &__time {
-            display: flex;
-            align-items: center;
-            justify-content: center;
-            line-height: 1.3;
-          }
-
-          &__msg {
-            font-family: $font-family-monospace;
-            overflow-wrap: anywhere;
-            line-height: 1.1;
-          }
-        }
-      }
-    }
-  }
 }
--- a/app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogs.tsx
+++ b/app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogs.tsx
@ -0,0 +1,90 @@
+import React, { FC, useEffect, useMemo } from "preact/compat";
+import { MouseEvent, useState } from "react";
+import "./style.scss";
+import { Logs } from "../../../api/types";
+import Accordion from "../../../components/Main/Accordion/Accordion";
+import { groupByMultipleKeys } from "../../../utils/array";
+import Tooltip from "../../../components/Main/Tooltip/Tooltip";
+import useCopyToClipboard from "../../../hooks/useCopyToClipboard";
+import GroupLogsItem from "./GroupLogsItem";
+
+interface TableLogsProps {
+  logs: Logs[];
+  columns: string[];
+}
+
+const GroupLogs: FC<TableLogsProps> = ({ logs }) => {
+  const copyToClipboard = useCopyToClipboard();
+
+  const [copied, setCopied] = useState<string | null>(null);
+
+  const groupData = useMemo(() => {
+    return groupByMultipleKeys(logs, ["_stream"]).map((item) => {
+      const streamValue = item.values[0]?._stream || "";
+      const pairs = streamValue.slice(1, -1).match(/(?:[^\\,]+|\\,)+?(?=,|$)/g) || [streamValue];
+      return {
+        ...item,
+        pairs: pairs.filter(Boolean),
+      };
+    });
+  }, [logs]);
+
+  const handleClickByPair = (pair: string) => async (e: MouseEvent<HTMLDivElement>) => {
+    e.stopPropagation();
+    const isCopied = await copyToClipboard(`${pair}`);
+    if (isCopied) {
+      setCopied(pair);
+    }
+  };
+
+  useEffect(() => {
+    if (copied === null) return;
+    const timeout = setTimeout(() => setCopied(null), 2000);
+    return () => clearTimeout(timeout);
+  }, [copied]);
+
+  return (
+    <div className="vm-group-logs">
+      {groupData.map((item) => (
+        <div
+          className="vm-group-logs-section"
+          key={item.keys.join("")}
+        >
+          <Accordion
+            defaultExpanded={true}
+            title={(
+              <div className="vm-group-logs-section-keys">
+                <span className="vm-group-logs-section-keys__title">Group by _stream:</span>
+                {item.pairs.map((pair) => (
+                  <Tooltip
+                    title={copied === pair ? "Copied" : "Copy to clipboard"}
+                    key={`${item.keys.join("")}_${pair}`}
+                    placement={"top-center"}
+                  >
+                    <div
+                      className="vm-group-logs-section-keys__pair"
+                      onClick={handleClickByPair(pair)}
+                    >
+                      {pair}
+                    </div>
+                  </Tooltip>
+                ))}
+              </div>
+            )}
+          >
+            <div className="vm-group-logs-section-rows">
+              {item.values.map((value) => (
+                <GroupLogsItem
+                  key={`${value._msg}${value._time}`}
+                  log={value}
+                />
+              ))}
+            </div>
+          </Accordion>
+        </div>
+      ))}
+    </div>
+  );
+};
+
+export default GroupLogs;
--- a/app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogsItem.tsx
+++ b/app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/GroupLogsItem.tsx
@ -0,0 +1,113 @@
+import React, { FC, useEffect, useState } from "preact/compat";
+import { Logs } from "../../../api/types";
+import "./style.scss";
+import useBoolean from "../../../hooks/useBoolean";
+import Button from "../../../components/Main/Button/Button";
+import Tooltip from "../../../components/Main/Tooltip/Tooltip";
+import { ArrowDropDownIcon, CopyIcon } from "../../../components/Main/Icons";
+import useCopyToClipboard from "../../../hooks/useCopyToClipboard";
+import classNames from "classnames";
+
+interface Props {
+  log: Logs;
+}
+
+const GroupLogsItem: FC<Props> = ({ log }) => {
+  const {
+    value: isOpenFields,
+    toggle: toggleOpenFields,
+  } = useBoolean(false);
+
+  const excludeKeys = ["_stream", "_msg", "_time", "_vmui_time", "_vmui_data"];
+  const fields = Object.entries(log).filter(([key]) => !excludeKeys.includes(key));
+  const hasFields = fields.length > 0;
+
+  const copyToClipboard = useCopyToClipboard();
+  const [copied, setCopied] = useState<number | null>(null);
+
+  const createCopyHandler = (copyValue:  string, rowIndex: number) => async () => {
+    if (copied === rowIndex) return;
+    try {
+      await copyToClipboard(copyValue);
+      setCopied(rowIndex);
+    } catch (e) {
+      console.error(e);
+    }
+  };
+
+  useEffect(() => {
+    if (copied === null) return;
+    const timeout = setTimeout(() => setCopied(null), 2000);
+    return () => clearTimeout(timeout);
+  }, [copied]);
+
+  return (
+    <div className="vm-group-logs-row">
+      <div
+        className="vm-group-logs-row-content"
+        onClick={toggleOpenFields}
+        key={`${log._msg}${log._time}`}
+      >
+        {hasFields && (
+          <div
+            className={classNames({
+              "vm-group-logs-row-content__arrow": true,
+              "vm-group-logs-row-content__arrow_open": isOpenFields,
+            })}
+          >
+            <ArrowDropDownIcon/>
+          </div>
+        )}
+        <div
+          className={classNames({
+            "vm-group-logs-row-content__time": true,
+            "vm-group-logs-row-content__time_missing": !log._vmui_time
+          })}
+        >
+          {log._vmui_time || "timestamp missing"}
+        </div>
+        <div
+          className={classNames({
+            "vm-group-logs-row-content__msg": true,
+            "vm-group-logs-row-content__msg_missing": !log._msg
+          })}
+        >
+          {log._msg || "message missing"}
+        </div>
+      </div>
+      {hasFields && isOpenFields && (
+        <div className="vm-group-logs-row-fields">
+          <table>
+            <tbody>
+              {fields.map(([key, value], i) => (
+                <tr
+                  key={key}
+                  className="vm-group-logs-row-fields-item"
+                >
+                  <td className="vm-group-logs-row-fields-item-controls">
+                    <div className="vm-group-logs-row-fields-item-controls__wrapper">
+                      <Tooltip title={copied === i ? "Copied" : "Copy to clipboard"}>
+                        <Button
+                          variant="text"
+                          color="gray"
+                          size="small"
+                          startIcon={<CopyIcon/>}
+                          onClick={createCopyHandler(`${key}: ${value}`, i)}
+                          ariaLabel="copy to clipboard"
+                        />
+                      </Tooltip>
+                    </div>
+                  </td>
+                  <td className="vm-group-logs-row-fields-item__key">{key}</td>
+                  <td className="vm-group-logs-row-fields-item__value">{value}</td>
+                </tr>
+              ))}
+            </tbody>
+          </table>
+        </div>
+      )}
+    </div>
+  );
+};
+
+export default GroupLogsItem;
--- a/app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/style.scss
+++ b/app/vmui/packages/vmui/src/pages/ExploreLogs/GroupLogs/style.scss
@ -0,0 +1,148 @@
+@use "src/styles/variables" as *;
+
+.vm-group-logs {
+  margin-top: calc(-1 * $padding-medium);
+
+  &-section {
+    &-keys {
+      display: flex;
+      align-items: center;
+      flex-wrap: wrap;
+      gap: $padding-small;
+      border-bottom: $border-divider;
+      padding: $padding-small 0;
+
+      &__title {
+        font-weight: bold;
+      }
+
+      &__pair {
+        padding: calc($padding-global / 2) $padding-global;
+        background-color: lighten($color-tropical-blue, 6%);
+        color: darken($color-dodger-blue, 20%);
+        border-radius: $border-radius-medium;
+        transition: background-color 0.3s ease-in, transform 0.1s ease-in;;
+
+        &:hover {
+          background-color: $color-tropical-blue;
+        }
+
+        &:active {
+          transform: translate(0, 3px);
+        }
+      }
+    }
+
+    &-rows {
+      display: grid;
+    }
+  }
+
+  &-row {
+    position: relative;
+    border-bottom: $border-divider;
+
+    &-content {
+      position: relative;
+      display: grid;
+      grid-template-columns: minmax(180px, max-content) 1fr;
+      gap: $padding-small;
+      padding: $padding-global;
+      cursor: pointer;
+      transition: background-color 0.2s ease-in;
+
+      &:hover {
+        background-color: $color-hover-black;
+      }
+
+      &__arrow {
+        position: absolute;
+        top: $padding-global;
+        left: 0;
+        display: flex;
+        align-items: center;
+        justify-content: center;
+        width: 14px;
+        height: 14px;
+        transform: rotate(-90deg);
+        transition: transform 0.2s ease-out;
+
+        &_open {
+          transform: rotate(0deg);
+        }
+      }
+
+      &__time {
+        display: flex;
+        align-items: flex-start;
+        justify-content: flex-end;
+        line-height: 1;
+        white-space: nowrap;
+
+        &_missing {
+          color: $color-text-disabled;
+          font-style: italic;
+          justify-content: center;
+        }
+      }
+
+      &__msg {
+        font-family: $font-family-monospace;
+        overflow-wrap: anywhere;
+        line-height: 1.1;
+
+        &_missing {
+          color: $color-text-disabled;
+          font-style: italic;
+          text-align: center;
+        }
+      }
+    }
+
+    &-fields {
+      grid-row: 2;
+      padding: $padding-small 0;
+      margin-bottom: $padding-small;
+      border: $border-divider;
+      border-radius: $border-radius-small;
+      overflow: auto;
+      max-height: 300px;
+
+      &-item {
+        border-radius: $border-radius-small;
+        transition: background-color 0.2s ease-in;
+
+        &:hover {
+          background-color: $color-hover-black;
+        }
+
+        &-controls {
+          padding: 0;
+
+          &__wrapper {
+            display: flex;
+            align-items: center;
+            justify-content: center;
+          }
+        }
+
+        &__key,
+        &__value {
+          vertical-align: top;
+          padding: calc($padding-small / 2) $padding-global;
+        }
+
+        &__key {
+          overflow-wrap: break-word;
+          width: max-content;
+        }
+
+        &__value {
+          width: 100%;
+          word-break: break-all;
+          white-space: pre-wrap;
+        }
+      }
+    }
+  }
+}
--- a/app/vmui/packages/vmui/src/styles/components/table.scss
+++ b/app/vmui/packages/vmui/src/styles/components/table.scss
@ -78,13 +78,15 @@
    }

    &_logs-time {
-      white-space: pre;
+      white-space: nowrap;
      overflow-wrap: normal;
    }

    &_logs {
      font-family: $font-family-monospace;
      line-height: 1.2;
+      width: 100%;
+      overflow-wrap: normal;
    }
  }

--- a/app/vmui/packages/vmui/src/utils/time.ts
+++ b/app/vmui/packages/vmui/src/utils/time.ts
@ -3,6 +3,7 @@ import dayjs, { UnitTypeShort } from "dayjs";
 import { getQueryStringValue } from "./query-string";
 import { DATE_ISO_FORMAT } from "../constants/date";
 import timezones from "../constants/timezones";
+import { AppType } from "../types/appType";

 const MAX_ITEMS_PER_CHART = window.innerWidth / 4;
 const MAX_ITEMS_PER_HISTOGRAM = window.innerWidth / 40;
@ -159,10 +160,11 @@ export const dateFromSeconds = (epochTimeInSeconds: number): Date => {
 const getYesterday = () => dayjs().tz().subtract(1, "day").endOf("day").toDate();
 const getToday = () => dayjs().tz().endOf("day").toDate();

+const isLogsApp = process.env.REACT_APP_TYPE === AppType.logs;
 export const relativeTimeOptions: RelativeTimeOption[] = [
-  { title: "Last 5 minutes", duration: "5m" },
+  { title: "Last 5 minutes", duration: "5m", isDefault: isLogsApp },
  { title: "Last 15 minutes", duration: "15m" },
-  { title: "Last 30 minutes", duration: "30m", isDefault: true },
+  { title: "Last 30 minutes", duration: "30m", isDefault: !isLogsApp },
  { title: "Last 1 hour", duration: "1h" },
  { title: "Last 3 hours", duration: "3h" },
  { title: "Last 6 hours", duration: "6h" },