VictoriaMetrics/lib/bytesutil/internstring.go
Roman Khavronenko 80768d53dd
docs: follow-up after aec4b5db81 ()
Signed-off-by: hagen1778 <roman@victoriametrics.com>
2023-07-19 14:48:17 -07:00

85 lines
2.8 KiB
Go

package bytesutil
import (
"flag"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
)
var (
internStringMaxLen = flag.Int("internStringMaxLen", 500, "The maximum length for strings to intern. A lower limit may save memory at the cost of higher CPU usage. "+
"See https://en.wikipedia.org/wiki/String_interning . See also -internStringDisableCache and -internStringCacheExpireDuration")
disableCache = flag.Bool("internStringDisableCache", false, "Whether to disable caches for interned strings. This may reduce memory usage at the cost of higher CPU usage. "+
"See https://en.wikipedia.org/wiki/String_interning . See also -internStringCacheExpireDuration and -internStringMaxLen")
cacheExpireDuration = flag.Duration("internStringCacheExpireDuration", 6*time.Minute, "The expiry duration for caches for interned strings. "+
"See https://en.wikipedia.org/wiki/String_interning . See also -internStringMaxLen and -internStringDisableCache")
)
func isSkipCache(s string) bool {
return *disableCache || len(s) > *internStringMaxLen
}
// InternBytes interns b as a string
func InternBytes(b []byte) string {
s := ToUnsafeString(b)
return InternString(s)
}
// InternString returns interned s.
//
// This may be needed for reducing the amounts of allocated memory.
func InternString(s string) string {
if isSkipCache(s) {
// Make a new copy for s in order to remove references from possible bigger string s refers to.
// This also protects from cases when s points to unsafe string - see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3227
return strings.Clone(s)
}
ct := fasttime.UnixTimestamp()
if v, ok := internStringsMap.Load(s); ok {
e := v.(*ismEntry)
if atomic.LoadUint64(&e.lastAccessTime)+10 < ct {
// Reduce the frequency of e.lastAccessTime update to once per 10 seconds
// in order to improve the fast path speed on systems with many CPU cores.
atomic.StoreUint64(&e.lastAccessTime, ct)
}
return e.s
}
// Make a new copy for s in order to remove references from possible bigger string s refers to.
sCopy := strings.Clone(s)
e := &ismEntry{
lastAccessTime: ct,
s: sCopy,
}
internStringsMap.Store(sCopy, e)
if needCleanup(&internStringsMapLastCleanupTime, ct) {
// Perform a global cleanup for internStringsMap by removing items, which weren't accessed during the last 5 minutes.
m := &internStringsMap
deadline := ct - uint64(cacheExpireDuration.Seconds())
m.Range(func(k, v interface{}) bool {
e := v.(*ismEntry)
if atomic.LoadUint64(&e.lastAccessTime) < deadline {
m.Delete(k)
}
return true
})
}
return sCopy
}
type ismEntry struct {
lastAccessTime uint64
s string
}
var (
internStringsMap sync.Map
internStringsMapLastCleanupTime uint64
)