glog: add JSON structured logging mode (#8708)
* glog: add JSON structured logging mode Add opt-in JSON output format for glog, enabling integration with log aggregation systems like ELK, Loki, and Datadog. - Add --log_json flag to enable JSON output at startup - Add SetJSONMode()/IsJSONMode() for runtime toggle - Add JSON branches in println, printDepth, printf, printWithFileLine - Use manual JSON construction (no encoding/json) for performance - Add jsonEscapeString() for safe string escaping - Include 8 unit tests and 1 benchmark Enabled via --log_json flag. Default behavior unchanged. * glog: prevent lazy flag init from overriding SetJSONMode If --log_json=true and SetJSONMode(false) was called at runtime, a subsequent IsJSONMode() call would re-enable JSON mode via the sync.Once lazy initialization. Mark jsonFlagOnce as done inside SetJSONMode so the runtime API always takes precedence. * glog: fix RuneError check to not misclassify valid U+FFFD The condition r == utf8.RuneError matches both invalid UTF-8 sequences (size=1) and a valid U+FFFD replacement character (size=3). Without checking size == 1, a valid U+FFFD input would be incorrectly escaped and only advance by 1 byte, corrupting the output. --------- Co-authored-by: Chris Lu <chris.lu@gmail.com>
This commit is contained in:
JARDEL ALVES
@@ -630,7 +630,21 @@ func (buf *buffer) someDigits(i, d int) int {
|
||||
return copy(buf.tmp[i:], buf.tmp[j:])
|
||||
}
|
||||
|
||||
// logJSON is a helper that formats and outputs a JSON log line.
|
||||
// Used by println, printDepth, and printf to avoid code duplication.
|
||||
// depth is incremented by 1 to account for this extra call frame.
|
||||
func (l *loggingT) logJSON(s severity, depth int, msg string) {
|
||||
buf, file, line := l.formatJSON(s, depth+1)
|
||||
buf.WriteString(jsonEscapeString(strings.TrimRight(msg, "\n")))
|
||||
finishJSON(buf)
|
||||
l.output(s, buf, file, line, false)
|
||||
}
|
||||
|
||||
func (l *loggingT) println(s severity, args ...interface{}) {
|
||||
if IsJSONMode() {
|
||||
l.logJSON(s, 0, fmt.Sprintln(args...))
|
||||
return
|
||||
}
|
||||
buf, file, line := l.header(s, 0)
|
||||
fmt.Fprintln(buf, args...)
|
||||
l.output(s, buf, file, line, false)
|
||||
@@ -641,6 +655,10 @@ func (l *loggingT) print(s severity, args ...interface{}) {
|
||||
}
|
||||
|
||||
func (l *loggingT) printDepth(s severity, depth int, args ...interface{}) {
|
||||
if IsJSONMode() {
|
||||
l.logJSON(s, depth, fmt.Sprint(args...))
|
||||
return
|
||||
}
|
||||
buf, file, line := l.header(s, depth)
|
||||
fmt.Fprint(buf, args...)
|
||||
if buf.Bytes()[buf.Len()-1] != '\n' {
|
||||
@@ -650,6 +668,10 @@ func (l *loggingT) printDepth(s severity, depth int, args ...interface{}) {
|
||||
}
|
||||
|
||||
func (l *loggingT) printf(s severity, format string, args ...interface{}) {
|
||||
if IsJSONMode() {
|
||||
l.logJSON(s, 0, fmt.Sprintf(format, args...))
|
||||
return
|
||||
}
|
||||
buf, file, line := l.header(s, 0)
|
||||
fmt.Fprintf(buf, format, args...)
|
||||
if buf.Bytes()[buf.Len()-1] != '\n' {
|
||||
@@ -662,6 +684,36 @@ func (l *loggingT) printf(s severity, format string, args ...interface{}) {
|
||||
// alsoLogToStderr is true, the log message always appears on standard error; it
|
||||
// will also appear in the log file unless --logtostderr is set.
|
||||
func (l *loggingT) printWithFileLine(s severity, file string, line int, alsoToStderr bool, args ...interface{}) {
|
||||
if IsJSONMode() {
|
||||
buf := l.getBuffer()
|
||||
now := timeNow()
|
||||
buf.WriteString(`{"ts":"`)
|
||||
buf.WriteString(now.UTC().Format(time.RFC3339Nano))
|
||||
buf.WriteString(`","level":"`)
|
||||
switch {
|
||||
case s == infoLog:
|
||||
buf.WriteString("INFO")
|
||||
case s == warningLog:
|
||||
buf.WriteString("WARNING")
|
||||
case s == errorLog:
|
||||
buf.WriteString("ERROR")
|
||||
case s >= fatalLog:
|
||||
buf.WriteString("FATAL")
|
||||
}
|
||||
buf.WriteString(`","file":"`)
|
||||
buf.WriteString(jsonEscapeString(file))
|
||||
buf.WriteString(`","line":`)
|
||||
if line < 0 {
|
||||
line = 0
|
||||
}
|
||||
buf.WriteString(itoa(line))
|
||||
buf.WriteString(`,"msg":"`)
|
||||
msg := fmt.Sprint(args...)
|
||||
buf.WriteString(jsonEscapeString(strings.TrimRight(msg, "\n")))
|
||||
finishJSON(buf)
|
||||
l.output(s, buf, file, line, alsoToStderr)
|
||||
return
|
||||
}
|
||||
buf := l.formatHeader(s, file, line)
|
||||
fmt.Fprint(buf, args...)
|
||||
if buf.Bytes()[buf.Len()-1] != '\n' {
|
||||
@@ -873,6 +925,11 @@ func (sb *syncBuffer) rotateFile(now time.Time) error {
|
||||
|
||||
sb.Writer = bufio.NewWriterSize(sb.file, bufferSize)
|
||||
|
||||
// Skip text header in JSON mode to keep files as valid NDJSON.
|
||||
if IsJSONMode() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Write header.
|
||||
var buf bytes.Buffer
|
||||
fmt.Fprintf(&buf, "Log file created at: %s\n", now.Format("2006/01/02 15:04:05"))
|
||||
|
||||
@@ -64,6 +64,10 @@ var logMaxFiles = flag.Int("log_max_files", 5, "Maximum number of log files to k
|
||||
// The default is 168 hours (7 days). Set to 0 to disable time-based rotation.
|
||||
var logRotateHours = flag.Int("log_rotate_hours", 168, "Rotate log files after this many hours (default: 168 = 7 days, 0 = disabled)")
|
||||
|
||||
// logJSON enables JSON-formatted log output (one JSON object per line).
|
||||
// Useful for integration with ELK, Loki, Datadog, and other log aggregation systems.
|
||||
var logJSON = flag.Bool("log_json", false, "Output logs in JSON format instead of glog text format")
|
||||
|
||||
// logCompress enables gzip compression of rotated log files.
|
||||
// Compressed files get a .gz suffix. Compression runs in the background.
|
||||
var logCompress = flag.Bool("log_compress", false, "Gzip-compress rotated log files to save disk space")
|
||||
|
||||
157
weed/glog/glog_json.go
Normal file
157
weed/glog/glog_json.go
Normal file
@@ -0,0 +1,157 @@
|
||||
package glog
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// JSONMode controls whether log output is in JSON format.
|
||||
// 0 = classic glog text (default), 1 = JSON lines.
|
||||
// Safe for concurrent access.
|
||||
var jsonMode int32
|
||||
|
||||
// jsonFlagOnce ensures the --log_json flag is applied on first use,
|
||||
// even when -logtostderr prevents createLogDirs() from running.
|
||||
var jsonFlagOnce sync.Once
|
||||
|
||||
// SetJSONMode enables or disables JSON-formatted log output.
|
||||
// When enabled, each log line is a single JSON object:
|
||||
//
|
||||
// {"ts":"2006-01-02T15:04:05.000000Z","level":"INFO","file":"server.go","line":42,"msg":"..."}
|
||||
//
|
||||
// This is useful for log aggregation systems (ELK, Loki, Datadog, etc).
|
||||
func SetJSONMode(enabled bool) {
|
||||
// Prevent lazy flag initialization from overriding this explicit call.
|
||||
jsonFlagOnce.Do(func() {})
|
||||
if enabled {
|
||||
atomic.StoreInt32(&jsonMode, 1)
|
||||
} else {
|
||||
atomic.StoreInt32(&jsonMode, 0)
|
||||
}
|
||||
}
|
||||
|
||||
// IsJSONMode returns whether JSON mode is currently active.
|
||||
// On first call, it applies the --log_json flag value if set.
|
||||
func IsJSONMode() bool {
|
||||
jsonFlagOnce.Do(func() {
|
||||
if logJSON != nil && *logJSON {
|
||||
atomic.StoreInt32(&jsonMode, 1)
|
||||
}
|
||||
})
|
||||
return atomic.LoadInt32(&jsonMode) == 1
|
||||
}
|
||||
|
||||
// formatJSON builds a JSON log line without using encoding/json
|
||||
// to avoid allocations and keep it as fast as the text path.
|
||||
// Output: {"ts":"...","level":"...","file":"...","line":N,"msg":"..."}\n
|
||||
func (l *loggingT) formatJSON(s severity, depth int) (*buffer, string, int) {
|
||||
_, file, line, ok := runtime.Caller(3 + depth)
|
||||
if !ok {
|
||||
file = "???"
|
||||
line = 1
|
||||
} else {
|
||||
slash := strings.LastIndex(file, "/")
|
||||
if slash >= 0 {
|
||||
file = file[slash+1:]
|
||||
}
|
||||
}
|
||||
|
||||
buf := l.getBuffer()
|
||||
now := timeNow()
|
||||
|
||||
buf.WriteString(`{"ts":"`)
|
||||
buf.WriteString(now.UTC().Format(time.RFC3339Nano))
|
||||
buf.WriteString(`","level":"`)
|
||||
|
||||
switch {
|
||||
case s == infoLog:
|
||||
buf.WriteString("INFO")
|
||||
case s == warningLog:
|
||||
buf.WriteString("WARNING")
|
||||
case s == errorLog:
|
||||
buf.WriteString("ERROR")
|
||||
case s >= fatalLog:
|
||||
buf.WriteString("FATAL")
|
||||
}
|
||||
|
||||
buf.WriteString(`","file":"`)
|
||||
buf.WriteString(jsonEscapeString(file))
|
||||
buf.WriteString(`","line":`)
|
||||
// Write line number without fmt.Sprintf
|
||||
buf.WriteString(itoa(line))
|
||||
buf.WriteString(`,"msg":"`)
|
||||
|
||||
return buf, file, line
|
||||
}
|
||||
|
||||
// finishJSON closes the JSON object and adds a newline.
|
||||
func finishJSON(buf *buffer) {
|
||||
buf.WriteString("\"}\n")
|
||||
}
|
||||
|
||||
// jsonEscapeString escapes a string for safe inclusion in JSON (RFC 8259).
|
||||
// Handles: \, ", \n, \r, \t, control characters, and invalid UTF-8 sequences.
|
||||
func jsonEscapeString(s string) string {
|
||||
// Fast path: no special chars and valid UTF-8
|
||||
needsEscape := false
|
||||
for i := 0; i < len(s); i++ {
|
||||
c := s[i]
|
||||
if c == '"' || c == '\\' || c < 0x20 || c > 0x7e {
|
||||
needsEscape = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !needsEscape {
|
||||
return s
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
b.Grow(len(s) + 10)
|
||||
for i := 0; i < len(s); {
|
||||
c := s[i]
|
||||
switch {
|
||||
case c == '"':
|
||||
b.WriteString(`\"`)
|
||||
i++
|
||||
case c == '\\':
|
||||
b.WriteString(`\\`)
|
||||
i++
|
||||
case c == '\n':
|
||||
b.WriteString(`\n`)
|
||||
i++
|
||||
case c == '\r':
|
||||
b.WriteString(`\r`)
|
||||
i++
|
||||
case c == '\t':
|
||||
b.WriteString(`\t`)
|
||||
i++
|
||||
case c < 0x20:
|
||||
fmt.Fprintf(&b, `\u%04x`, c)
|
||||
i++
|
||||
case c < utf8.RuneSelf:
|
||||
b.WriteByte(c)
|
||||
i++
|
||||
default:
|
||||
r, size := utf8.DecodeRuneInString(s[i:])
|
||||
if r == utf8.RuneError && size == 1 {
|
||||
b.WriteString(`\ufffd`)
|
||||
i++
|
||||
} else {
|
||||
b.WriteString(s[i : i+size])
|
||||
i += size
|
||||
}
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// itoa converts an integer to a string.
|
||||
func itoa(i int) string {
|
||||
return strconv.Itoa(i)
|
||||
}
|
||||
188
weed/glog/glog_json_test.go
Normal file
188
weed/glog/glog_json_test.go
Normal file
@@ -0,0 +1,188 @@
|
||||
package glog
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestJSONMode_Toggle(t *testing.T) {
|
||||
// Default is off
|
||||
if IsJSONMode() {
|
||||
t.Error("JSON mode should be off by default")
|
||||
}
|
||||
|
||||
SetJSONMode(true)
|
||||
if !IsJSONMode() {
|
||||
t.Error("JSON mode should be on after SetJSONMode(true)")
|
||||
}
|
||||
|
||||
SetJSONMode(false)
|
||||
if IsJSONMode() {
|
||||
t.Error("JSON mode should be off after SetJSONMode(false)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONMode_Output(t *testing.T) {
|
||||
setFlags()
|
||||
defer logging.swap(logging.newBuffers())
|
||||
defer SetJSONMode(false)
|
||||
|
||||
SetJSONMode(true)
|
||||
|
||||
defer func(previous func() time.Time) { timeNow = previous }(timeNow)
|
||||
timeNow = func() time.Time {
|
||||
return time.Date(2026, 3, 19, 15, 30, 0, 0, time.UTC)
|
||||
}
|
||||
|
||||
Info("hello json")
|
||||
|
||||
output := contents(infoLog)
|
||||
if !strings.HasPrefix(output, "{") {
|
||||
t.Fatalf("JSON output should start with '{', got: %q", output)
|
||||
}
|
||||
|
||||
// Parse as JSON
|
||||
var parsed map[string]interface{}
|
||||
// Trim trailing newline
|
||||
line := strings.TrimSpace(output)
|
||||
if err := json.Unmarshal([]byte(line), &parsed); err != nil {
|
||||
t.Fatalf("output is not valid JSON: %v\noutput: %q", err, line)
|
||||
}
|
||||
|
||||
// Check required fields
|
||||
if _, ok := parsed["ts"]; !ok {
|
||||
t.Error("JSON missing 'ts' field")
|
||||
}
|
||||
if level, ok := parsed["level"]; !ok || level != "INFO" {
|
||||
t.Errorf("expected level=INFO, got %v", level)
|
||||
}
|
||||
if _, ok := parsed["file"]; !ok {
|
||||
t.Error("JSON missing 'file' field")
|
||||
}
|
||||
if _, ok := parsed["line"]; !ok {
|
||||
t.Error("JSON missing 'line' field")
|
||||
}
|
||||
if msg, ok := parsed["msg"]; !ok || msg != "hello json" {
|
||||
t.Errorf("expected msg='hello json', got %v", msg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONMode_AllLevels(t *testing.T) {
|
||||
setFlags()
|
||||
defer logging.swap(logging.newBuffers())
|
||||
defer SetJSONMode(false)
|
||||
SetJSONMode(true)
|
||||
|
||||
Info("info msg")
|
||||
Warning("warn msg")
|
||||
Error("error msg")
|
||||
|
||||
for _, sev := range []severity{infoLog, warningLog, errorLog} {
|
||||
output := contents(sev)
|
||||
if output == "" {
|
||||
continue
|
||||
}
|
||||
// Each line should be valid JSON
|
||||
for _, line := range strings.Split(strings.TrimSpace(output), "\n") {
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
var parsed map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(line), &parsed); err != nil {
|
||||
t.Errorf("severity %d: invalid JSON: %v\nline: %q", sev, err, line)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONMode_Infof(t *testing.T) {
|
||||
setFlags()
|
||||
defer logging.swap(logging.newBuffers())
|
||||
defer SetJSONMode(false)
|
||||
SetJSONMode(true)
|
||||
|
||||
Infof("count=%d name=%s", 42, "test")
|
||||
|
||||
output := strings.TrimSpace(contents(infoLog))
|
||||
var parsed map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(output), &parsed); err != nil {
|
||||
t.Fatalf("Infof JSON invalid: %v\noutput: %q", err, output)
|
||||
}
|
||||
msg := parsed["msg"].(string)
|
||||
if !strings.Contains(msg, "count=42") || !strings.Contains(msg, "name=test") {
|
||||
t.Errorf("Infof message wrong: %q", msg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONEscapeString(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
want string
|
||||
}{
|
||||
{"hello", "hello"},
|
||||
{`say "hi"`, `say \"hi\"`},
|
||||
{"line1\nline2", `line1\nline2`},
|
||||
{"tab\there", `tab\there`},
|
||||
{`back\slash`, `back\\slash`},
|
||||
{"ctrl\x00char", `ctrl\u0000char`},
|
||||
{"", ""},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
got := jsonEscapeString(tt.input)
|
||||
if got != tt.want {
|
||||
t.Errorf("jsonEscapeString(%q) = %q, want %q", tt.input, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestItoa(t *testing.T) {
|
||||
tests := []struct {
|
||||
input int
|
||||
want string
|
||||
}{
|
||||
{0, "0"},
|
||||
{5, "5"},
|
||||
{9, "9"},
|
||||
{10, "10"},
|
||||
{42, "42"},
|
||||
{12345, "12345"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got := itoa(tt.input)
|
||||
if got != tt.want {
|
||||
t.Errorf("itoa(%d) = %q, want %q", tt.input, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONMode_TextFallback(t *testing.T) {
|
||||
setFlags()
|
||||
defer logging.swap(logging.newBuffers())
|
||||
|
||||
// With JSON mode OFF, output should NOT be JSON
|
||||
SetJSONMode(false)
|
||||
Info("text mode")
|
||||
|
||||
output := contents(infoLog)
|
||||
if strings.HasPrefix(output, "{") {
|
||||
t.Error("text mode should not produce JSON output")
|
||||
}
|
||||
if !strings.Contains(output, "text mode") {
|
||||
t.Error("text mode output missing message")
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkJSONMode(b *testing.B) {
|
||||
setFlags()
|
||||
defer logging.swap(logging.newBuffers())
|
||||
defer SetJSONMode(false)
|
||||
SetJSONMode(true)
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
Info("benchmark json message")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user