diff --git a/weed/glog/glog.go b/weed/glog/glog.go index de1e02f57..93d41a972 100644 --- a/weed/glog/glog.go +++ b/weed/glog/glog.go @@ -630,7 +630,21 @@ func (buf *buffer) someDigits(i, d int) int { return copy(buf.tmp[i:], buf.tmp[j:]) } +// logJSON is a helper that formats and outputs a JSON log line. +// Used by println, printDepth, and printf to avoid code duplication. +// depth is incremented by 1 to account for this extra call frame. +func (l *loggingT) logJSON(s severity, depth int, msg string) { + buf, file, line := l.formatJSON(s, depth+1) + buf.WriteString(jsonEscapeString(strings.TrimRight(msg, "\n"))) + finishJSON(buf) + l.output(s, buf, file, line, false) +} + func (l *loggingT) println(s severity, args ...interface{}) { + if IsJSONMode() { + l.logJSON(s, 0, fmt.Sprintln(args...)) + return + } buf, file, line := l.header(s, 0) fmt.Fprintln(buf, args...) l.output(s, buf, file, line, false) @@ -641,6 +655,10 @@ func (l *loggingT) print(s severity, args ...interface{}) { } func (l *loggingT) printDepth(s severity, depth int, args ...interface{}) { + if IsJSONMode() { + l.logJSON(s, depth, fmt.Sprint(args...)) + return + } buf, file, line := l.header(s, depth) fmt.Fprint(buf, args...) if buf.Bytes()[buf.Len()-1] != '\n' { @@ -650,6 +668,10 @@ func (l *loggingT) printDepth(s severity, depth int, args ...interface{}) { } func (l *loggingT) printf(s severity, format string, args ...interface{}) { + if IsJSONMode() { + l.logJSON(s, 0, fmt.Sprintf(format, args...)) + return + } buf, file, line := l.header(s, 0) fmt.Fprintf(buf, format, args...) if buf.Bytes()[buf.Len()-1] != '\n' { @@ -662,6 +684,36 @@ func (l *loggingT) printf(s severity, format string, args ...interface{}) { // alsoLogToStderr is true, the log message always appears on standard error; it // will also appear in the log file unless --logtostderr is set. func (l *loggingT) printWithFileLine(s severity, file string, line int, alsoToStderr bool, args ...interface{}) { + if IsJSONMode() { + buf := l.getBuffer() + now := timeNow() + buf.WriteString(`{"ts":"`) + buf.WriteString(now.UTC().Format(time.RFC3339Nano)) + buf.WriteString(`","level":"`) + switch { + case s == infoLog: + buf.WriteString("INFO") + case s == warningLog: + buf.WriteString("WARNING") + case s == errorLog: + buf.WriteString("ERROR") + case s >= fatalLog: + buf.WriteString("FATAL") + } + buf.WriteString(`","file":"`) + buf.WriteString(jsonEscapeString(file)) + buf.WriteString(`","line":`) + if line < 0 { + line = 0 + } + buf.WriteString(itoa(line)) + buf.WriteString(`,"msg":"`) + msg := fmt.Sprint(args...) + buf.WriteString(jsonEscapeString(strings.TrimRight(msg, "\n"))) + finishJSON(buf) + l.output(s, buf, file, line, alsoToStderr) + return + } buf := l.formatHeader(s, file, line) fmt.Fprint(buf, args...) if buf.Bytes()[buf.Len()-1] != '\n' { @@ -873,6 +925,11 @@ func (sb *syncBuffer) rotateFile(now time.Time) error { sb.Writer = bufio.NewWriterSize(sb.file, bufferSize) + // Skip text header in JSON mode to keep files as valid NDJSON. + if IsJSONMode() { + return nil + } + // Write header. var buf bytes.Buffer fmt.Fprintf(&buf, "Log file created at: %s\n", now.Format("2006/01/02 15:04:05")) diff --git a/weed/glog/glog_file.go b/weed/glog/glog_file.go index 5911e0f2f..bb9ab3c78 100644 --- a/weed/glog/glog_file.go +++ b/weed/glog/glog_file.go @@ -64,6 +64,10 @@ var logMaxFiles = flag.Int("log_max_files", 5, "Maximum number of log files to k // The default is 168 hours (7 days). Set to 0 to disable time-based rotation. var logRotateHours = flag.Int("log_rotate_hours", 168, "Rotate log files after this many hours (default: 168 = 7 days, 0 = disabled)") +// logJSON enables JSON-formatted log output (one JSON object per line). +// Useful for integration with ELK, Loki, Datadog, and other log aggregation systems. +var logJSON = flag.Bool("log_json", false, "Output logs in JSON format instead of glog text format") + // logCompress enables gzip compression of rotated log files. // Compressed files get a .gz suffix. Compression runs in the background. var logCompress = flag.Bool("log_compress", false, "Gzip-compress rotated log files to save disk space") diff --git a/weed/glog/glog_json.go b/weed/glog/glog_json.go new file mode 100644 index 000000000..b732a6795 --- /dev/null +++ b/weed/glog/glog_json.go @@ -0,0 +1,157 @@ +package glog + +import ( + "fmt" + "runtime" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + "unicode/utf8" +) + +// JSONMode controls whether log output is in JSON format. +// 0 = classic glog text (default), 1 = JSON lines. +// Safe for concurrent access. +var jsonMode int32 + +// jsonFlagOnce ensures the --log_json flag is applied on first use, +// even when -logtostderr prevents createLogDirs() from running. +var jsonFlagOnce sync.Once + +// SetJSONMode enables or disables JSON-formatted log output. +// When enabled, each log line is a single JSON object: +// +// {"ts":"2006-01-02T15:04:05.000000Z","level":"INFO","file":"server.go","line":42,"msg":"..."} +// +// This is useful for log aggregation systems (ELK, Loki, Datadog, etc). +func SetJSONMode(enabled bool) { + // Prevent lazy flag initialization from overriding this explicit call. + jsonFlagOnce.Do(func() {}) + if enabled { + atomic.StoreInt32(&jsonMode, 1) + } else { + atomic.StoreInt32(&jsonMode, 0) + } +} + +// IsJSONMode returns whether JSON mode is currently active. +// On first call, it applies the --log_json flag value if set. +func IsJSONMode() bool { + jsonFlagOnce.Do(func() { + if logJSON != nil && *logJSON { + atomic.StoreInt32(&jsonMode, 1) + } + }) + return atomic.LoadInt32(&jsonMode) == 1 +} + +// formatJSON builds a JSON log line without using encoding/json +// to avoid allocations and keep it as fast as the text path. +// Output: {"ts":"...","level":"...","file":"...","line":N,"msg":"..."}\n +func (l *loggingT) formatJSON(s severity, depth int) (*buffer, string, int) { + _, file, line, ok := runtime.Caller(3 + depth) + if !ok { + file = "???" + line = 1 + } else { + slash := strings.LastIndex(file, "/") + if slash >= 0 { + file = file[slash+1:] + } + } + + buf := l.getBuffer() + now := timeNow() + + buf.WriteString(`{"ts":"`) + buf.WriteString(now.UTC().Format(time.RFC3339Nano)) + buf.WriteString(`","level":"`) + + switch { + case s == infoLog: + buf.WriteString("INFO") + case s == warningLog: + buf.WriteString("WARNING") + case s == errorLog: + buf.WriteString("ERROR") + case s >= fatalLog: + buf.WriteString("FATAL") + } + + buf.WriteString(`","file":"`) + buf.WriteString(jsonEscapeString(file)) + buf.WriteString(`","line":`) + // Write line number without fmt.Sprintf + buf.WriteString(itoa(line)) + buf.WriteString(`,"msg":"`) + + return buf, file, line +} + +// finishJSON closes the JSON object and adds a newline. +func finishJSON(buf *buffer) { + buf.WriteString("\"}\n") +} + +// jsonEscapeString escapes a string for safe inclusion in JSON (RFC 8259). +// Handles: \, ", \n, \r, \t, control characters, and invalid UTF-8 sequences. +func jsonEscapeString(s string) string { + // Fast path: no special chars and valid UTF-8 + needsEscape := false + for i := 0; i < len(s); i++ { + c := s[i] + if c == '"' || c == '\\' || c < 0x20 || c > 0x7e { + needsEscape = true + break + } + } + if !needsEscape { + return s + } + + var b strings.Builder + b.Grow(len(s) + 10) + for i := 0; i < len(s); { + c := s[i] + switch { + case c == '"': + b.WriteString(`\"`) + i++ + case c == '\\': + b.WriteString(`\\`) + i++ + case c == '\n': + b.WriteString(`\n`) + i++ + case c == '\r': + b.WriteString(`\r`) + i++ + case c == '\t': + b.WriteString(`\t`) + i++ + case c < 0x20: + fmt.Fprintf(&b, `\u%04x`, c) + i++ + case c < utf8.RuneSelf: + b.WriteByte(c) + i++ + default: + r, size := utf8.DecodeRuneInString(s[i:]) + if r == utf8.RuneError && size == 1 { + b.WriteString(`\ufffd`) + i++ + } else { + b.WriteString(s[i : i+size]) + i += size + } + } + } + return b.String() +} + +// itoa converts an integer to a string. +func itoa(i int) string { + return strconv.Itoa(i) +} diff --git a/weed/glog/glog_json_test.go b/weed/glog/glog_json_test.go new file mode 100644 index 000000000..9c81f961d --- /dev/null +++ b/weed/glog/glog_json_test.go @@ -0,0 +1,188 @@ +package glog + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +func TestJSONMode_Toggle(t *testing.T) { + // Default is off + if IsJSONMode() { + t.Error("JSON mode should be off by default") + } + + SetJSONMode(true) + if !IsJSONMode() { + t.Error("JSON mode should be on after SetJSONMode(true)") + } + + SetJSONMode(false) + if IsJSONMode() { + t.Error("JSON mode should be off after SetJSONMode(false)") + } +} + +func TestJSONMode_Output(t *testing.T) { + setFlags() + defer logging.swap(logging.newBuffers()) + defer SetJSONMode(false) + + SetJSONMode(true) + + defer func(previous func() time.Time) { timeNow = previous }(timeNow) + timeNow = func() time.Time { + return time.Date(2026, 3, 19, 15, 30, 0, 0, time.UTC) + } + + Info("hello json") + + output := contents(infoLog) + if !strings.HasPrefix(output, "{") { + t.Fatalf("JSON output should start with '{', got: %q", output) + } + + // Parse as JSON + var parsed map[string]interface{} + // Trim trailing newline + line := strings.TrimSpace(output) + if err := json.Unmarshal([]byte(line), &parsed); err != nil { + t.Fatalf("output is not valid JSON: %v\noutput: %q", err, line) + } + + // Check required fields + if _, ok := parsed["ts"]; !ok { + t.Error("JSON missing 'ts' field") + } + if level, ok := parsed["level"]; !ok || level != "INFO" { + t.Errorf("expected level=INFO, got %v", level) + } + if _, ok := parsed["file"]; !ok { + t.Error("JSON missing 'file' field") + } + if _, ok := parsed["line"]; !ok { + t.Error("JSON missing 'line' field") + } + if msg, ok := parsed["msg"]; !ok || msg != "hello json" { + t.Errorf("expected msg='hello json', got %v", msg) + } +} + +func TestJSONMode_AllLevels(t *testing.T) { + setFlags() + defer logging.swap(logging.newBuffers()) + defer SetJSONMode(false) + SetJSONMode(true) + + Info("info msg") + Warning("warn msg") + Error("error msg") + + for _, sev := range []severity{infoLog, warningLog, errorLog} { + output := contents(sev) + if output == "" { + continue + } + // Each line should be valid JSON + for _, line := range strings.Split(strings.TrimSpace(output), "\n") { + if line == "" { + continue + } + var parsed map[string]interface{} + if err := json.Unmarshal([]byte(line), &parsed); err != nil { + t.Errorf("severity %d: invalid JSON: %v\nline: %q", sev, err, line) + } + } + } +} + +func TestJSONMode_Infof(t *testing.T) { + setFlags() + defer logging.swap(logging.newBuffers()) + defer SetJSONMode(false) + SetJSONMode(true) + + Infof("count=%d name=%s", 42, "test") + + output := strings.TrimSpace(contents(infoLog)) + var parsed map[string]interface{} + if err := json.Unmarshal([]byte(output), &parsed); err != nil { + t.Fatalf("Infof JSON invalid: %v\noutput: %q", err, output) + } + msg := parsed["msg"].(string) + if !strings.Contains(msg, "count=42") || !strings.Contains(msg, "name=test") { + t.Errorf("Infof message wrong: %q", msg) + } +} + +func TestJSONEscapeString(t *testing.T) { + tests := []struct { + input string + want string + }{ + {"hello", "hello"}, + {`say "hi"`, `say \"hi\"`}, + {"line1\nline2", `line1\nline2`}, + {"tab\there", `tab\there`}, + {`back\slash`, `back\\slash`}, + {"ctrl\x00char", `ctrl\u0000char`}, + {"", ""}, + } + + for _, tt := range tests { + got := jsonEscapeString(tt.input) + if got != tt.want { + t.Errorf("jsonEscapeString(%q) = %q, want %q", tt.input, got, tt.want) + } + } +} + +func TestItoa(t *testing.T) { + tests := []struct { + input int + want string + }{ + {0, "0"}, + {5, "5"}, + {9, "9"}, + {10, "10"}, + {42, "42"}, + {12345, "12345"}, + } + for _, tt := range tests { + got := itoa(tt.input) + if got != tt.want { + t.Errorf("itoa(%d) = %q, want %q", tt.input, got, tt.want) + } + } +} + +func TestJSONMode_TextFallback(t *testing.T) { + setFlags() + defer logging.swap(logging.newBuffers()) + + // With JSON mode OFF, output should NOT be JSON + SetJSONMode(false) + Info("text mode") + + output := contents(infoLog) + if strings.HasPrefix(output, "{") { + t.Error("text mode should not produce JSON output") + } + if !strings.Contains(output, "text mode") { + t.Error("text mode output missing message") + } +} + +func BenchmarkJSONMode(b *testing.B) { + setFlags() + defer logging.swap(logging.newBuffers()) + defer SetJSONMode(false) + SetJSONMode(true) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + Info("benchmark json message") + } +}