Add support for messages in a nested object. (#31)

djson/unmarshal.go

@@ -29,7 +29,9 @@ func process(data []byte, val interface{}, elem reflect.Type, i int) {
 	if !ok {
 		return
 	}
-	for _, key := range strings.Split(keys, ",") {
+	keylist := strings.Split(keys, ",")
+	reverse(keylist) // to prioritize the keys in the beginning of the list
+	for _, key := range keylist {
 		result := gjson.GetBytes(data, strings.TrimSpace(key))
 		if !result.Exists() {
 			result = gjson.GetBytes(data, strings.ReplaceAll(strings.TrimSpace(key), ".", "\\."))
@@ -67,3 +69,9 @@ func convert(value, field reflect.Value) reflect.Value {
 	}
 	return reflect.ValueOf(nil)
 }
+
+func reverse[S ~[]T, T any](s S) {
+	for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
+		s[i], s[j] = s[j], s[i]
+	}
+}

djson/unmarshal_test.go

@@ -140,3 +140,25 @@ func TestDoubleNestedType(t *testing.T) {
 		t.Error("failed to set .Level from nested path")
 	}
 }
+
+func TestWildcardNested(t *testing.T) {
+	t.Parallel()
+	val := struct {
+		Message string `djson:"message,*.message"`
+	}{}
+
+	djson.Unmarshal([]byte(`{"fields.message": "Hello"}`), &val)
+	if val.Message != "Hello" {
+		t.Error("failed to set .Message from static flat path")
+	}
+
+	djson.Unmarshal([]byte(`{"fields": {"message": "Hello"}}`), &val)
+	if val.Message != "Hello" {
+		t.Error("failed to set .Message from nested path")
+	}
+
+	djson.Unmarshal([]byte(`{"message": "First", "fields.message": "Second"}`), &val)
+	if val.Message != "First" {
+		t.Error("failed to set .Message from first key")
+	}
+}

examples/mocks/myprogram

@@ -12,6 +12,9 @@ elif [ "$1" = "--complex" ]; then
 elif [ "$1" = "--no-message" ]; then
   echo '{"message": "", "severity": "trace", "timestamp": "2017-09-28T06:43:13Z", "user": "john"}'
   echo '{"msg": "", "severity": "error", "timestamp": "2017-09-28T06:43:14Z", "@version": "1.0.0"}'
+elif [ "$1" = "--nested-message" ]; then
+  echo '{"timestamp":"2022-02-15T18:47:10.821422Z","level":"INFO","fields":{"message":"shaving yaks","yaks":7},"target":"fmt_json","spans":{"yaks":7,"name":"shaving_yaks"}}'
+  echo '{"timestamp":"2022-02-15T18:47:10.821495Z","level":"TRACE","fields":{"message":"hello! Im gonna shave a yak","excitement":"yay!"},"target":"fmt_json","spans":{"yaks":7,"name":"shaving_yaks"}}'
 else
   echo '{"message": "Hello, world!!", "severity": "info"}'
   echo '{"message": "skipping file", "severity": "warn", "file": "empty.txt"}'

examples/nested_fields.md

@@ -0,0 +1,33 @@
+# Nested Fields
+
+By default `jl` will ignore nested fields unless you explicitly include them:
+
+    $ echo '{"msg":"Booting...", "level": "INFO", "meta": {"app": "backend", "server": 6}}' | jl
+       INFO: Booting...
+
+    $ echo '{"msg":"Booting...", "level": "INFO", "meta": {"app": "backend", "server": 6}}' | jl -f meta.app
+       INFO: Booting... [meta.app=backend]
+
+    $ echo '{"msg":"Booting...", "level": "INFO", "meta": {"app": "backend", "server": 6}}' | jl -f meta
+       INFO: Booting... [meta.app=backend meta.server=6]
+
+
+Some logging formats have their message in a nested fields, like
+[tokio/tracing](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/fmt/format/struct.Json.html#example-output)
+for example, have the log message in the `.fields.message` field:
+
+    $ myprogram --nested-message
+    {"timestamp":"2022-02-15T18:47:10.821422Z","level":"INFO","fields":{"message":"shaving yaks","yaks":7},"target":"fmt_json","spans":{"yaks":7,"name":"shaving_yaks"}}
+    {"timestamp":"2022-02-15T18:47:10.821495Z","level":"TRACE","fields":{"message":"hello! Im gonna shave a yak","excitement":"yay!"},"target":"fmt_json","spans":{"yaks":7,"name":"shaving_yaks"}}
+
+`jl` will also look for the json path `.*.message`, and when it finds `.fields.message` it will output that message and automatically include the `fields` object:
+
+    $ myprogram --nested-message | jl
+    [2022-02-15 18:47:10]    INFO: shaving yaks [fields.yaks=7 target=fmt_json]
+    [2022-02-15 18:47:10]   TRACE: hello! Im gonna shave a yak [fields.excitement=yay! target=fmt_json]
+
+You can also explicitly include other nested objects as well:
+
+    $ myprogram --nested-message | jl -f spans
+    [2022-02-15 18:47:10]    INFO: shaving yaks [fields.yaks=7 spans.name=shaving_yaks spans.yaks=7 target=fmt_json]
+    [2022-02-15 18:47:10]   TRACE: hello! Im gonna shave a yak [fields.excitement=yay! spans.name=shaving_yaks spans.yaks=7 target=fmt_json]

main.go

@@ -31,7 +31,7 @@ func main() {
 	formatter.ShowSuffix = showSuffix
 	formatter.ShowFields = showFields
 	formatter.MaxFieldLength = maxFieldLength
-	formatter.IncludeFields = includeFields
+	formatter.IncludeFields = strings.Split(includeFields, ",")
 	formatter.ExcludeFields = append(formatter.ExcludeFields, strings.Split(excludeFields, ",")...)
 
 	r, err := openFiles(files)
@@ -42,9 +42,7 @@ func main() {
 	s := stream.New(r)
 	for line := range s.Lines() {
 		var err error
-		entry := &structure.Entry{
-			SkipFields: make(map[string]bool),
-		}
+		entry := &structure.Entry{}
 		if line.JSON != nil && len(line.JSON) > 0 {
 			var unused interface{}
 			err = json.Unmarshal(line.JSON, &unused)

processors/journald.go

@@ -30,7 +30,7 @@ func (p *JournaldProcessor) Detect(line *stream.Line, entry *structure.Entry) bo
 
 func (p *JournaldProcessor) Process(line *stream.Line, entry *structure.Entry) error {
 	entry.Message = gjson.GetBytes(line.JSON, "MESSAGE").String()
-	entry.SkipFields["MESSAGE"] = true
+	entry.ExcludeFields = append(entry.ExcludeFields, "MESSAGE")
 
 	rawTimestamp := gjson.GetBytes(line.JSON, "__REALTIME_TIMESTAMP").String()
 	micro, err := strconv.ParseInt(rawTimestamp, 10, 64)
@@ -40,23 +40,23 @@ func (p *JournaldProcessor) Process(line *stream.Line, entry *structure.Entry) e
 		t := time.Unix(micro/1_000_000, micro%1_000_000*1_000).UTC()
 		entry.Timestamp = &t
 	}
-	entry.SkipFields["__REALTIME_TIMESTAMP"] = true
+	entry.ExcludeFields = append(entry.ExcludeFields, "__REALTIME_TIMESTAMP")
 
 	prioField := gjson.GetBytes(line.JSON, "PRIORITY")
 	switch prioField.Type {
 	case gjson.Number:
 		priority := prioField.Int()
 		entry.Severity = priorityMapping[priority]
-		entry.SkipFields["PRIORITY"] = true
+		entry.ExcludeFields = append(entry.ExcludeFields, "PRIORITY")
 	case gjson.String:
 		priority := prioField.String()
 		if i, err := strconv.ParseInt(priority, 10, 64); err == nil {
 			entry.Severity = priorityMapping[i]
-			entry.SkipFields["PRIORITY"] = true
+			entry.ExcludeFields = append(entry.ExcludeFields, "PRIORITY")
 			break
 		} else if len(priority) < 12 {
 			entry.Severity = strings.ToUpper(priority)
-			entry.SkipFields["PRIORITY"] = true
+			entry.ExcludeFields = append(entry.ExcludeFields, "PRIORITY")
 			break
 		}
 		fallthrough

processors/journald_test.go

@@ -3,6 +3,7 @@ package processors
 import (
 	"bytes"
 	"encoding/json"
+	"sort"
 	"testing"
 	"time"
 
@@ -43,7 +44,7 @@ func TestJournald_Happypath(t *testing.T) {
 	_ = json.Compact(raw, []byte(in))
 
 	line := &stream.Line{Raw: raw.Bytes(), JSON: raw.Bytes()}
-	entry := &structure.Entry{SkipFields: make(map[string]bool)}
+	entry := &structure.Entry{}
 	djson.Unmarshal(line.JSON, entry)
 
 	p := &JournaldProcessor{}
@@ -70,15 +71,15 @@ func TestJournald_Happypath(t *testing.T) {
 		t.Errorf("entry.Severity = %v, want %v", got, want)
 	}
 
-	if got, want := entry.SkipFields["MESSAGE"], true; got != want {
+	if got, want := has(entry.ExcludeFields, "MESSAGE"), true; got != want {
 		t.Errorf("entry.SkipFields['MESSAGE'] = %v, want %v", got, want)
 	}
 
-	if got, want := entry.SkipFields["__REALTIME_TIMESTAMP"], true; got != want {
+	if got, want := has(entry.ExcludeFields, "__REALTIME_TIMESTAMP"), true; got != want {
 		t.Errorf("entry.SkipFields['__REALTIME_TIMESTAMP'] = %v, want %v", got, want)
 	}
 
-	if got, want := entry.SkipFields["PRIORITY"], true; got != want {
+	if got, want := has(entry.ExcludeFields, "PRIORITY"), true; got != want {
 		t.Errorf("entry.SkipFields['PRIORITY'] = %v, want %v", got, want)
 	}
 }
@@ -116,7 +117,7 @@ func TestJournald_PriorityString(t *testing.T) {
 		t.Errorf("entry.Severity = %v, want %v", got, want)
 	}
 
-	if got, want := entry.SkipFields["PRIORITY"], true; got != want {
+	if got, want := has(entry.ExcludeFields, "PRIORITY"), true; got != want {
 		t.Errorf("entry.SkipFields['PRIORITY'] = %v, want %v", got, want)
 	}
 }
@@ -135,7 +136,7 @@ func TestJournald_PriorityInt(t *testing.T) {
 		t.Errorf("entry.Severity = %v, want %v", got, want)
 	}
 
-	if got, want := entry.SkipFields["PRIORITY"], true; got != want {
+	if got, want := has(entry.ExcludeFields, "PRIORITY"), true; got != want {
 		t.Errorf("entry.SkipFields['PRIORITY'] = %v, want %v", got, want)
 	}
 }
@@ -154,7 +155,7 @@ func TestJournald_InvalidPriority(t *testing.T) {
 		t.Errorf("entry.Severity = %v, want %v", got, want)
 	}
 
-	if got, want := entry.SkipFields["PRIORITY"], false; got != want {
+	if got, want := has(entry.ExcludeFields, "PRIORITY"), false; got != want {
 		t.Errorf("entry.SkipFields['PRIORITY'] = %v, want %v", got, want)
 	}
 }
@@ -166,7 +167,7 @@ func process(t *testing.T, input string) *structure.Entry {
 	_ = json.Compact(raw, []byte(input))
 
 	line := &stream.Line{Raw: raw.Bytes(), JSON: raw.Bytes()}
-	entry := &structure.Entry{SkipFields: make(map[string]bool)}
+	entry := &structure.Entry{}
 	djson.Unmarshal(line.JSON, entry)
 
 	p := &JournaldProcessor{}
@@ -183,3 +184,9 @@ func process(t *testing.T, input string) *structure.Entry {
 
 	return entry
 }
+
+func has(slice []string, s string) bool {
+	sort.Strings(slice)
+	ix := sort.SearchStrings(slice, s)
+	return ix < len(slice) && slice[ix] == s
+}

processors/nested.go

@@ -0,0 +1,27 @@
+package processors
+
+import (
+	"strings"
+
+	"github.com/koenbollen/jl/stream"
+	"github.com/koenbollen/jl/structure"
+	"github.com/tidwall/gjson"
+)
+
+type NestedProcessor struct {
+}
+
+func (p *NestedProcessor) Detect(line *stream.Line, entry *structure.Entry) bool {
+	result := gjson.GetBytes(line.JSON, "*.message")
+	return result.Exists() && result.String() != "" && entry.Message == result.String()
+}
+
+func (p *NestedProcessor) Process(line *stream.Line, entry *structure.Entry) error {
+	result := gjson.GetBytes(line.JSON, "*.message")
+	path := result.Path(string(line.JSON))
+	if field, _, found := strings.Cut(path, ".message"); found {
+		entry.IncludeFields = append(entry.IncludeFields, field)
+		entry.ExcludeFields = append(entry.ExcludeFields, path)
+	}
+	return nil
+}

processors/nested_test.go

@@ -0,0 +1,48 @@
+package processors
+
+import (
+	"bytes"
+	"encoding/json"
+	"testing"
+
+	"github.com/koenbollen/jl/djson"
+	"github.com/koenbollen/jl/stream"
+	"github.com/koenbollen/jl/structure"
+)
+
+func TestNested(t *testing.T) {
+
+	in := `{
+		"level": "INFO",
+		"nested": {
+			"message": "Hi",
+			"user": 56
+		}
+	}`
+
+	raw := &bytes.Buffer{}
+	_ = json.Compact(raw, []byte(in))
+
+	line := &stream.Line{Raw: raw.Bytes(), JSON: raw.Bytes()}
+	entry := &structure.Entry{Message: "Hi"}
+	djson.Unmarshal(line.JSON, entry)
+
+	p := &NestedProcessor{}
+
+	detected := p.Detect(line, entry)
+	if got, want := detected, true; got != want {
+		t.Errorf("Detect() = %v, want %v", got, want)
+	}
+
+	err := p.Process(line, entry)
+	if err != nil {
+		t.Errorf("Process() = %v, want nil", err)
+	}
+
+	if got, want := entry.Message, "Hi"; got != want {
+		t.Errorf("entry.Message = %v, want %v", got, want)
+	}
+	if got, want := entry.IncludeFields[0], "nested"; got != want {
+		t.Errorf("entry.IncludeFields = %v, want %v", got, want)
+	}
+}

processors/shared.go

@@ -11,5 +11,6 @@ type Processor interface {
 }
 
 var All = []Processor{
+	&NestedProcessor{},
 	&JournaldProcessor{},
 }

structure/colors.go

@@ -4,7 +4,7 @@ import "github.com/fatih/color"
 
 var messageColor = color.New(color.FgHiCyan, color.Bold).SprintFunc()
 var severityColors = map[string]func(a ...interface{}) string{
-	"TRACE":   color.New(color.FgBlack).SprintFunc(),
+	"TRACE":   color.New(color.FgHiBlack).SprintFunc(),
 	"DEBUG":   color.New(color.FgHiBlack).SprintFunc(),
 	"INFO":    color.New(color.FgCyan).SprintFunc(),
 	"WARNING": color.New(color.FgRed).SprintFunc(),

structure/entry.go

@@ -8,10 +8,13 @@ type Entry struct {
 	RawTimestamp   string     `djson:"timestamp,@timestamp,time,date,ts"`
 	FloatTimestamp float64    `djson:"timestamp,@timestamp,time,date,ts"`
 	Severity       string     `djson:"severity,level,log.level"`
-	Message        string     `djson:"message,msg,text"`
+	Message        string     `djson:"message,msg,text,*.message"`
 
 	Name string `djson:"app,name,service.name"`
 
-	// SkipFields is used by processors to indicate which fields should be skipped
-	SkipFields map[string]bool
+	// IncludeFields is used by processors to indicate which fields should be included
+	IncludeFields []string
+
+	// ExcludeFields is used by processors to indicate which fields should be skipped
+	ExcludeFields []string
 }

structure/format.go

@@ -44,7 +44,7 @@ type Formatter struct {
 	MaxFieldLength int
 	ShowPrefix     bool
 	ShowSuffix     bool
-	IncludeFields  string
+	IncludeFields  []string
 	ExcludeFields  []string
 }
 
@@ -66,7 +66,6 @@ func NewFormatter(w io.Writer, fmt string) (*Formatter, error) {
 		MaxFieldLength: 30,
 		ShowPrefix:     true,
 		ShowSuffix:     true,
-		IncludeFields:  "",
 		ExcludeFields:  defaultExcludes,
 	}, nil
 }
@@ -181,16 +180,26 @@ func (f *Formatter) outputFields(entry *Entry, raw json.RawMessage) {
 }
 
 func (f *Formatter) shouldSkipField(entry *Entry, field, path string, value interface{}) bool {
-	if strings.Contains(f.IncludeFields, field) || strings.Contains(f.IncludeFields, path) {
+	if contains(f.IncludeFields, field) || contains(f.IncludeFields, path) {
 		return false
 	}
-	if strings.Count(path, ".") > 1 { // Only include nested fields when the are in the IncludeFields
+	if contains(entry.IncludeFields, field) {
+		return false
+	}
+	if contains(entry.ExcludeFields, field) {
 		return true
 	}
-	if f.MaxFieldLength > 0 && len(path+fmt.Sprintf("%v", value)) >= f.MaxFieldLength {
+	if strings.Count(path, ".") > 1 {
+		first, _, _ := strings.Cut(strings.Trim(path, "."), ".")
+		if contains(f.IncludeFields, first) {
+			return false
+		}
+		if contains(entry.IncludeFields, first) {
+			return false
+		}
 		return true
 	}
-	if _, skip := entry.SkipFields[field]; skip {
+	if f.MaxFieldLength > 0 && len(path+fmt.Sprintf("%v", value)) >= f.MaxFieldLength {
 		return true
 	}
 	return contains(f.ExcludeFields, field)