Compacting JSON Representation in GoLang

In this post we will review a method to shrink JSON representation of Go structures. This is critical in case we need to save the state by marshaling the state objects and save them in Redis, which works really bad with large bulk of strings.

Let dive in quickly with an example. Let assume our state is represented by a struct, and see the JSON representation of it:

package main

import (
 "encoding/json"
 "fmt"
 "time"
)

type DetectorConfig struct {
 AnomalyThreshold float32
 AnomalyPattern   string
 EnableDetection  bool
}

type State struct {
 Counter         int
 Stations        []string
 Detector        DetectorConfig
 LastUpdateEpoch int64
}

func ProduceDefaultState() *State {
 return &State{
  Counter:  0,
  Stations: []string{"load", "build", "deploy", "test", "deliver"},
  Detector: DetectorConfig{
   AnomalyThreshold: 5.55,
   AnomalyPattern:   ".*",
   EnableDetection:  true,
  },
  LastUpdateEpoch: time.Now().Unix(),
 }
}

func main() {

 state := ProduceDefaultState()
 bytes, err := json.Marshal(state)
 if err != nil {
  panic(err)
 }

 jsonText := string(bytes)
 fmt.Printf("JSON length is: %v, JSON text is: %v", len(jsonText), jsonText)
}

And the output is:

JSON length is: 178, JSON text is: {"Counter":0,"Stations":["load","build","deploy","test","deliver"],"Detector":{"AnomalyThreshold":5.55,"AnomalyPattern":".*","EnableDetection":true},"LastUpdateEpoch":1691996599}

How can we compact it?

We could use the `json` annotation to use shorter names for the elements, but then we will not be able to display a clear and user friendly JSON to the system admin. A better method would be to decide upon need whether to use clear and user friendly JSON representation when displaying the state to a human, and whether to use a compact JSON representation when saving the state to a DBMS such as redis.

Here is an example of using the compact form:

package main

import (
 "fmt"
 jsoniter "github.com/json-iterator/go"
 "time"
)

type DetectorConfig struct {
 AnomalyThreshold float32 `compact:"a"`
 AnomalyPattern   string  `compact:"b"`
 EnableDetection  bool    `compact:"c"`
}

type State struct {
 Counter         int            `compact:"a"`
 Stations        []string       `compact:"b"`
 Detector        DetectorConfig `compact:"c"`
 LastUpdateEpoch int64          `compact:"d"`
}

func ProduceDefaultState() *State {
 return &State{
  Counter:  0,
  Stations: []string{"load", "build", "deploy", "test", "deliver"},
  Detector: DetectorConfig{
   AnomalyThreshold: 5.55,
   AnomalyPattern:   ".*",
   EnableDetection:  true,
  },
  LastUpdateEpoch: time.Now().Unix(),
 }
}

func main() {
 state := ProduceDefaultState()
 jsonCompact := jsoniter.Config{TagKey: "compact"}.Froze()
 bytes, err := jsonCompact.Marshal(state)
 if err != nil {
  panic(err)
 }

 jsonText := string(bytes)
 fmt.Printf("Compact JSON length is: %v, compact JSON text is: %v", len(jsonText), jsonText)
}

and the output is:

Compact JSON length is: 102, compact JSON text is: {"a":0,"b":["load","build","deploy","test","deliver"],"c":{"a":5.55,"b":".*","c":true},"d":1691996466}

But can we do better?

What if out state is mostly static, and only a few fields are changing? Then we can list only the fields that change, and merge them in to the default config.

package main

import (
 "encoding/json"
 "fmt"
 "radware.com/proximity/commons/global/reflectionapi"
 "time"
)

type DetectorConfig struct {
 AnomalyThreshold float32 `compact:"a"`
 AnomalyPattern   string  `compact:"b"`
 EnableDetection  bool    `compact:"c"`
}

type State struct {
 Counter         int            `compact:"a"`
 Stations        []string       `compact:"b"`
 Detector        DetectorConfig `compact:"c"`
 LastUpdateEpoch int64          `compact:"d"`
}

func ProduceDefaultState() *State {
 return &State{
  Counter:  0,
  Stations: []string{"load", "build", "deploy", "test", "deliver"},
  Detector: DetectorConfig{
   AnomalyThreshold: 5.55,
   AnomalyPattern:   ".*",
   EnableDetection:  true,
  },
  LastUpdateEpoch: time.Now().Unix(),
 }
}

func main() {
 state := ProduceDefaultState()
 state.LastUpdateEpoch = time.Now().Add(time.Second).Unix()
 state.Detector.EnableDetection = false

 defaultState := ProduceDefaultState()
 diffMap := reflectionapi.CreateDiffMap("compact", defaultState, state)
 bytes, err := json.Marshal(diffMap)
 if err != nil {
  panic(err)
 }

 jsonText := string(bytes)
 fmt.Printf("Diff JSON length is: %v, diff JSON text is: %v", len(jsonText), jsonText)
}

And the output is:

Diff JSON length is: 32, diff JSON text is: {"c":{"c":false},"d":1691996360}

Of course the length had significantly reduced, and will be reduced much further the bigger is our state, and the less updated fields it includes.

The reflection library is below:

package reflectionapi

import (
 "fmt"
 "reflect"
 "strings"
)

type DiffHandler func(
 elementPath string,
 value1 interface{},
 value2 interface{},
)

func FindDiff(
 tagForName string,
 item1 interface{},
 item2 interface{},
 handler DiffHandler,
) {
 findDiffRecursive(
  tagForName,
  "",
  item1,
  item2,
  handler,
 )
}

func findDiffRecursive(
 tagForName string,
 prefix string,
 item1 interface{},
 item2 interface{},
 handler DiffHandler,
) {
 reflectType := reflect.TypeOf(item2).Elem()
 reflectValue1 := reflect.ValueOf(item1).Elem()
 reflectValue2 := reflect.ValueOf(item2).Elem()

 for i := 0; i < reflectType.NumField(); i++ {
  fieldType := reflectType.Field(i)
  useName := fieldType.Name
  if tagForName != "" {
   useName = fieldType.Tag.Get(tagForName)
  }

  value1 := reflectValue1.Field(i).Interface()
  value2 := reflectValue2.Field(i).Interface()

  path := prefix + "/" + useName
  switch reflectValue2.Field(i).Kind() {
  case reflect.Struct:
   interface1 := reflectValue1.Field(i).Addr().Interface()
   interface2 := reflectValue2.Field(i).Addr().Interface()
   findDiffRecursive(tagForName, path, interface1, interface2, handler)
   break
  case reflect.Slice:
   value1String := fmt.Sprintf("%v", value1)
   value2String := fmt.Sprintf("%v", value2)
   if value1String != value2String {
    handler(path, value1, value2)
   }
   break
  default:
   if value2 != value1 {
    handler(path, value1, value2)
   }
   break
  }
 }
}

func CreateDiffMap(
 tagForName string,
 itemBaseline interface{},
 itemChanged interface{},
) map[string]interface{} {
 diffMap := make(map[string]interface{})

 handler := func(elementPath string, valueBaseline interface{}, valueChanged interface{}) {
  diffMapEntry := diffMap

  sections := strings.Split(elementPath, "/")
  sections = sections[1:]

  for {
   sectionName := sections[0]
   if len(sections) == 1 {
    diffMapEntry[sectionName] = valueChanged
    break
   } else {
    sections = sections[1:]
    nextEntry := diffMapEntry[sectionName]
    if nextEntry == nil {
     nextEntry = make(map[string]interface{})
     diffMapEntry[sectionName] = nextEntry
    }
    diffMapEntry = nextEntry.(map[string]interface{})
   }
  }
 }

 FindDiff(tagForName, itemBaseline, itemChanged, handler)

 return diffMap
}