In this post we will review a method to shrink JSON representation of Go structures. This is critical in case we need to save the state by marshaling the state objects and save them in Redis, which works really bad with large bulk of strings.
Let dive in quickly with an example. Let assume our state is represented by a struct, and see the JSON representation of it:
package main
import (
"encoding/json"
"fmt"
"time"
)
type DetectorConfig struct {
AnomalyThreshold float32
AnomalyPattern string
EnableDetection bool
}
type State struct {
Counter int
Stations []string
Detector DetectorConfig
LastUpdateEpoch int64
}
func ProduceDefaultState() *State {
return &State{
Counter: 0,
Stations: []string{"load", "build", "deploy", "test", "deliver"},
Detector: DetectorConfig{
AnomalyThreshold: 5.55,
AnomalyPattern: ".*",
EnableDetection: true,
},
LastUpdateEpoch: time.Now().Unix(),
}
}
func main() {
state := ProduceDefaultState()
bytes, err := json.Marshal(state)
if err != nil {
panic(err)
}
jsonText := string(bytes)
fmt.Printf("JSON length is: %v, JSON text is: %v", len(jsonText), jsonText)
}
And the output is:
JSON length is: 178, JSON text is: {"Counter":0,"Stations":["load","build","deploy","test","deliver"],"Detector":{"AnomalyThreshold":5.55,"AnomalyPattern":".*","EnableDetection":true},"LastUpdateEpoch":1691996599}
How can we compact it?
We could use the `json` annotation to use shorter names for the elements, but then we will not be able to display a clear and user friendly JSON to the system admin. A better method would be to decide upon need whether to use clear and user friendly JSON representation when displaying the state to a human, and whether to use a compact JSON representation when saving the state to a DBMS such as redis.
Here is an example of using the compact form:
package main
import (
"fmt"
jsoniter "github.com/json-iterator/go"
"time"
)
type DetectorConfig struct {
AnomalyThreshold float32 `compact:"a"`
AnomalyPattern string `compact:"b"`
EnableDetection bool `compact:"c"`
}
type State struct {
Counter int `compact:"a"`
Stations []string `compact:"b"`
Detector DetectorConfig `compact:"c"`
LastUpdateEpoch int64 `compact:"d"`
}
func ProduceDefaultState() *State {
return &State{
Counter: 0,
Stations: []string{"load", "build", "deploy", "test", "deliver"},
Detector: DetectorConfig{
AnomalyThreshold: 5.55,
AnomalyPattern: ".*",
EnableDetection: true,
},
LastUpdateEpoch: time.Now().Unix(),
}
}
func main() {
state := ProduceDefaultState()
jsonCompact := jsoniter.Config{TagKey: "compact"}.Froze()
bytes, err := jsonCompact.Marshal(state)
if err != nil {
panic(err)
}
jsonText := string(bytes)
fmt.Printf("Compact JSON length is: %v, compact JSON text is: %v", len(jsonText), jsonText)
}
and the output is:
Compact JSON length is: 102, compact JSON text is: {"a":0,"b":["load","build","deploy","test","deliver"],"c":{"a":5.55,"b":".*","c":true},"d":1691996466}
But can we do better?
What if out state is mostly static, and only a few fields are changing? Then we can list only the fields that change, and merge them in to the default config.
package main
import (
"encoding/json"
"fmt"
"radware.com/proximity/commons/global/reflectionapi"
"time"
)
type DetectorConfig struct {
AnomalyThreshold float32 `compact:"a"`
AnomalyPattern string `compact:"b"`
EnableDetection bool `compact:"c"`
}
type State struct {
Counter int `compact:"a"`
Stations []string `compact:"b"`
Detector DetectorConfig `compact:"c"`
LastUpdateEpoch int64 `compact:"d"`
}
func ProduceDefaultState() *State {
return &State{
Counter: 0,
Stations: []string{"load", "build", "deploy", "test", "deliver"},
Detector: DetectorConfig{
AnomalyThreshold: 5.55,
AnomalyPattern: ".*",
EnableDetection: true,
},
LastUpdateEpoch: time.Now().Unix(),
}
}
func main() {
state := ProduceDefaultState()
state.LastUpdateEpoch = time.Now().Add(time.Second).Unix()
state.Detector.EnableDetection = false
defaultState := ProduceDefaultState()
diffMap := reflectionapi.CreateDiffMap("compact", defaultState, state)
bytes, err := json.Marshal(diffMap)
if err != nil {
panic(err)
}
jsonText := string(bytes)
fmt.Printf("Diff JSON length is: %v, diff JSON text is: %v", len(jsonText), jsonText)
}
And the output is:
Diff JSON length is: 32, diff JSON text is: {"c":{"c":false},"d":1691996360}
Of course the length had significantly reduced, and will be reduced much further the bigger is our state, and the less updated fields it includes.
The reflection library is below:
package reflectionapi
import (
"fmt"
"reflect"
"strings"
)
type DiffHandler func(
elementPath string,
value1 interface{},
value2 interface{},
)
func FindDiff(
tagForName string,
item1 interface{},
item2 interface{},
handler DiffHandler,
) {
findDiffRecursive(
tagForName,
"",
item1,
item2,
handler,
)
}
func findDiffRecursive(
tagForName string,
prefix string,
item1 interface{},
item2 interface{},
handler DiffHandler,
) {
reflectType := reflect.TypeOf(item2).Elem()
reflectValue1 := reflect.ValueOf(item1).Elem()
reflectValue2 := reflect.ValueOf(item2).Elem()
for i := 0; i < reflectType.NumField(); i++ {
fieldType := reflectType.Field(i)
useName := fieldType.Name
if tagForName != "" {
useName = fieldType.Tag.Get(tagForName)
}
value1 := reflectValue1.Field(i).Interface()
value2 := reflectValue2.Field(i).Interface()
path := prefix + "/" + useName
switch reflectValue2.Field(i).Kind() {
case reflect.Struct:
interface1 := reflectValue1.Field(i).Addr().Interface()
interface2 := reflectValue2.Field(i).Addr().Interface()
findDiffRecursive(tagForName, path, interface1, interface2, handler)
break
case reflect.Slice:
value1String := fmt.Sprintf("%v", value1)
value2String := fmt.Sprintf("%v", value2)
if value1String != value2String {
handler(path, value1, value2)
}
break
default:
if value2 != value1 {
handler(path, value1, value2)
}
break
}
}
}
func CreateDiffMap(
tagForName string,
itemBaseline interface{},
itemChanged interface{},
) map[string]interface{} {
diffMap := make(map[string]interface{})
handler := func(elementPath string, valueBaseline interface{}, valueChanged interface{}) {
diffMapEntry := diffMap
sections := strings.Split(elementPath, "/")
sections = sections[1:]
for {
sectionName := sections[0]
if len(sections) == 1 {
diffMapEntry[sectionName] = valueChanged
break
} else {
sections = sections[1:]
nextEntry := diffMapEntry[sectionName]
if nextEntry == nil {
nextEntry = make(map[string]interface{})
diffMapEntry[sectionName] = nextEntry
}
diffMapEntry = nextEntry.(map[string]interface{})
}
}
}
FindDiff(tagForName, itemBaseline, itemChanged, handler)
return diffMap
}
