Analyze Redis Memory usage

We have used Redis as our persistence layer, and were very pleased from its performance.

Everything went fine for several months, until we've found that the production Redis memory consumption go out of control. We are using many GB of RAM on each one of the Redis cluster nodes.

How can we analyze the problem?

We have so many keys, for various components in the system.

How much RAM is used by each component?

What is the RAM distribution among each type of Redis key?

Gladly, I have found that Redis can assist us.
It had supplied the MEMORY USAGE command, which print the memory bytes used by each key.

BUT... It can only run for a single key, and we have millions of keys.

This is when I've decided to create a small GO application to analyze the memory distribution among the various keys. 

The general idea is that each component has different prefixes for its keys, for example:

• BOOK-book-name
• AUTHOR-author-name
• CUSTOMER-customer-id

So we can group the memory usage per each key prefix.

Here is the code that handles this.

package main

import (
   "fmt"
   "github.com/go-redis/redis/v7"
   "log"
   "strings"
)

type Memory struct {
   Key  string
   Size uint
}

type Stats struct {
   count uint
   min   uint
   max   uint
   sum   uint
   avg   uint
}

var client *redis.Client

func main() {
   client = redis.NewClient(&redis.Options{
      Addr:     "127.0.0.1:6379",
      Password: "",
      DB:       0,
   })

   memories := getMemories()

   prefixes := make(map[string]Stats)
   prefixes["BOOK"] = Stats{}
   prefixes["AUTHOR"] = Stats{}
   prefixes["CUSTOMER"] = Stats{}

   for _, memory := range memories {
      updateStats(prefixes, memory)
   }

   fmt.Printf("\n\nsummary\n\n")
   fmt.Printf("prefix,totalKeys,totalSize,avgKeySize,minKeySize,maxKeySize\n")
   for prefix, stats := range prefixes {
      if stats.count > 0 {
         stats.avg = stats.sum / stats.count
         fmt.Printf("%v,%+v,%+v,%+v,%+v,%+v\n", prefix, stats.count, stats.sum, stats.avg, stats.min, stats.max)
      }
   }
}

func getMemories() []Memory {
   cmd := client.Keys("*")
   err := cmd.Err()
   if err != nil {
      log.Fatal(err)
   }

   keys, err := cmd.Result()
   if err != nil {
      log.Fatal(err)
   }

   var memories []Memory
   for _, key := range keys {
      cmd := client.MemoryUsage(key)
      err := cmd.Err()
      if err != nil {
         log.Fatal(err)
      }

      value, err := cmd.Result()
      if err != nil {
         log.Fatal(err)
      }
      memory := Memory{
         Key:  key,
         Size: uint(value),
      }
      memories = append(memories, memory)
   }
   return memories
}

func updateStats(prefixes map[string]Stats, memory Memory) {
   for prefix, stats := range prefixes {
      if strings.HasPrefix(memory.Key, prefix) {
         updateByMemory(&stats, memory)
         prefixes[prefix] = stats
         return
      }
   }
   stats := Stats{}
   updateByMemory(&stats, memory)
   prefixes[memory.Key] = stats
}

func updateByMemory(stats *Stats, memory Memory) {
   stats.count++
   if stats.max < memory.Size {
      stats.max = memory.Size
   }
   if stats.min == 0 || stats.min > memory.Size {
      stats.min = memory.Size
   }
   stats.sum += memory.Size
}

The output of this small application is a CSV file:

And can be displayed of course as a chart:

Final Notes

While working on this, I've found what seems to be a bug on Redis.

For more details, check this question at stackoverflow, and the final result is that it was merged into the go-redis library in this pull request.