Full Blog TOC

Full Blog Table Of Content with Keywords Available HERE

Wednesday, December 4, 2024

Graphical Game in Go



In this post we review how to create a graphical application in Go.

The graphics is base on Ebitengine - a great game framework. For this post we create a bouncing balls animation. To use this library, first install the dependencies.

The main simply runs the game:

package main

import (
"balls/game"
"github.com/hajimehoshi/ebiten/v2"
"log"
)

func main() {
g := game.ProduceGame()
ebiten.SetWindowResizingMode(ebiten.WindowResizingModeEnabled)
ebiten.SetWindowTitle("Balls")

if err := ebiten.RunGame(g); err != nil {
log.Fatal(err)
}
}


The game class is the interaction with the ebittengine framework. It implements the Layout, Update, and Draw methods.


package game

import (
"balls/balls"
"balls/circle"
"balls/point"
"balls/rectangle"
"balls/velocity"
"github.com/hajimehoshi/ebiten/v2"
"image/color"
"math/rand/v2"
)

type Game struct {
balls []*balls.Ball
size *point.Point
bounds []*rectangle.Rectangle
}

func ProduceGame() *Game {
g := Game{
size: point.ProducePoint(1000, 1000),
}

wall := rectangle.ProduceRectangle(
&color.RGBA{
R: 40,
G: 40,
B: 40,
A: 0,
},
point.ProducePoint(0, 0),
g.size,
)
innerBox := rectangle.ProduceRectangle(
g.randomColor(),
point.ProducePoint(300, 300),
point.ProducePoint(600, 600),
)

g.bounds = []*rectangle.Rectangle{
wall,
innerBox,
}

for range 30 {
g.addBall()
}

return &g
}

func (g *Game) randomShort() uint8 {
return uint8(rand.Uint() % 255)
}

func (g *Game) randomColor() color.Color {
return &color.RGBA{
R: g.randomShort(),
G: g.randomShort(),
B: g.randomShort(),
A: 0,
}
}

func (g *Game) addBall() {
radiusLimit := rand.Float32() * g.size.GetX() / 10
radius := radiusLimit
center := point.RandomPoint(g.size.AddAxis(-2 * radius))
center = center.AddAxis(radius)

ballCircle := circle.ProduceCircle(center, radius)
newBall := balls.ProduceBall(
ballCircle,
g.randomColor(),
velocity.ProduceRandomVelocity(4+radiusLimit-radius),
)
g.balls = append(g.balls, newBall)
}

func (g *Game) Update() error {
for _, ball := range g.balls {
ball.Update(g.bounds)
}
return nil
}

func (g *Game) Draw(screen *ebiten.Image) {
for _, bound := range g.bounds {
bound.Draw(screen)
}
for _, ball := range g.balls {
ball.Draw(screen)
}
}

func (g *Game) Layout(_, _ int) (screenWidth, screenHeight int) {
return g.size.GetXTruncated(), g.size.GetYTruncated()
}


We have basic point class:

package point

import (
"fmt"
"math"
"math/rand/v2"
)

type Point struct {
x float32
y float32
}

func ProducePoint(
x float32,
y float32,
) *Point {
return &Point{
x: x,
y: y,
}
}

func RandomPoint(
limits *Point,
) *Point {
x := rand.Float32() * limits.GetX()
y := rand.Float32() * limits.GetY()
return ProducePoint(x, y)
}

func (p *Point) GetX() float32 {
return p.x
}

func (p *Point) GetY() float32 {
return p.y
}

func (p *Point) GetXTruncated() int {
return int(p.x)
}

func (p *Point) GetYTruncated() int {
return int(p.y)
}

func (p *Point) AddPoint(
other *Point,
) *Point {
return ProducePoint(p.x+other.x, p.y+other.y)
}

func (p *Point) AddAxis(
value float32,
) *Point {
return ProducePoint(p.x+value, p.y+value)
}

func (p *Point) DivideAxis(
value float32,
) *Point {
return ProducePoint(p.x/value, p.y/value)
}

func (p *Point) MultiplyAxis(
value float32,
) *Point {
return ProducePoint(p.x*value, p.y*value)
}

func (p *Point) SubtractPoint(
other *Point,
) *Point {
return ProducePoint(p.x-other.x, p.y-other.y)
}

func (p *Point) String() string {
return fmt.Sprintf("(%.2f, %.2f)", p.x, p.y)
}

func (p *Point) Distance(
other *Point,
) float32 {
delta := p.SubtractPoint(other)
return float32(math.Sqrt(float64(delta.x*delta.x + delta.y*delta.y)))
}


And a circle class:


package circle

import (
"balls/point"
"fmt"
)

type Circle struct {
center *point.Point
radius float32
}

func ProduceCircle(
center *point.Point,
radius float32,
) *Circle {
return &Circle{
center: center,
radius: radius,
}
}

func (c *Circle) GetCenter() *point.Point {
return c.center
}

func (c *Circle) GetRadius() float32 {
return c.radius
}

func (c *Circle) String() string {
return fmt.Sprintf("center %v, radius %v", c.center, c.radius)
}


The line class is more complex as it need to find intersection points:


package line

import (
"balls/circle"
"balls/kitmath"
"balls/point"
"fmt"
"math"
)

type Line struct {
start *point.Point
end *point.Point
}

func ProduceLine(
start *point.Point,
end *point.Point,
) *Line {
return &Line{
start: start,
end: end,
}
}

func (l *Line) Extend(
distance float32,
) *Line {
delta := l.end.SubtractPoint(l.start)
magnitude := math.Sqrt(float64(delta.GetX()*delta.GetX() + delta.GetY()*delta.GetY()))
normalized := delta.DivideAxis(float32(magnitude))
extendedVector := normalized.MultiplyAxis(distance)

newEnd := point.ProducePoint(
l.end.GetX()+extendedVector.GetX(),
l.end.GetY()+extendedVector.GetY(),
)

return ProduceLine(
l.start,
newEnd,
)
}

func (l *Line) GetStart() *point.Point {
return l.start

}

func (l *Line) IsVertical() bool {
return l.start.GetX() == l.end.GetX()
}

func (l *Line) FindIntersectionLine(
other *Line,
) *point.Point {
if l.IsVertical() || other.IsVertical() {
return l.findIntersectionAtLeastOneVertical(other)
}
return l.findIntersectionNonVertical(other)
}

func (l *Line) findIntersectionAtLeastOneVertical(
other *Line,
) *point.Point {
if l.IsVertical() && other.IsVertical() {
// if we have multiple crossing point - ignore
return nil
}

if l.IsVertical() {
return l.findIntersectionOnlyIVertical(other)
}
return other.findIntersectionOnlyIVertical(l)
}

func (l *Line) findIntersectionNonVertical(
other *Line,
) *point.Point {
mySlope := l.getEquationSlope()
otherSlope := other.getEquationSlope()
if kitmath.Float32Equal(mySlope, otherSlope) {
// if we have multiple crossing point - ignore
return nil
}

myB := l.getEquationB()
otherB := other.getEquationB()

x := (otherB - myB) / (mySlope - otherSlope)
y := mySlope*x + myB

intersection := point.ProducePoint(x, y)
if l.containsPoint(intersection) && other.containsPoint(intersection) {
return intersection
}

return nil
}

func (l *Line) containsY(
y float32,
) bool {
minY := min(l.start.GetY(), l.end.GetY())
maxY := max(l.start.GetY(), l.end.GetY())
return minY <= y && y <= maxY
}

func (l *Line) containsX(
x float32,
) bool {
minX := min(l.start.GetX(), l.end.GetX())
maxX := max(l.start.GetX(), l.end.GetX())
return minX <= x && x <= maxX
}

func (l *Line) getEquationSlope() float32 {
delta := l.start.SubtractPoint(l.end)
return delta.GetY() / delta.GetX()
}

func (l *Line) getEquationB() float32 {
slope := l.getEquationSlope()
return l.start.GetY() - slope*l.start.GetX()
}

func (l *Line) findIntersectionOnlyIVertical(
other *Line,
) *point.Point {
x := l.start.GetX()

if !other.containsX(x) {
return nil
}

y := other.getEquationSlope()*x + other.getEquationB()
if !l.containsY(y) {
return nil
}

return point.ProducePoint(x, y)
}

func (l *Line) containsPoint(
location *point.Point,
) bool {
return l.containsX(location.GetX()) && l.containsY(location.GetY())
}

func (l *Line) FindIntersectionCircle(
circle *circle.Circle,
) []*point.Point {
var limitedIntersections []*point.Point
intersections := l.findIntersectionCircleUnlimited(circle)
for _, intersection := range intersections {
if !l.containsPoint(intersection) {
continue
}
limitedIntersections = append(limitedIntersections, intersection)
}

return limitedIntersections
}

func (l *Line) findIntersectionCircleUnlimited(
circle *circle.Circle,
) []*point.Point {
if l.IsVertical() {
return l.findIntersectionCircleVertical(circle)
}
return l.findIntersectionCircleHorizontal(circle)
}

func (l *Line) findIntersectionCircleVertical(
circle *circle.Circle,
) []*point.Point {
c := l.start.GetX()
x0 := circle.GetCenter().GetX()
y0 := circle.GetCenter().GetY()
r := circle.GetRadius()

inner := r*r - (c-x0)*(c-x0)
if inner < 0 {
return nil
}

if inner == 0 {
return []*point.Point{
point.ProducePoint(c, y0),
}
}

innerSqrt := float32(math.Sqrt(float64(inner)))
return []*point.Point{
point.ProducePoint(c, y0+innerSqrt),
point.ProducePoint(c, y0-innerSqrt),
}
}

func (l *Line) findIntersectionCircleHorizontal(
circle *circle.Circle,
) []*point.Point {
c := l.start.GetY()
x0 := circle.GetCenter().GetX()
y0 := circle.GetCenter().GetY()
r := circle.GetRadius()

inner := r*r - (c-y0)*(c-y0)
if inner < 0 {
return nil
}

if inner == 0 {
return []*point.Point{
point.ProducePoint(x0, c),
}
}

innerSqrt := float32(math.Sqrt(float64(inner)))
return []*point.Point{
point.ProducePoint(x0+innerSqrt, c),
point.ProducePoint(x0-innerSqrt, c),
}
}

func (l *Line) String() string {
return fmt.Sprintf("start %v end %v", l.start, l.end)
}


The rectangle class is used to represent both the screen edges, and obstacles.


package rectangle

import (
"balls/circle"
"balls/line"
"balls/point"
"github.com/hajimehoshi/ebiten/v2"
"github.com/hajimehoshi/ebiten/v2/vector"
"image/color"
)

type Rectangle struct {
color color.Color
topLeft *point.Point
bottomRight *point.Point
}

func ProduceRectangle(
color color.Color,
topLeft *point.Point,
bottomRight *point.Point,
) *Rectangle {
return &Rectangle{
color: color,
topLeft: topLeft,
bottomRight: bottomRight,
}
}

func (r *Rectangle) getLines() []*line.Line {
topRight := point.ProducePoint(r.bottomRight.GetX(), r.topLeft.GetY())
bottomLeft := point.ProducePoint(r.topLeft.GetX(), r.bottomRight.GetY())

return []*line.Line{
line.ProduceLine(r.topLeft, topRight),
line.ProduceLine(topRight, r.bottomRight),
line.ProduceLine(r.bottomRight, bottomLeft),
line.ProduceLine(bottomLeft, r.topLeft),
}
}

func (r *Rectangle) FindIntersectionCircle(
circle *circle.Circle,
) (*point.Point, *line.Line) {
var minDistance float32
var intersectionLine *line.Line
var intersectionPoint *point.Point
for _, lineBound := range r.getLines() {
boundIntersections := lineBound.FindIntersectionCircle(circle)
for _, boundIntersection := range boundIntersections {
distance := circle.GetCenter().Distance(boundIntersection)
if intersectionLine == nil || distance < minDistance {
minDistance = distance
intersectionLine = lineBound
intersectionPoint = boundIntersection
}
}
}
return intersectionPoint, intersectionLine
}

func (r *Rectangle) FindIntersectionLine(
moveLine *line.Line,
) (*point.Point, *line.Line) {
var minDistance float32
var intersectionLine *line.Line
var intersectionPoint *point.Point
for _, lineBound := range r.getLines() {
boundIntersection := lineBound.FindIntersectionLine(moveLine)
if boundIntersection == nil {
continue
}
distance := moveLine.GetStart().Distance(boundIntersection)
if intersectionLine == nil || distance < minDistance {
minDistance = distance
intersectionLine = lineBound
intersectionPoint = boundIntersection
}
}
return intersectionPoint, intersectionLine
}

func (r *Rectangle) Draw(screen *ebiten.Image) {
delta := r.bottomRight.SubtractPoint(r.topLeft)
vector.DrawFilledRect(
screen,
r.topLeft.GetX(),
r.topLeft.GetY(),
delta.GetX(),
delta.GetY(),
r.color,
false,
)
}



The velocity class is:


package velocity

import (
"balls/line"
"balls/point"
"fmt"
"math"
"math/rand/v2"
)

type Velocity struct {
speed float32
angleRadian float32
}

func ProduceVelocity(
speed float32,
angle float32,
) *Velocity {
return &Velocity{
speed: speed,
angleRadian: angle,
}
}
func ProduceRandomVelocity(
speedLimit float32,
) *Velocity {
speed := rand.Float32() * speedLimit
angle := rand.Float32() * 2 * math.Pi
return ProduceVelocity(speed, angle)
}

func (v *Velocity) Apply(
location *point.Point,
) *point.Point {
x := location.GetX() + v.speed*float32(math.Cos(float64(v.angleRadian)))
y := location.GetY() + v.speed*float32(math.Sin(float64(v.angleRadian)))
return point.ProducePoint(x, y)
}

func (v *Velocity) isLeft() bool {
return v.angleRadian > math.Pi/2 && v.angleRadian < 1.5*math.Pi
}

func (v *Velocity) isRight() bool {
return v.angleRadian < math.Pi/2 || v.angleRadian > 1.5*math.Pi
}

func (v *Velocity) Bounce(
line *line.Line,
) *Velocity {
if line.IsVertical() {
return v.bounceVertical()
}
return v.bounceHorizontal()
}

func (v *Velocity) bounceVertical() *Velocity {
if v.angleRadian < math.Pi {
return ProduceVelocity(v.speed, math.Pi-v.angleRadian)
}
return ProduceVelocity(v.speed, 3*math.Pi-v.angleRadian)
}

func (v *Velocity) bounceHorizontal() *Velocity {
return ProduceVelocity(v.speed, 2*math.Pi-v.angleRadian)
}

func (v *Velocity) String() string {
return fmt.Sprintf("speed %.2f angle %.2f", v.speed, v.angleRadian)
}


And the ball class represents a ball with circle an velocity. It handles it own movement.


package balls

import (
"balls/circle"
"balls/line"
"balls/rectangle"
"balls/velocity"
"fmt"
"github.com/hajimehoshi/ebiten/v2"
"github.com/hajimehoshi/ebiten/v2/vector"
"image/color"
)

type Ball struct {
circle *circle.Circle
color color.Color
velocity *velocity.Velocity
}

func ProduceBall(
circle *circle.Circle,
color color.Color,
velocity *velocity.Velocity,
) *Ball {
return &Ball{
circle: circle,
color: color,
velocity: velocity,
}
}

func (b *Ball) Update(
bounds []*rectangle.Rectangle,
) {
newCenter := b.velocity.Apply(b.circle.GetCenter())
newCircle := circle.ProduceCircle(newCenter, b.circle.GetRadius())

for _, bound := range bounds {
currentIntersection, _ := bound.FindIntersectionCircle(b.circle)
if currentIntersection != nil {
// while we are intersecting, we do not bounce
b.circle = circle.ProduceCircle(newCenter, b.circle.GetRadius())
return
}
}

var minDistance *float32
var bouncedLine *line.Line
for _, bound := range bounds {
intersectionPoint, intersectionLine := bound.FindIntersectionCircle(newCircle)
if intersectionPoint == nil {
continue
}

distance := b.circle.GetCenter().Distance(intersectionPoint)
if minDistance == nil || distance < *minDistance {
minDistance = &distance
bouncedLine = intersectionLine
}
}

if minDistance != nil {
b.velocity = b.velocity.Bounce(bouncedLine)
newCenter = b.velocity.Apply(b.circle.GetCenter())
}

b.circle = circle.ProduceCircle(newCenter, b.circle.GetRadius())
}

func (b *Ball) Draw(screen *ebiten.Image) {
vector.DrawFilledCircle(
screen,
b.circle.GetCenter().GetX(),
b.circle.GetCenter().GetY(),
b.circle.GetRadius(),
b.color,
false,
)
}

func (b *Ball) String() string {
return fmt.Sprintf("%v velocity %v color %v", b.circle, b.velocity, b.color)
}


We also have a math helper class:

package kitmath

var grace = float32(0.001)

func Float32Equal(
f1 float32,
f2 float32,
) bool {
delta := f1 - f2
return -grace <= delta && delta <= grace
}



Wednesday, November 27, 2024

Redpanda Connect Introduction

 



Redpanda Connect, previously known as Benthos, is a streaming pipeline that reads and write messages from/to many connectors. It enables transforming the messages using built-in processors. The goal of this framework is to enable us connect and convert data streams without writing a proprietary code, but only using a configuration file with minimal dedicated processing code. 


To install use the following:

curl -LO https://github.com/redpanda-data/redpanda/releases/latest/download/rpk-linux-amd64.zip
unzip rpk-linux-amd64.zip
sudo mv rpk /usr/local/bin/
rm rpk-linux-amd64.zip


Create a file named connect.yaml:


input:
stdin: {}

pipeline:
processors:
- mapping: root = content().uppercase()

output:
stdout: {}


And run it:

rpk connect run connect.yaml


Now any input to the STDIN is copied to the STDOUT.


While Redpanda connect can manage string messages, most f its abilities are built toward JSON messages handling. For example, consider the following connect.yaml file:


input:
generate:
interval: 1s
count: 0
mapping: |
let first_name = fake("first_name")
let last_name = fake("last_name")

root.id = counter()
root.name = ($first_name + " " + $last_name)
root.timestamp = now()

pipeline:
processors:
- sleep:
duration: 100ms
- group_by:
- check: this.id % 2 == 0
processors:
- mapping: |
root.original_doc = this
root.encoded = this.name.hash("sha256").encode("base64")

output:
stdout: {}


It will generate the following output:

{"id":1,"name":"Ivah Mohr","timestamp":"2024-11-27T16:36:37.542410543+02:00"}
{"encoded":"oaaBKF/7oz0N6j6VlSZs14u8FD2dAwPSBAoJvIIMpWI=","original_doc":{"id":2,"name":"Darrion Miller","timestamp":"2024-11-27T16:36:38.541429372+02:00"}}
{"id":3,"name":"Maddison Paucek","timestamp":"2024-11-27T16:36:39.542047832+02:00"}
{"encoded":"+e6EyBEILJYStdV+DH6cohEVvfW04VTo1q2YLXp4ft8=","original_doc":{"id":4,"name":"Eleazar Sporer","timestamp":"2024-11-27T16:36:40.542410537+02:00"}}
{"id":5,"name":"Junius Renner","timestamp":"2024-11-27T16:36:41.542298415+02:00"}
{"encoded":"A4o97ySPf9yWFMXcutPBiI6a6Fd19ofqBmK/7s84ZZ4=","original_doc":{"id":6,"name":"Carlie Osinski","timestamp":"2024-11-27T16:36:42.542277315+02:00"}}
{"id":7,"name":"Raphaelle Reichel","timestamp":"2024-11-27T16:36:43.542382662+02:00"}
{"encoded":"U+pLsYjRaWH87nUFdZmRLJwvGrIfUOCYeUrazKGoEHA=","original_doc":{"id":8,"name":"Elmira Douglas","timestamp":"2024-11-27T16:36:44.542245761+02:00"}}
{"id":9,"name":"Ambrose Hudson","timestamp":"2024-11-27T16:36:45.542307678+02:00"}
{"encoded":"J2NxrIaC1cvHtqIduSOx85TlyFLQrT48QaYw8iR9To0=","original_doc":{"id":10,"name":"Jedidiah Veum","timestamp":"2024-11-27T16:36:46.542276779+02:00"}}



Final Words

Yeah, that's cool: You save time coding and fixing bugs. This is nice if you need a simple fast and reliable pipeline.
However, in real life you will usually require more the the builtin abilities, and then you need to extend Redpanda Connect with your own code in GO. Is it better than just writing all in GO? I think it depends on how much bugs do you expect from the programmer.







Monday, November 18, 2024

Go ReadLine for Long Lines


 


In this post we present the correct method of reading lines from a long text in GO.


The naive method of reading lines in go is using bufio.Scanner:

scanner := bufio.NewScanner(file)
for scanner.Scan() {
fmt.Println(scanner.Text())
}


This does not work in case of long lines. Instead, we should use bufio.Reader. However, the compiling of long lines is cumbersome, and a KISS wrapper is missing in the GO standard library, hence I've created it myself.


import (
"bufio"
"io"
"strings"
)

type LineReader struct {
reader *bufio.Reader
isEof bool
}

func ProduceLineReader(
text string,
) *LineReader {
reader := bufio.NewReader(strings.NewReader(text))

return &LineReader{
reader: reader,
isEof: false,
}
}

func (r *LineReader) GetLine() string {
var longLineBuffer *strings.Builder
multiLines := false
for {
line, isPrefix, err := r.reader.ReadLine()
if err == io.EOF {
r.isEof = true
return ""
}

if isPrefix {
multiLines = true
}

if !multiLines {
// simple single line
return string(line)
}

if longLineBuffer == nil {
// create only if needed - better performance
longLineBuffer = &strings.Builder{}
}

longLineBuffer.Write(line)
if !isPrefix {
// end of long line
return longLineBuffer.String()
}
}
}

func (r *LineReader) IsEof() bool {
return r.isEof
}


An example of usage is:

reader := kitstring.ProduceLineReader(text)
for {
line := reader.GetLine()
if reader.IsEof() {
break
}
t.Log("read line: %v", line)
}



Monday, November 11, 2024

OpenAPI Schema In Go



 

In this post we present a utilty to produce an OpenAPI file using GO commands. This is useful in case the application detects the schema, and is required to supply an OpenAPI file of the detected schema.



package openapi

import (
"gopkg.in/yaml.v3"
"strings"
)

const applicationJson = "application/json"

type InLocation int

const (
InLocationQuery InLocation = iota + 1
InLocationHeader
InLocationCookie
InLocationPath
)

type SchemaType int

const (
SchemaTypeString SchemaType = iota + 1
SchemaTypeInt
SchemaTypeObject
SchemaTypeArray
)

type Info struct {
Title string `yaml:"title,omitempty"`
Description string `yaml:"description,omitempty"`
Version string `yaml:"version,omitempty"`
}

type Schema struct {
Type string `yaml:"type,omitempty"`
Enum []string `yaml:"enum,omitempty"`
Properties map[string]*Schema `yaml:"properties,omitempty"`
Items *Schema `yaml:"items,omitempty"`
}

type Parameter struct {
In string `yaml:"in,omitempty"`
Name string `yaml:"name,omitempty"`
Description string `yaml:"description,omitempty"`
Required bool `yaml:"required,omitempty"`
Schema *Schema `yaml:"schema,omitempty"`
}

type Content map[string]*Schema

type RequestBody struct {
Description string `yaml:"description,omitempty"`
Required bool `yaml:"required,omitempty"`
Content *Content `yaml:"content,omitempty"`
}

type Response struct {
Description string `yaml:"description,omitempty"`
Content *Content `yaml:"content,omitempty"`
}

type Method struct {
Summary string `yaml:"summary,omitempty"`
Description string `yaml:"description,omitempty"`
Deprecated string `yaml:"deprecated,omitempty"`
Parameters []*Parameter `yaml:"parameters,omitempty"`
RequestBody *RequestBody `yaml:"requestBody,omitempty"`
Responses map[string]*Response `yaml:"responses,omitempty"`
}

type Path map[string]*Method

type OpenApi struct {
OpenApi string `yaml:"openapi,omitempty"`
Info *Info `yaml:"info,omitempty"`
Paths map[string]*Path `yaml:"paths,omitempty"`
}

func produceSchema() *Schema {
return &Schema{
Properties: make(map[string]*Schema),
}
}

func ProduceOpenApi() *OpenApi {
return &OpenApi{
OpenApi: "3.0.0",
Paths: make(map[string]*Path),
}
}

func (o *OpenApi) CreateYamlBytes() []byte {
bytes, err := yaml.Marshal(o)
kiterr.RaiseIfError(err)
return bytes
}

func (o *OpenApi) CreateYamlString() string {
return string(o.CreateYamlBytes())
}

func (o *OpenApi) SetPath(
path string,
) *Path {
for pathUrl, pathObject := range o.Paths {
if pathUrl == path {
return pathObject
}
}
pathObject := make(Path)
o.Paths[path] = &pathObject
return &pathObject
}

func (p *Path) SetMethod(
method string,
) *Method {
method = strings.ToLower(method)

pathObject := *p
existingMethod := pathObject[method]
if existingMethod != nil {
return existingMethod
}
methodObject := Method{
Responses: make(map[string]*Response),
}
pathObject[method] = &methodObject
return &methodObject
}

func (m *Method) SetParameter(
name string,
) *Parameter {
for _, parameter := range m.Parameters {
if parameter.Name == name {
return parameter
}
}

parameter := Parameter{
Name: name,
}
m.Parameters = append(m.Parameters, &parameter)
return &parameter
}

func (p *Parameter) SetInLocation(
in InLocation,
) *Parameter {
switch in {
case InLocationQuery:
p.In = "query"
case InLocationCookie:
p.In = "cookie"
case InLocationHeader:
p.In = "header"
case InLocationPath:
p.In = "path"
}
return p
}

func (p *Parameter) SetSchema(
schemaType SchemaType,
) *Parameter {
schema := p.Schema
if schema == nil {
schema = produceSchema()
p.Schema = schema
}

schema.SetType(schemaType)
return p
}

func (s *Schema) SetType(
schemaType SchemaType,
) *Schema {
switch schemaType {
case SchemaTypeString:
s.Type = "string"
case SchemaTypeInt:
s.Type = "integer"
case SchemaTypeObject:
s.Type = "object"
case SchemaTypeArray:
s.Type = "array"
}
return s
}

func (s *Schema) SetProperty(
name string,
schemaType SchemaType,
) *Schema {
property := s.Properties[name]
if property == nil {
property = produceSchema()
s.Properties[name] = property
}

property.SetType(schemaType)
return property
}

func (s *Schema) SetPropertyArray(
name string,
) *Schema {
array := s.SetProperty(name, SchemaTypeArray)
array.Items = produceSchema()
return array.Items
}

func (m *Method) SetRequestContent(
contentType string,
) *Schema {
body := m.RequestBody
if body == nil {
body = &RequestBody{}
m.RequestBody = body
}

content := body.Content
if content == nil {
content = &Content{}
body.Content = content
}

contentObject := *content
schema := contentObject[contentType]
if schema == nil {
schema = produceSchema()
contentObject[contentType] = schema
}

return schema
}

func (m *Method) SetContentApplicationJson() *Schema {
return m.SetRequestContent(applicationJson)
}

func (m *Method) SetResponseContent(
responseCode string,
contentType string,
) *Schema {
response := m.Responses[responseCode]
if response == nil {
response = &Response{}
m.Responses[responseCode] = response
}

content := response.Content
if content == nil {
content = &Content{}
response.Content = content
}

contentObject := *content
schema := contentObject[contentType]
if schema == nil {
schema = produceSchema()
contentObject[contentType] = schema
}

return schema
}

func (m *Method) SetResponseSuccessContentApplicationJson() *Schema {
return m.SetResponseContent("200", applicationJson)
}



The sample usage below creates 2 endpoints of list-items and add-item.


api := openapi.ProduceOpenApi()

method := api.SetPath("/api/list-items").SetMethod("GET")
method.Description = "list all store items"

method.SetParameter("store-id").
SetInLocation(openapi.InLocationQuery).
SetSchema(openapi.SchemaTypeInt)

listStoreSchema := method.SetResponseSuccessContentApplicationJson()
existingItemSchema := listStoreSchema.SetPropertyArray("items")
existingItemSchema.SetProperty("id", openapi.SchemaTypeInt)
existingItemSchema.SetProperty("name", openapi.SchemaTypeString)
existingItemSchema.SetProperty("price", openapi.SchemaTypeInt)

method = api.SetPath("/api/add-item").SetMethod("POST")
method.Description = "add item to store"
addItemSchema := method.SetContentApplicationJson()
addItemSchema.SetType(openapi.SchemaTypeObject)
newItemSchema := addItemSchema.SetProperty("item", openapi.SchemaTypeObject)
newItemSchema.SetProperty("name", openapi.SchemaTypeString)
newItemSchema.SetProperty("price", openapi.SchemaTypeInt)

t.Log("Schema is:\n\n%v", api.CreateYamlString())


The result openAPI file is:


openapi: 3.0.0
paths:
/api/add-item:
post:
description: add item to store
requestBody:
content:
application/json:
type: object
properties:
item:
type: object
properties:
name:
type: string
price:
type: integer
/api/list-items:
get:
description: list all store items
parameters:
- in: query
name: store-id
schema:
type: integer
responses:
"200":
content:
application/json:
properties:
items:
type: array
items:
properties:
id:
type: integer
name:
type: string
price:
type: integer



Tuesday, November 5, 2024

Streamlit


 


In this post we present a simple example of a streamlit based application. Streamlit is a framework providing simple and fast GUI development for internal use. The nice thing is that it is very simple, the coding is pure python without any Javascript and without multiple processes, Notice that the GUI is not for the end user, but mostly for internal developers use, since due to its simplicity it is also limited.



The following code handles configuration for a long running process, and rerunning the process if and only if the configuration changes. Some screenshots are below:








The simple python code is below.


import random
import time
from datetime import datetime

import pandas as pd
import streamlit as st


def display_bars(amount):
data = []
for i in range(amount):
data.append(['person {}'.format(i), random.randint(10, 50)])
df = pd.DataFrame(data, columns=['Name', 'Age'])
st.bar_chart(df, x="Name", y="Age")


@st.cache_data
def run_long_processing(compute_time):
progress = st.empty()

with progress.container():
st.text('starting with {} times'.format(compute_time))
for i in range(compute_time):
time.sleep(1)
st.text('iteration {}'.format(i))
time.sleep(1)
progress.empty()

st.text('task with {} times is complete'.format(compute_time))
display_bars(compute_time)


@st.fragment(run_every='1s')
def display_time():
st.write(datetime.now())


def main():
st.title('Demo Application')
display_time()

if st.button('clear cache'):
st.cache_data.clear()

tab1, tab2 = st.tabs(['config', 'results'])

with tab1:
st.session_state['compute_time'] = st.slider('select compute time', 1, 10, 3)
with tab2:
compute_time = st.session_state['compute_time']
run_long_processing(compute_time)


main()




Tuesday, October 29, 2024

Grafana K6


 

In this post we review using Grafana K6, a tool for testing applications, using a java script like scripts.

We will use the following test file as our test:

test.js

import http from 'k6/http';
import {check, sleep} from 'k6';

export const options = {
vus: 2,
duration: '5s',
};
export default function () {
const url = 'http://test.k6.io/login';
const payload = JSON.stringify({
email: 'aaa',
password: 'bbb',
});

const params = {
headers: {
'Content-Type': 'application/json',
},
};

const result = http.post(url, payload, params);
check(result, {
'is status 200': (r) => r.status === 200,
});
sleep(1);
}



And we will review 2 methods of running the tool: docker and kubernetes.


Running in docker

To run in docker, we use the following dockerfile

Dockerfile

FROM grafana/k6
WORKDIR /usr/src/app
COPY . .
CMD [ "k6", "run", "test.js" ]


Next we build and run the docker

#!/usr/bin/env bash

set -e
cd "$(dirname "$0")"

docker build . -t my-k6
docker run --rm my-k6 run test.js


The output is:

[+] Building 1.1s (8/8) FINISHED                                                                                                                                  docker:default
=> [internal] load build definition from Dockerfile 0.0s
=> => transferring dockerfile: 114B 0.0s
=> [internal] load metadata for docker.io/grafana/k6:latest 0.7s
=> [internal] load .dockerignore 0.0s
=> => transferring context: 2B 0.0s
=> [1/3] FROM docker.io/grafana/k6:latest@sha256:d39047ea6c5981ac0abacec2ea32389f22a7aa68bc8902c08b356cc5dd74aac9 0.0s
=> [internal] load build context 0.0s
=> => transferring context: 904B 0.0s
=> CACHED [2/3] WORKDIR /usr/src/app 0.0s
=> [3/3] COPY . . 0.1s
=> exporting to image 0.1s
=> => exporting layers 0.0s
=> => writing image sha256:dd14bff333a7714cf09ff4c60a6ae820174bb575404ed4c63acf871a27148878 0.0s
=> => naming to docker.io/library/my-k6 0.0s

/\ Grafana /‾‾/
/\ / \ |\ __ / /
/ \/ \ | |/ / / ‾‾\
/ \ | ( | (‾) |
/ __________ \ |_|\_\ \_____/

execution: local
script: test.js
output: -

scenarios: (100.00%) 1 scenario, 2 max VUs, 35s max duration (incl. graceful stop):
* default: 2 looping VUs for 5s (gracefulStop: 30s)


running (01.0s), 2/2 VUs, 0 complete and 0 interrupted iterations
default [ 20% ] 2 VUs 1.0s/5s

running (02.0s), 2/2 VUs, 0 complete and 0 interrupted iterations
default [ 40% ] 2 VUs 2.0s/5s

running (03.0s), 2/2 VUs, 2 complete and 0 interrupted iterations
default [ 60% ] 2 VUs 3.0s/5s

running (04.0s), 2/2 VUs, 4 complete and 0 interrupted iterations
default [ 80% ] 2 VUs 4.0s/5s

running (05.0s), 2/2 VUs, 6 complete and 0 interrupted iterations
default [ 100% ] 2 VUs 5s

running (06.0s), 2/2 VUs, 6 complete and 0 interrupted iterations
default ↓ [ 100% ] 2 VUs 5s

✗ is status 200
↳ 0% — ✓ 0 / ✗ 8

checks.........................: 0.00% 0 out of 8
data_received..................: 16 kB 2.6 kB/s
data_sent......................: 3.8 kB 605 B/s
http_req_blocked...............: avg=75.92ms min=4.21µs med=17.25µs max=384.4ms p(90)=302.09ms p(95)=379.63ms
http_req_connecting............: avg=176.26µs min=0s med=0s max=717.99µs p(90)=703.73µs p(95)=707.57µs
http_req_duration..............: avg=191.36ms min=150.99ms med=156.8ms max=354.78ms p(90)=252.73ms p(95)=278.68ms
{ expected_response:true }...: avg=202.91ms min=153.28ms med=167.17ms max=354.78ms p(90)=283.75ms p(95)=319.27ms
http_req_failed................: 50.00% 8 out of 16
http_req_receiving.............: avg=155.79µs min=77.92µs med=154.46µs max=292.35µs p(90)=237.78µs p(95)=269.18µs
http_req_sending...............: avg=147.67µs min=27.02µs med=81.32µs max=533.87µs p(90)=296.64µs p(95)=377.56µs
http_req_tls_handshaking.......: avg=47.53ms min=0s med=0s max=383.46ms p(90)=188.55ms p(95)=378.7ms
http_req_waiting...............: avg=191.06ms min=150.59ms med=156.55ms max=353.98ms p(90)=252.5ms p(95)=278.36ms
http_reqs......................: 16 2.55197/s
iteration_duration.............: avg=1.53s min=1.3s med=1.33s max=2.21s p(90)=2.1s p(95)=2.16s
iterations.....................: 8 1.275985/s
vus............................: 2 min=2 max=2
vus_max........................: 2 min=2 max=2


running (06.3s), 0/2 VUs, 8 complete and 0 interrupted iterations
default ✓ [ 100% ] 2 VUs 5s


Running in kubernetes

To run in kubernetes, we deploy a k6 operator which will create jobs for test runs.


curl https://raw.githubusercontent.com/grafana/k6-operator/main/bundle.yaml | kubectl apply -f -


The outputs from the testrun in this case are sent to prometheus, so we create a dedicated image that handles the write to promethues.

Dockerfile

FROM golang:1.20 AS builder

RUN go install go.k6.io/xk6/cmd/xk6@latest

RUN xk6 build \
--with github.com/grafana/xk6-output-prometheus-remote@latest \
--output /k6

FROM grafana/k6:latest
COPY --from=builder /k6 /usr/bin/k6


Build the test runner image:

docker build -t k6-extended:local .
kind load docker-image k6-extended:local


We use the TestRun CRD to configure the test run.


k8s_testrun.yml

apiVersion: k6.io/v1alpha1
kind: TestRun
metadata:
name: my-testrun
spec:
cleanup: post
parallelism: 2
script:
configMap:
name: my-test
file: test.js
runner:
image: k6-extended:local
env:
- name: K6_PROMETHEUS_RW_SERVER_URL
value: http://prometheus/api/v1/write


And we run the test:


kubectl delete configmap my-test --ignore-not-found=true
kubectl create configmap my-test --from-file ./test.js
kubectl apply -f ./k8s_testrun.yml



The results can be viewed in grafana using a predefined dashboard.




Monday, October 14, 2024

Split Load Using NATS Partitions



 

In this post we will review NATS partitioning and how to use it to split load among multiple pods.

We've already reviewed the steps to setup a NATS cluster in kubernetes in this post. As part of the NATS statefulset template we have a config map, which is mounted into the NATS server container.



- name: nats
args:
- --config
- /etc/nats-config/nats.conf


volumeMounts:
- mountPath: /etc/nats-config
name: config


volumes:
- configMap:
name: nats-config



The ConfigMap is as follows:


apiVersion: v1
kind: ConfigMap
metadata:
name: nats-config
data:
nats.conf: |
{
"cluster": {
"name": "nats",
"no_advertise": true,
"port": 6222,
"routes": [
"nats://nats-0.nats-headless:6222",
"nats://nats-1.nats-headless:6222"
]
},
"http_port": 8222,
"lame_duck_duration": "30s",
"lame_duck_grace_period": "10s",
"pid_file": "/var/run/nats/nats.pid",
"port": 4222,
"server_name": $SERVER_NAME,
"mappings": {
"application.*": "application.{{partition(3,1)}}.{{wildcard(1)}}"
}
}



The NATS partitioning is configured by the mapping section in the config. In this case the producer publish messages to the NATS queue "application-<APPLICATION_ID>".

We want to split the load among 3 pods, and hence we configure the following:

"application.*": "application.{{partition(3,1)}}.{{wildcard(1)}}"


This provides the following instruction to the NATS:

Take the value used in the first wild card "application.*" , and use it to split into 3 partitions 0,1,2. Then publish the message in the following queue "application.<PARTITION_ID>.<APPLICATION ID>".


Now we can have 3 pods, each subsribing to the prefix: "application.<PARTITION_ID>.*", and the messages are split accordingly. Notice that a specific APPLICATION ID is always assigned to the same PARTITION_ID.

Also notice that this is a static assignment, regardless of the load on each application. In case we have hightly unbalanced applications load, this might not be a suitable method.