Avoiding Huge Docker Image For LLM Model

 

In this post we will review a method to reduce docker image size for an LLM inference image.

In general using LLM in a docker container tend to create images whose size might be over 20G.

Downloaded docker images from ECR or from NEXUS might be very long. In addition, saving images in these technologies might present a challenge. 

To avoid this we use the following:

1. We extract the high disk space usage folders out from the docker image, and save it into S3. Then we create a new slim image without these folders.

2. Upon deployment, we run an init container to download the file from S3, and extract it to an emptyDir volume.

Step 1: Extract folders and create slim image

The following script runs the original huge image build.

Then it extracts the huge folders ExportFolder1, ExportFolder2 into the OutputFile. This file should be loaded in AWS S3.

Next the script creates a new slim image without the folders ExportFolder1, ExportFolder2.

#!/usr/bin/env bash
set -e

cd "$(dirname "$0")"

DockerRegistry=my-registry.my-company
ProjectVersion=latest
BuildNumber=1234

OutputFile="${OutputFile:-llm-image-${BuildNumber}.tar.gz}"
SlimImageName="${DockerRegistry}/llm-image-slim${ProjectVersion}"

ExportFolder1="root/.cache"
ExportFolder2="usr/local/lib/python3.12"

TempFolder="$(mktemp -d)"
ImageFileSystem="${TempFolder}/rootfs"
TempContainer=llm-image-tmp

date
echo "build full image"
./build.sh


date
echo "extracting container to ${ImageFileSystem}"
mkdir -p "${ImageFileSystem}"
docker rm -f ${TempContainer}
docker create --name ${TempContainer} llm-image/dev:latest
docker export ${TempContainer} | tar -x -C "${ImageFileSystem}"
docker rm -f ${TempContainer}

date
echo "compressing folders to ${OutputFile}"
tar -czf "${OutputFile}" -C "${ImageFileSystem}"  "${ExportFolder1}"  "${ExportFolder2}"

date
echo "creating slim image ${SlimImageName}"
rm -rf "${ImageFileSystem}/${ExportFolder1}" "${ImageFileSystem}/${ExportFolder2}"
tar -C "${ImageFileSystem}" -c . | docker import - "${SlimImageName}"
docker push ${SlimImageName}

date
echo "cleaning up"
rm -rf "${TempFolder}"

date
echo "Done"

Step 2: Deployment

To handle the S3 download we create an extractor image.

Dockerfile:

FROM amazon/aws-cli


COPY files /

ENTRYPOINT ["/entrypoint.sh"]

Downloader script: entrypoint.sh

#!/usr/bin/env bash
set -e

localFileName="/models-local.tar.gz"

cd /local-storage

echo "Downloading ${MODELS_S3_PATH}"
aws s3 cp ${MODELS_S3_PATH} ${localFileName}

echo "Extracting ${localFileName}"
tar -xzf ${localFileName}

echo "Cleaning up..."
rm ${localFileName}

In the kubernetes deployment file, we add an init container.

initContainers:
  - name: extract
    image: modelsextract
    env:
      - name: MODELS_S3_PATH
        value: {{ .Values.modelsS3Path | quote }}
    volumeMounts:
      - mountPath: /local-storage
        name: local-storage

An emptyDir volume and mount for the exported folders:

volumes:
  - name: local-storage
    emptyDir:
      sizeLimit: 100Gi

volumeMounts:
  - mountPath: /root/.cache
    subPath: root/.cache
    name: local-storage
  - mountPath: /usr/local/lib/python3.12
    subPath: usr/local/lib/python3.12
    name: local-storage

Also, make sure to use the slim image instead of the huge image in the deployment container.

Final Note

We have reviewed creation of a slim docker image that later downloads the extra data from AWS S3. To save cost and gain speed, the actual download should be done in the same region as the deployment, so make sure to copy it the a relevant regional AWS S3 bucket.

Using DuckDB in GO

In this post we show a simple example of using DuckDB in GO.

DuckDB features:

• Can run in-memory 
• Supports multiple files types, such as : JSON, NDJSON, CSV EXCEL
• Supports multiple storage types, such as: AWS S3, GCP Storage, PostgreSQL

In the following example we store our data in parquet files in AWS S3.

We order the file using a folders structure that would later enable use to fetch only the files we need.

package duckdb

import (
  "database/sql"
  "fmt"
  "testing"
  "time"

  _ "github.com/marcboeker/go-duckdb"
)

func TestValidation(_ *testing.T) {
  db, err := sql.Open("duckdb", "")
  kiterr.RaiseIfError(err)
  defer db.Close()

  _, err = db.Exec(`
  INSTALL httpfs;
  LOAD httpfs;
`)
  kiterr.RaiseIfError(err)

  _, err = db.Exec(`
  SET s3_region='us-east-1';
  SET s3_access_key_id='XXX';
  SET s3_secret_access_key='XXX';
`)
  kiterr.RaiseIfError(err)

  createTable(db)
  createData(db)
  exportData(db)
}

func createTable(
  db *sql.DB,
) {
  _, err := db.Exec(`
  CREATE TABLE IF NOT EXISTS events (
   my_text TEXT,
   my_value INTEGER,
   event_time TIMESTAMP
  );
`)
  kiterr.RaiseIfError(err)
}

func createData(
  db *sql.DB,
) {
  statement, err := db.Prepare(`
   INSERT INTO events (my_text, my_value, event_time)
   VALUES (?, ?, ?)
  `)
  kiterr.RaiseIfError(err)

  startTime := kittime.ParseDay("2000-01-01")
  for day := range 2 {
   for hour := range 24 {
    for dataIndex := range 2 {
     dayTime := startTime.Add(time.Duration(day) * 24 * time.Hour)
     hourTime := dayTime.Add(time.Duration(hour) * time.Hour)
     fmt.Printf("saving data %v for %v\n", dataIndex, kittime.NiceTime(&hourTime))
     _, err = statement.Exec("aaa", 11, &hourTime)
     kiterr.RaiseIfError(err)
    }
   }
  }
}

func exportData(
  db *sql.DB,
) {
  _, err := db.Exec(`
  COPY (
   SELECT
    my_text,
    my_value,
    event_time,
    CAST(event_time AS DATE) AS event_date,
    strftime(event_time , '%H') AS event_hour
   FROM events
  )
  TO 's3://my-bucket/duck/'
  (
   FORMAT PARQUET,
   COMPRESSION GZIP,
   PARTITION_BY (event_date, event_hour)
  );
`)
  kiterr.RaiseIfError(err)
}

The result in AWS S3 is:

$ aws s3 ls --recursive s3://my-bucket
2026-02-04 11:27:40        542 duck/event_date=2000-01-02/event_hour=10/data_0.parquet
2026-02-04 11:27:45        541 duck/event_date=2000-01-02/event_hour=13/data_0.parquet
2026-02-04 11:27:47        541 duck/event_date=2000-01-02/event_hour=14/data_0.parquet
2026-02-04 11:27:49        542 duck/event_date=2000-01-02/event_hour=15/data_0.parquet
2026-02-04 11:27:50        541 duck/event_date=2000-01-02/event_hour=16/data_0.parquet
2026-02-04 11:27:52        542 duck/event_date=2000-01-02/event_hour=17/data_0.parquet
2026-02-04 11:27:54        541 duck/event_date=2000-01-02/event_hour=18/data_0.parquet
2026-02-04 11:27:58        541 duck/event_date=2000-01-02/event_hour=20/data_0.parquet
2026-02-04 11:28:00        542 duck/event_date=2000-01-02/event_hour=21/data_0.parquet
2026-02-04 11:28:03        542 duck/event_date=2000-01-02/event_hour=23/data_0.parquet

So for a quick start project that needs to store data overtime, and read it up need it is nice solution. For a heavier project that requires more data with high control over the storage type, parallelism, and timing, it might require creation of proprietary code.

NATS Performance Tuning

 

In this post we can see how to install NATS cluster with configuration tuned for high throughput.

The following is an example script to install a tuned NATS cluster:

#!/usr/bin/env bash
set -e
cd "$(dirname "$0")"

helm repo add nats https://nats-io.github.io/k8s/helm/charts/
helm repo update

rm -f nats.yaml

cat <<EOF > nats.yaml
# nats-values.yaml

config:
  jetstream:
    enabled: false

  cluster:
    enabled: true
    replicas: 4


  merge:
    # Bigger messages & buffers
    max_payload: 8388608        # 8 * 1024 * 1024 bytes
    write_deadline: "2s"
    max_pending: 536870912       # 512 * 1024 * 1024 bytes
    # Connection & subscription scaling
    max_connections: 100000
    max_subscriptions: 1000000

    # Disable anything not needed
    debug: false
    trace: false
    logtime: false


container:
  merge:
    resources:
      limits:
        cpu: "4"
        memory: "2Gi"
EOF

helm delete nats --ignore-not-found

helm upgrade --install nats \
  nats/nats \
  --namespace default \
  --create-namespace \
  -f nats.yaml

rm -f nats.yaml

We are increasing the NATS cluster memory using the max_pending. This configure NATS to buffer messages allowing producer faster publish, and eventuall higher consumption by the consumer.

The producers and consumers actually connect to a random available NATS pod, so in case of small amount of consumers, producers, and NATS pods, we will probably have an unbalanced load for the NATS pods.

To avoid this, use a connection pool on both consumers and producers, hence spreading the load among the NATS pods.

Using messages size 1K, a single NATS pod can handle ~1M messages/seconds. Make sure checking the NATS pod is not saturated by CPU and memory.

Kind Load vs. Local Repository

 

In this post we reviewed using kind for local development. One of the requirement for such deployment is to load the image into kind using:

kind load docker-image ${DockerTag}

The kind load CLI sends the entire image to the kind, which means it does not use the docker image layers concept. This means that any image update will take much longer time, especially for large images.

To improve this we can use a local docker repository, and configure kind to load the images from it. To deploy a local repository our development laptop we use:

docker run -d \
  --restart=always \
  -p 5000:5000 \
  --name kind-registry \
   -v $HOME/.kind/registry:/var/lib/registry \
  registry:2

Then we re-create the kind cluster with configuration to use the local registry:

cat <<EOF | kind create cluster --config=-
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
containerdConfigPatches:
- |
  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:5000"]
    endpoint = ["http://kind-registry:5000"]
nodes:
- role: control-plane
  kubeadmConfigPatches:
  - |
    kind: InitConfiguration
    nodeRegistration:
      kubeletExtraArgs:
        node-labels: "ingress-ready=true"
  extraPortMappings:
  - containerPort: 80
    hostPort: 80
    protocol: TCP
  - containerPort: 443
    hostPort: 443
    protocol: TCP
EOF

To enable kind to communicate with the docker repository container, we connect the networks:

docker network connect kind kind-registry

And it is ready!

We can now load the build images into the local repository as part of our build script:

LocalRegistryTag=localhost:5000/my-image:latest
docker tag ${DockerTag} ${LocalRegistryTag}
docker push ${LocalRegistryTag}

We also should use an updated helm configuration for the images location, as well as instruct kubernetes to always pull the images from the registry. This is required since we usually do not change the image version upon local build, and simply use `latest`:

image:
  registry: localhost:5000/
  version: /dev:latest
  pullPolicy: Always

The result is a faster build and deployment process on the local laptop.

GO Coding Guidelines

 

In this post I add specify example of GO coding guidelines. 

Many of these guidelines are language agnostic, so this can be used as a base for any other language.

Use full words

Names are important for clear text.

Always use full words.

For camel case make sure to use event for abbreviation:

• myInstance

• ssnFullText

• clientIpAddress

No Comments

Our code should be clear and readable.

This means we should have comments only in case we do something unusual.

Do not comment on the obvious.

Look around for standards

Always look around for standards before making a change.

Someone had invested some thought about this code, and decided on a standard,

so we want to continue using the same standard and not create a new standard to confuse future development.

Each class in its own package

Each class should be its own package.

Do not include more than a single class in a package.

Clear TODOs

Do not leave TODOs in the code

Switch case default

Do not assume the default in a switch case is something that should not be handled.

In case we should not get to the default, raise an error.

Each Parameter in its own line

Each function parameter should be in its own line, even when it is the only parameter

Boolean condition

Boolean variable is a gift for usage in a condition.

Never compare boolean to true or false,

Unused code

All unused code must be removed. This includes: variables, functions, consts, comments, commented code and anything…

One letter self

Always use one letter for the “self”.

Explicit struct

Always create explicit structs.

Each struct should have a name and a definition.

Avoid using struct on the fly.

Bad Example:

Limit indentation

We are human, and we cannot understand complicated code with more than 3 indentations.

Use functions, and build a better clear design.

Empty array

When you need an empty array, always use nil

Max 15 codes line per function

Keep the functions short and clean

In case you have a function with more than 15 code lines,

split it wisely while keeping a clear code and reduce coupling.

Remove unnecessary code

Avoid adding code that is already handled by the GO language, for example:

* loop over nil array is ok, no need to add codition and check if the array is nil

Two words for public items

To bypass the IDE bug, any public struct field and public method must have at least 2 words.

Struct is always a pointer in function

Whenever sending a struct to a function and receiving a struct from a function,

use a pointer to the struct to avoid memory duplication. 

Notice that time.Time is also a struct.

Function for nothing

In case you have a function that does only a simple statement, rethink whether it is really required.

You might still want to keep it to avoid code duplication,

but otherwise remove the function and just call the statement directly..

Reduce Coupling in function parameters

Design the functions to receive as few parameters as possible.

This reduces the coupling between the calling code and the function.

Reduce coupling in object fields

The idea of an object is that we have fields that are shared among all methods.

Hence, do not send the fields as parameters.

Keep you private stuff private

Use public only if you must expose.

Upper case function & variable & structure should be avoided otherwise.

Don’t return errors

When you have errors, raise them, do not return an error in the function return code.

The only case when you need to handle errors is when you have a new top level code, such as a new go routine.

No hard coded configuration

Avoid using hard coded configuration.

Use environment variables, or better - use some kind of mechanism that can be updated on the fly.

JSON tags

Usage of JSON tags is required only when communicating with an external entity, or when saving to redis.

Don’t add JSON tags just because you can.

Use OOD

In a component that receives some common input parameters that are used across its functions,

avoid sending the parameters over and over again.

Instead, create an object, set the input in the object’s fields, and avoid resending the parameters.

Constants usage

Constants should be used if and only if a value is used more than once

Tests

Any code should have a project test.

Make sure to check the code also manually in the local KIND cluster.

Memory usage 

When using a persistent in-memory state, we must make sure the state memory is limited.

We must make sure the pod will not crash due to memory usage.

Passwords to frontend

Never return passwords from the backend to the GUI, this is a security issue

Logging to the relevant logger

Always use a common logger library for logging.

Make sure to include the ability to dynamically update the log level for a running pod.

Avoid complex types

We are simple humans, and we understand simple types.

If you have a complex type, create structures to simplify it.

Array of structs

Array of struct should always use point to the struct

Do not confuse this with a pointer to array.

Provide error details

When we have an error, we can find the exact location using the error stack trace.

However, we cannot know the values that cause the error.

Provide more details in such cases.

Don’t hide errors

When we have errors we should not hide them.

The default should be to raise the error.

If this is something we expect to have errors and we are 100% sure we cannot overcome this by retries or similar, 

at least log with WARN.

Don’t bypass bugs

If we have bugs, we want to raise errors, and not to avoid them.

Do not create code that ignores bugs.

Avoid code duplication

Duplicate code makes the understanding of the code harder,

and leaves room for bugs in case someone updates only one copy of the code.

Safe Locking

Always prefer clean locking that guarantees cleanup of locking in case of error.

So we should have a function starting with the following:

a.mutex.Lock()
defer a.mutex.Lock()

Create Module Federation Using NX

 

In this post we will review the steps to create a module federation using NX

What is NX?

NX is a monorepo tool for managing multiple projects in a single repository. In this scenario it is used both as a wizard to create the projects, and as the project runner.

What is Module Federation?

We can see a module federation as a "live" remote import of a library. The module federation includes:

Host: the main web server the browser is connecting to.

Remote: the module web server that the host is importing a library from.

The general idea is to have a microservice approach to the front-end architecture, allowing independent update of each microservice.

Creation Steps

Well it is working, but you will get many warnings.

Don't panic.

Step 1 - Create a new Nx workspace

npx create-nx-workspace@latest my-nx-workspace

• Select Minimal starter 
• Try the full Nx platform? … NO! This will use NX Cloud

 

cd my-nx-workspace
npm install --save-dev @nx/react
nx g @nx/react:app shop --bundler=webpack --style=css

• Would you like to add routing to this application? No
• Which linter would you like to use? … eslint
• E2E test runner? None
• Which port? 4200

nx serve shop

Open browser on http://localhost:4200, and verify it is working.

Press Ctrl+C on the NX command to stop and continue to the next step.

Step 2 - Convert shop into a Module Federation host 

nx g @nx/react:host shop --bundler=webpack

• Which stylesheet format would you like to use? … CSS
• Which test runner? None
• Which bundler? webpack

Recheck everything is still running

nx serve shop

Open browser on http://localhost:4200, and verify it is working.

Press Ctrl+C on the NX command to stop and continue to the next step.

Step 3 - Create Remote App

nx g @nx/react:remote cart --host=shop --style=css --bundler=webpack

In the first terminal window run the remote:

nx serve cart

In the second terminal window run the host:

nx serve shop

Open browser on http://localhost:4200, and verify you can see both the host (Home) and the remote (Cart).

Final Note

We have reviewed the steps to create a module federation.

This is just a simple tutorial, but in real life it is much complex. You need to handle cases where the remote is down, proxy AJAX requests from the remote, handle CORS, mismatch dependencies and version and more. 

Bottom line, avoid this architecture whenever possible, and use it only if you have to for a very complex infrastructure that has high maintenance costs.

Collecting Kubernetes Logs using LOKI

 

In this post we review the step to deploy a kubernetes logs collection system using LOKI, Promtail, and grafana.

• Grafana is a front-end GUI enabling view of the logs.
  I've already covered grafana deployment on kubernetes in this post.
  
  
• LOKI is a logs storage and log query component that works like a charm in kubernetes environment
  
  
• Promtail is responsible for sending the pods logs to LOKI.

Let's review the deployment steps.

Part of this is based on the LOKI deployment guide.

LOKI AWS entities

Create LOKI logs storage buckets:

aws s3api create-bucket --bucket  agentic-loki-chunks --region us-east-1  
aws s3api create-bucket --bucket  agentic-loki-ruler  --region us-east-1 

Create a permission policy for LOKI

rm -f loki-s3-policy.json

cat <<EOF > loki-s3-policy.json
{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "LokiStorage",
            "Effect": "Allow",
            "Action": [
                "s3:ListBucket",
                "s3:PutObject",
                "s3:GetObject",
                "s3:DeleteObject"
            ],
            "Resource": [
                "arn:aws:s3:::agentic-loki-chunks",
                "arn:aws:s3:::agentic-loki-chunks/*",
                "arn:aws:s3:::agentic-loki-ruler",
                "arn:aws:s3:::agentic-loki-ruler/*"
            ]
        }
    ]
}
EOF


aws iam create-policy --policy-name LokiS3AccessPolicy --policy-document file://loki-s3-policy.json
rm loki-s3-policy.json

Create LOKI trust policy to enable it to use the role in the kubernetes cluster

rm -f trust-policy.json

cat << EOF > trust-policy.json
{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Principal": {
                "Federated": "arn:aws:iam::662909476770:oidc-provider/oidc.eks.us-east-1.amazonaws.com/id/873FB195FF4FAEC482E18822F7D4CBF9"
            },
            "Action": "sts:AssumeRoleWithWebIdentity",
            "Condition": {
                "StringEquals": {
                    "oidc.eks.us-east-1.amazonaws.com/id/873FB195FF4FAEC482E18822F7D4CBF9:sub": "system:serviceaccount:loki:loki",
                    "oidc.eks.us-east-1.amazonaws.com/id/873FB195FF4FAEC482E18822F7D4CBF9:aud": "sts.amazonaws.com"
                }
            }
        }
    ]
}
EOF

Create role and attach the policies

aws iam create-role --role-name LokiServiceAccountRole --assume-role-policy-document file://trust-policy.json
aws iam attach-role-policy --role-name LokiServiceAccountRole --policy-arn arn:aws:iam::662909476770:policy/LokiS3AccessPolicy

LOKI Deployment

We use helm to deploy LOKI on the kubernetes cluster.

kubectl create namespace loki

helm repo add grafana https://grafana.github.io/helm-charts
helm repo update

rm -f values.yaml

cat << EOF > values.yaml

loki:
   schemaConfig:
     configs:
       - from: "2024-04-01"
         store: tsdb
         object_store: s3
         schema: v13
         index:
           prefix: loki_index_
           period: 24h
   storage_config:
     aws:
       region: us-east-1  
       bucketnames: agentic-loki-chunks
       s3forcepathstyle: false
   ingester:
       chunk_encoding: snappy
   pattern_ingester:
       enabled: true
   limits_config:
     allow_structured_metadata: true
     volume_enabled: true
     retention_period: 672h # 28 days retention
   compactor:
     retention_enabled: true 
     delete_request_store: s3
   ruler:
    enable_api: true
    storage:
      type: s3
      s3:
        region: us-east-1
        bucketnames: agentic-loki-ruler
        s3forcepathstyle: false
      alertmanager_url: http://prom:9093 # The URL of the Alertmanager to send alerts (Prometheus, Mimir, etc.)

   querier:
      max_concurrent: 4

   storage:
      type: s3
      bucketNames:
        chunks: "agentic-loki-chunks" 
        ruler: "agentic-loki-ruler" 
      s3:
        region: us-east-1

serviceAccount:
 create: true
 annotations:
   "eks.amazonaws.com/role-arn": "arn:aws:iam::662909476770:role/LokiServiceAccountRole" 

deploymentMode: Distributed

ingester:
 replicas: 2
 zoneAwareReplication:
  enabled: false

querier:
 replicas: 2
 maxUnavailable: 1

queryFrontend:
 replicas: 2
 maxUnavailable: 1

queryScheduler:
 replicas: 2

distributor:
 replicas: 2
 maxUnavailable: 1
compactor:
 replicas: 1

indexGateway:
 replicas: 2
 maxUnavailable: 1

ruler:
 replicas: 1
 maxUnavailable: 1


gateway:
 service:
   type: ClusterIP
 basicAuth: 
     enabled: false

lokiCanary:
  extraArgs: []
  extraEnv: []

minio:
 enabled: false

backend:
 replicas: 0
read:
 replicas: 0
write:
 replicas: 0

singleBinary:
 replicas: 0

EOF

helm upgrade -i --values values.yaml loki grafana/loki -n loki 
rm values.yaml

Promtail

We use helm to deploy Promtail

rm -f values.yaml

cat << EOF > values.yaml

extraVolumes:
  - name: positions
    emptyDir: {}

extraVolumeMounts:
  - name: positions
    mountPath: /promtail/positions

config:
  clients:
    - url: http://loki-distributor.loki.svc.cluster.local:3100/loki/api/v1/push
      tenant_id: test

  positions:
    filename: /promtail/positions/positions.yaml

  scrape_configs:
    - job_name: kubernetes-pods
      kubernetes_sd_configs:
        - role: pod
      relabel_configs:
        - source_labels: [__meta_kubernetes_namespace]
          action: replace
          target_label: namespace
        - source_labels: [__meta_kubernetes_pod_name]
          action: replace
          target_label: pod


EOF

helm upgrade --install promtail grafana/promtail \
  --namespace promtail --create-namespace \
  -f values.yaml

rm values.yaml

Grafana

To view logs in grafana,  Add LOKI as datasource under Connections, Data sources:

http://loki-gateway.loki.svc.cluster.local

and add a customer header:

X-Scope-OrgID = test

Then add a visualization:

• New dashboard
• Settings, variables, add
• Show on dashboard: "Label and Value"
• Query type: "Label Values"
• Label: "container"
• Save
• Add visualization
• select on the top right "Logs"
• select LOKI as data source
• add filter container=${container}

Final Note

In this post we have reviewed the steps to collect kubernetes pods logs using Promtail, LOKI, and grafana. We have used the LOKI Simple Scalable Mode, but for a large kubernetes the LOKI Microservice mode is preferred.