React-Vis Graph with Million Points and Zoom

 

In this post we will review how to display a react-vis graph with million points, including zoom support.

React-Vis is a great library that can display many times of charts. In this post we will use a simple line chart with react and redux. We also include a zoom support. 

Another important issue in this post is handling millions of points. If we try to display millions of points, the react-vis library starts slowing down, hence we add special handling in transforming the original huge amount of points to up to 1000 visible points. We use a transformation to join several points into one average point to get to this requirement.

First, let review the reduce slice.

slice.js

import {createSlice} from '@reduxjs/toolkit'


const maxPoints = 1000

const initialState = {
  selections: {},
  points: {},
  series: {},
}


function getSelectedPoints(points,selection) {
  if (!selection) {
    return points
  }
  const selectedPoints = []

  points.forEach(point => {
    if (point.x >= selection.left && point.x <= selection.right) {
      selectedPoints.push(point)
    }
  })
  return selectedPoints
}

function reducePoints(points, ratio) {
  if (ratio <= 1) {
    return points
  }

  const reduced = []
  let slice = []
  points.forEach(point => {
    slice.push(point)
    if (slice.length === ratio) {
      let x = 0
      let y = 0
      slice.forEach(slicePoint => {
        x += slicePoint.x
        y += slicePoint.y
      })


      const averagePoint = {
        x: x / slice.length,
        y: y / slice.length,
      }

      reduced.push(averagePoint)

      slice = []
    }
  })
  return reduced
}

function convertPoints(state,graphId) {
  const selection = state.selections[graphId]
  const points = state.points[graphId]
  const xWidth = points[points.length - 1].x - points[0].x
  const selectedPoints = getSelectedPoints(points,selection)

  let allowedPoints
  if (selection) {
    const selectedWidth = Math.trunc(selection.right - selection.left)
    allowedPoints = Math.trunc(maxPoints * xWidth / selectedWidth)
  } else {
    allowedPoints = maxPoints
  }

  const ratio = Math.trunc(points.length / allowedPoints)
  return reducePoints(selectedPoints, ratio)
}

function updateSeries(state, graphId) {
  state.series[graphId] = [
    {
      title: 'Apples',
      disabled: false,
      data: convertPoints(state,graphId),
    },
  ]
}


export const slice = createSlice({
  name: 'graph',
  initialState,
  reducers: {
    setPoints: (state, action) => {
      const {graphId, points} = action.payload
      state.points[graphId] = points
      updateSeries(state,graphId)
    },
    setSelection: (state, action) => {
      const {graphId, selection} = action.payload
      state.selections[graphId] = selection
      updateSeries(state,graphId)
    },
    clearSelection: (state, action) => {
      const {graphId} = action.payload
      delete state.selections[graphId]
      updateSeries(state,graphId)
    },
  },
})

export const {setSelection, clearSelection, setPoints} = slice.actions


export function selectState(state) {
  return state.graph
}

export default slice.reducer

 We supply 3 actions: set points, set selection (for zoom), and clear selection (for zoom removal).

The convertPoints function, first selects only the relevant points according to the zoom selection area, and the reduce the points according to the amount of points. For example, if we have 1,000,000 points, and we zoom on scale 400,000-500,000, then we're left with 100,000 points. Then we convert each 100 points to a single average point so we only use 1000 points for the actual display.

The following class controls the way the axis labels are displayed:

custom-axis-label.js

import React, { PureComponent } from 'react';
import './index.css'

class CustomAxisLabel extends PureComponent {

  render() {

    const yLabelOffset = {
      y: this.props.marginTop + this.props.innerHeight / 2 + this.props.title.length*2,
      x: 10
    };

    const xLabelOffset = {
      x: this.props.marginLeft + (this.props.innerWidth)/2 - this.props.title.length*2,
      y: 1.2 * this.props.innerHeight
    };

    const transform = this.props.xAxis
      ? `translate(${xLabelOffset.x}, ${xLabelOffset.y})`
      : `translate(${yLabelOffset.x}, ${yLabelOffset.y}) rotate(-90)`;

    return (
      <g transform={transform}>
        <text className= 'unselectable axis-labels'>
          {this.props.title}
        </text>
      </g>
    );
  }
}

CustomAxisLabel.displayName = 'CustomAxisLabel';
CustomAxisLabel.requiresSVG = true;
export default CustomAxisLabel;

We set the labels orientation, and the margins for the labels.

The highlight class handles the zoom in the graph by sending and event with the selection zoom rectangle.

highlight.js

import React from "react";
import { ScaleUtils, AbstractSeries } from "react-vis";

export default class Highlight extends AbstractSeries {
  static displayName = "HighlightOverlay";
  static defaultProps = {
    allow: "x",
    color: "rgb(77, 182, 172)",
    opacity: 0.3
  };

  state = {
    drawing: false,
    drawArea: { top: 0, right: 0, bottom: 0, left: 0 },
    x_start: 0,
    y_start: 0,
    x_mode: false,
    y_mode: false,
    xy_mode: false
  };

  constructor(props){
    super(props);
    document.addEventListener("mouseup", function (e) {
      this.stopDrawing()
    }.bind(this));
  }

  _cropDimension(loc, startLoc, minLoc, maxLoc) {
    if (loc < startLoc) {
      return {
        start: Math.max(loc, minLoc),
        stop: startLoc
      };
    }

    return {
      stop: Math.min(loc, maxLoc),
      start: startLoc
    };
  }

  _getDrawArea(loc) {
    const { innerWidth, innerHeight } = this.props;
    const { x_mode, y_mode, xy_mode } = this.state;
    const { drawArea, x_start, y_start } = this.state;
    const { x, y } = loc;
    let out = drawArea;

    if (x_mode | xy_mode) {
      // X mode or XY mode
      const { start, stop } = this._cropDimension(x, x_start, 0, innerWidth);
      out = {
        ...out,
        left: start,
        right: stop
      }
    }
    if (y_mode | xy_mode) {
      // Y mode or XY mode
      const { start, stop } = this._cropDimension(y, y_start, 0, innerHeight);
      out = {
        ...out,
        top: innerHeight - start,
        bottom: innerHeight - stop
      }
    }
    return out
  }

  onParentMouseDown(e) {
    const { innerHeight, innerWidth, onBrushStart } = this.props;
    const { x, y } = this._getMousePosition(e);
    const y_rect = innerHeight - y;

    // Define zoom mode
    if (x < 0 & y >= 0) {
      // Y mode
      this.setState({
        y_mode: true,
        drawing: true,
        drawArea: {
          top: y_rect,
          right: innerWidth,
          bottom: y_rect,
          left: 0
        },
        y_start: y
      });

    } else if (x >= 0 & y < 0) {
      // X mode
      this.setState({
        x_mode: true,
        drawing: true,
        drawArea: {
          top: innerHeight,
          right: x,
          bottom: 0,
          left: x
        },
        x_start: x
      });

    } else if (x >= 0 & y >= 0) {
      // XY mode
      this.setState({
        xy_mode: true,
        drawing: true,
        drawArea: {
          top: y_rect,
          right: x,
          bottom: y_rect,
          left: x
        },
        x_start: x,
        y_start: y
      });
    }

    // onBrushStart callback
    if (onBrushStart) {
      onBrushStart(e);
    }

  }

  stopDrawing() {
    // Reset zoom state
    this.setState({
      x_mode: false,
      y_mode: false,
      xy_mode: false
    });

    // Quickly short-circuit if the user isn't drawing in our component
    if (!this.state.drawing) {
      return;
    }

    const { onBrushEnd } = this.props;
    const { drawArea } = this.state;
    const xScale = ScaleUtils.getAttributeScale(this.props, "x");
    const yScale = ScaleUtils.getAttributeScale(this.props, "y");

    // Clear the draw area
    this.setState({
      drawing: false,
      drawArea: { top: 0, right: 0, bottom: 0, left: 0 },
      x_start: 0,
      y_start: 0
    });

    // Invoke the callback with null if the selected area was < 5px
    if (Math.abs(drawArea.right - drawArea.left) < 5) {
      onBrushEnd(null);
      return;
    }

    // Compute the corresponding domain drawn
    const domainArea = {
      bottom: yScale.invert(drawArea.top),
      right: xScale.invert(drawArea.right),
      top: yScale.invert(drawArea.bottom),
      left: xScale.invert(drawArea.left)
    };

    if (onBrushEnd) {
      onBrushEnd(domainArea);
    }
  }

  _getMousePosition(e) {
    // Get graph size
    const { marginLeft, marginTop, innerHeight } = this.props;

    // Compute position in pixels relative to axis
    const loc_x = e.nativeEvent.offsetX - marginLeft;
    const loc_y = innerHeight + marginTop - e.nativeEvent.offsetY;

    // Return (x, y) coordinates
    return {
      x: loc_x,
      y: loc_y
    }

  }

  onParentMouseMove(e) {
    const { drawing } = this.state;

    if (drawing) {
      const pos = this._getMousePosition(e);
      const newDrawArea = this._getDrawArea(pos);
      this.setState({ drawArea: newDrawArea });
    }

  }

  render() {
    const {
      marginLeft,
      marginTop,
      innerWidth,
      innerHeight,
      color,
      opacity
    } = this.props;
    const { drawArea: { left, right, top, bottom } } = this.state;
    return (
      <g
        transform={`translate(${marginLeft}, ${marginTop})`}
        className="highlight-container">
        <rect
          className="mouse-target"
          fill="black"
          opacity="0"
          x={0}
          y={0}
          width={innerWidth}
          height={innerHeight}
        />
        <rect
          className="highlight"
          pointerEvents="none"
          opacity={opacity}
          fill={color}
          x={left}
          y={bottom}
          width={right - left}
          height={top - bottom}
        />
      </g>
    );
  }
}

Last one is the graph class with uses all of the above.

component.js

import React from 'react'
import '../../node_modules/react-vis/dist/style.css'

import {
  Borders,
  DiscreteColorLegend,
  HorizontalGridLines,
  LineSeries,
  VerticalGridLines,
  XAxis,
  XYPlot,
  YAxis,
} from 'react-vis'
import Highlight from './highlight'
import {useDispatch, useSelector} from 'react-redux'
import {selectState, setSelection} from './slice'

function Graph(props) {
  const {graphId} = props
  const dispatch = useDispatch()
  const state = useSelector(selectState)
  const selection = state.selections[graphId]
  const series = state.series[graphId]

  if (!series) {
    return null
  }

  const width = 1000

  function highlightArea(area) {
    dispatch(setSelection({
      graphId,
      selection: area,
    }))
  }

  return (
    <div>
      <div className="legend">
        <DiscreteColorLegend
          width={180}
          items={series}/>
      </div>

      <div className="chart no-select" onDragStart={function (e) {
        e.preventDefault()
      }}>
        <XYPlot
          xDomain={selection && [selection.left, selection.right]}
          yDomain={selection && [selection.bottom, selection.top]}
          height={500}
          width={width}
          margin={{left: 45, right: 20, top: 10, bottom: 200}}>

          <HorizontalGridLines/>
          <VerticalGridLines/>

          {series.map(entry => (
            <LineSeries
              key={entry.title}
              data={entry.data}
            />
          ))}

          <Highlight
            onBrushEnd={highlightArea}
          />
          <Borders style={{all: {fill: '#fff'}}}/>
          <XAxis tickFormat={(v) => new Date(v * 3600 * 1000).toISOString()} tickLabelAngle={-60}/>
          <YAxis tickFormat={(v) => (<tspan className="unselectable"> {v} </tspan>)}/>
        </XYPlot>
      </div>
    </div>
  )
}

export default Graph

Notice that the graph supports multiple instance by using the graphId property. In this case we treat the x-axis as hours so we multiply it we 3600 seconds. The graph display a list of points, each including the x and y properties.

Final Note

While the react-vis response time is good, redux slows the GUI down. To prevent this, configure redux to skip the data of the reducer containing the huge amount of data. For example, to ignore the graph reducer data, use:

import {configureStore} from '@reduxjs/toolkit'

import notification from './notification/slice'
import dashboard from './dashboard/slice'
import graph from './graph/slice'
import histogram from './histogram/slice'

export const store = configureStore({
  middleware: (getDefaultMiddleware) => getDefaultMiddleware({
    immutableCheck: {ignoredPaths: ['graph']},
    serializableCheck: {ignoredPaths: ['graph']},
  }),
  reducer: {
    dashboard,
    histogram,
    graph,
    notification,
  },
})

Custom Marshaling in GoLang

 

In this post we will review how to handle custom marshaling in GoLang.

I've recently had to marshal a structure containing a map with float as key. Then I got this error:

json: unsupported type: map[float64]int

Reading the documents I've found that since JSON does not support float as keys, the GoLang json marshaling does not automatically convert the float to string, and instead is returning an error. The solution in this case is to implement a custom marshaling. Let's examine the structures.

type MainStruct struct {
 Name      string
 InnerData InnerStruct
}

type InnerStruct struct {
 Exists bool
 Count  int
 Values map[float64]int
}

We have a main struct, and an inner struct. I've included two structures to emphasis that the marshaling is done on the main struct, but still we will add our custom marshaling on the inner struct, and it will be used even that the marshaling is not done directly on it. Let's examine the marshal example:

func TestJson(t *testing.T) {
 data := MainStruct{
  Name: "john",
  InnerData: InnerStruct{
   Exists: true,
   Count:  72,
   Values: map[float64]int{
    1.2: 42,
    3.4: 56,
   },
  },
 }

 jsonBytes, err := json.Marshal(data)
 if err != nil {
  t.Fatal(err)
 }

 t.Log(string(jsonBytes))

 var loadedData MainStruct
 err = json.Unmarshal(jsonBytes, &loadedData)
 if err != nil {
  t.Fatal(err)
 }

 t.Log(loadedData)
}

If we will run the test now, we will get the unsupported type error displayed above. To solve the issue we add 2 methods to handle the custom marshaling and unmarshaling.

func (i InnerStruct) MarshalJSON() ([]byte, error) {
 newStruct := struct {
  Exists bool
  Count  int
  Values map[string]int
 }{
  Exists: i.Exists,
  Count:  i.Count,
 }

 if i.Values != nil {
  newStruct.Values = make(map[string]int)
  for key, value := range i.Values {
   keyString := fmt.Sprintf("%v", key)
   newStruct.Values[keyString] = value
  }
 }
 return json.Marshal(&newStruct)
}

func (i *InnerStruct) UnmarshalJSON(data []byte) error {
 newStruct := struct {
  Exists bool
  Count  int
  Values map[string]int
 }{}

 err := json.Unmarshal(data, &newStruct)
 if err != nil {
  return fmt.Errorf("custom unmarshal failed: %v", err)
 }

 i.Exists = newStruct.Exists
 i.Count = newStruct.Count

 if newStruct.Values != nil {
  i.Values = make(map[float64]int)
  for key, value := range newStruct.Values {
   valueFloat, err := strconv.ParseFloat(key, 64)
   if err != nil {
    return fmt.Errorf("parse float failed: %v", err)
   }
   i.Values[valueFloat] = value
  }
 }
 return nil
}

The methods convert the map of floats keys to map of strings keys, and hence bypass the GoLang non-supporting the float as key. Both of the methods are using a temporary structure to covert the float to string and vise versa.

Final Note

In this post we have reviewed how to overcome the unsupported type error for GoLang marshaling. Note that other languages, such as javascript automatically handle this conversion.

 

Javascript Deobfuscation Tips

 

Last week I've had to de-obfuscate a javascript file. The javascript file included about 10K lines of code, and I had to struggle to understand the hidden meaning of the code. After a week of struggle I was success, and I want to share some of the tricks and insights that I've found as part of this process.

First, copy the file to an online deobfuscator site such as de4js or obfuscateIO. The site will handle the first pass of the code, such as removing proxy functions, expression simplifications and more.

The code you get after this step is still a big mess. Don't expect anything that you can work with.

Next copy the code into a javascript editor, such as WebStorm, and format the code, so it would match the editor formatting standard.

You will probably want to simplify some of the common expressions, which are usually based on the minifier actions.

Examples of these are listed below.

• Change from:  !0 to: true
• Change from:  !1 to: false
• Change from: void 0 to: undefined

Some obfuscation might be done manually by the code creator, for example, change of all string constants to a base64 encoded strings to hide the real consts, e.g. instead of:

var a = object["left"]

You might find:

const e = '\x95çí'
var a = object[btoa(e)]

In this case, a good approach might be to create a script to automatically translate all of the strings in the obfuscated code.

The last step is the real challenge. In this step we rename the functions to their actual meaningful name. When renaming a function, use the IDE, so it will rename all of the usages as well. 

The best tip for this step is to work bottom up. Look for functions that use items that cannot be renamed, such as document, window, navigator. These functions can be easily deciphered. Then, once the basic functions are handled, you can move up the hierarchy, and understand the next level. 

Make sure to rename not only the functions, but also the parameters names, and the the arguments names in the calling function. This will solve the puzzle piece by piece.

Good luck!

 

Symmetric Encrypt and Decrypt in Golang

 

In this post we will review AES-CBC symmetric encryption in Golang.

We start by key generation.

import (
 "crypto/aes"
 "crypto/cipher"
 "crypto/rand"
 "encoding/base64"
 "encoding/json"
 "fmt"
 "radware.com/euda/commons/global/encryption/padding"
)

const keySizeBytes = 32

type EncryptedData struct {
 Cipher string `json:"cipher"`
 Iv     string `json:"iv"`
}

type Key struct {
 key []byte
}

func GenerateKey() (*Key, error) {
 key := make([]byte, keySizeBytes)
 _, err := rand.Read(key)
 if err != nil {
  return nil, fmt.Errorf("read failed: %v", err)
 }
 k := Key{key: key}
 return &k, nil
}

The key can be easily exported and imported.

func LoadKey(keyBase64 string) (*Key, error) {
 bytes, err := base64.StdEncoding.DecodeString(keyBase64)
 if err != nil {
  return nil, fmt.Errorf("base64 decode failed: %v", err)
 }
 k := Key{key: bytes}
 return &k, nil
}

func (k *Key) ExportKey() string {
 return base64.StdEncoding.EncodeToString(k.key)
}

And finally we can encrypt and decrypt using the key.

func (k *Key) Encrypt(clearText string) (string, error) {
 iv := make([]byte, aes.BlockSize)
 _, err := rand.Read(iv)
 if err != nil {
  return "", fmt.Errorf("read failed: %v", err)
 }

 aesCipher, err := aes.NewCipher(k.key)
 if err != nil {
  return "", fmt.Errorf("create cipher failed: %v", err)
 }

 cbc := cipher.NewCBCEncrypter(aesCipher, iv)

 clearBytes := []byte(clearText)
 clearBytesPadded, err := padding.Pkcs7Pad(clearBytes, aes.BlockSize)
 if err != nil {
  return "", fmt.Errorf("padding failed: %v", err)
 }

 cipherBytes := make([]byte, len(clearBytesPadded))

 cbc.CryptBlocks(cipherBytes, clearBytesPadded)
 data := EncryptedData{
  Cipher: base64.StdEncoding.EncodeToString(cipherBytes),
  Iv:     base64.StdEncoding.EncodeToString(iv),
 }

 jsonBytes, err := json.Marshal(data)
 if err != nil {
  return "", fmt.Errorf("marshal failed: %v", err)
 }

 return string(jsonBytes), nil
}

func (k *Key) Decrypt(jsonData string) (string, error) {
 var data EncryptedData
 err := json.Unmarshal([]byte(jsonData), &data)
 if err != nil {
  return "", fmt.Errorf("unmarshal failed: %v", err)
 }

 iv, err := base64.StdEncoding.DecodeString(data.Iv)
 if err != nil {
  return "", fmt.Errorf("decode iv failed: %v", err)
 }

 cipherBytes, err := base64.StdEncoding.DecodeString(data.Cipher)
 if err != nil {
  return "", fmt.Errorf("decode cipher failed: %v", err)
 }

 aesCipher, err := aes.NewCipher(k.key)
 if err != nil {
  return "", fmt.Errorf("create cipher failed: %v", err)
 }
 cbc := cipher.NewCBCDecrypter(aesCipher, iv)

 paddedClearTextBytes := make([]byte, len(cipherBytes))
 cbc.CryptBlocks(paddedClearTextBytes, cipherBytes)

 clearTextBytes, err := padding.Pkcs7Unpad(paddedClearTextBytes, aes.BlockSize)
 if err != nil {
  return "", fmt.Errorf("un-padding failed: %v", err)
 }

 clearText := string(clearTextBytes)
 return clearText, nil
}

Notice that the encryption result includes Initialization Vector (IV), which is a random buffer that is used in the encryption. The IV is returned from the encrypt function unchanged. The decrypt function uses both the cipher result and the IV along with the AES key to do its work. To simplify the usage of both IV and cipher, we use a JSON format.

We can test our code using the following:

func Test(t *testing.T) {
 key, err := GenerateKey()
 if err != nil {
  t.Fatal(err)
 }

 encrypted, err := key.Encrypt("123456789012345")
 if err != nil {
  t.Fatal(err)
 }

 t.Logf("encrypted is %v", encrypted)

 exportedKey := key.ExportKey()
 t.Logf("exported key is %v", exportedKey)

 importedKey, err := LoadKey(exportedKey)
 if err != nil {
  t.Fatal(err)
 }

 decrypted, err := importedKey.Decrypt(encrypted)
 if err != nil {
  t.Fatal(err)
 }
 t.Logf("decrypted is %v", decrypted)
}

Final Note

This post is part of a series of posts about encryption. The full posts list is below.

• Using ECDSA in GO to Sign and Verify messages
• A-Symmetic Encrypt and Decrypt on Golang
• Symmetric Encrypt and Decrypt in Golang
• Using ECDSA in JavaScript to Sign and Verify messages
• A-Symmetic Encrypt and Decrypt on JavaScript
• Symmetric Encrypt and Decrypt in Javascript
• Using ECDSA in Python to Sign and Verify messages

Symmetric Encrypt and Decrypt in Javascript

 

In this post we will review the steps to use AES-CBC encryption in javascript. We use the SubtleCrypto API, which is available on on secure context pages. Will also use some functions that are not supported on old browsers.

First, to check if the page is considered a secure context, use the following:

console.log('secure context', window.isSecureContext)

The SubtleCrypto API uses ArrayBuffer for it functions, but as we want to send the data to server, will will convert it from and to a base64 string using the following helper functions:

function arrayBufferToBase64(arrayBuffer) {
  const bytes = String.fromCharCode.apply(null, new Uint8Array(arrayBuffer))
  return window.btoa(bytes)
}

function base64ToArrayBuffer(base64String) {
  const chars = window.atob(base64String)
  const arrayBuffer = new ArrayBuffer(chars.length)
  const bufferView = new Uint8Array(arrayBuffer)
  for (let i = 0, strLen = chars.length; i < strLen; i++) {
    bufferView[i] = chars.charCodeAt(i)
  }
  return arrayBuffer
}

Special notes for NodeJS users

As btoa() does not exist in NodeJS, we need to use the Buffer, BUT we must specify the latin1 encoding to have the same encoding as in the browser.

NodeJS version:

function arrayBufferToBase64(arrayBuffer) {
  const bytes = String.fromCharCode.apply(null, new Uint8Array(arrayBuffer))
  return Buffer.from(bytes,'latin1').toString('base64')
}

Now we can generate the AES key.

const ALGORITHM = 'AES-CBC'


async function generateKey() {
  return await window.crypto.subtle.generateKey(
    {
      name: ALGORITHM,
      length: 256,
    },
    true,
    ['encrypt', 'decrypt'],
  )
}

The key can be exported and imported.

async function exportKey(key) {
  const exported = await window.crypto.subtle.exportKey(
    'raw',
    key,
  )

  return arrayBufferToBase64(exported)
}

async function importKey(base64Key) {
  const bytes = base64ToArrayBuffer(base64Key)
  return await window.crypto.subtle.importKey(
    'raw',
    bytes,
    'AES-CBC',
    true,
    ['encrypt', 'decrypt'],
  )
}

And finally we can encrypt and decrypt using AES-CBC:

async function encrypt(key, clearText) {
  const encodedText = new TextEncoder().encode(clearText)
  const iv = window.crypto.getRandomValues(new Uint8Array(16))
  const cipherText = await window.crypto.subtle.encrypt(
    {
      name: ALGORITHM,
      iv,
    },
    key,
    encodedText,
  )

  const data = {
    cipher: arrayBufferToBase64(cipherText),
    iv: arrayBufferToBase64(iv),
  }

  return JSON.stringify(data)
}

async function decrypt(key, jsonData) {
  const data = JSON.parse(jsonData)
  const ciphertext = base64ToArrayBuffer(data.cipher)
  const iv = base64ToArrayBuffer(data.iv)
  const encodedText = await window.crypto.subtle.decrypt(
    {
      name: 'AES-CBC',
      iv,
    },
    key,
    ciphertext,
  )

  return new TextDecoder().decode(encodedText)
}

Notice that the encryption result includes Initialization Vector (IV), which is a random buffer that is used in the encryption. The IV is returned from the encrypt function unchanged. The decrypt function uses both the cipher result and the IV along with the AES key to do its work. To simplify the usage of both IV and cipher, we use a JSON format.

We can now test our code. The following is an example of using the encryption.

async function unitTest() {
  const logPrefix = 'symmetric unit-test'
  const key = await generateKey()
  console.log(logPrefix, 'key', key)
  const clearText = '123456789sdfghjkl%$^&*(XXX'
  console.log(logPrefix, 'clearText', clearText)
  const cipher = await encrypt(key, clearText)
  console.log(logPrefix, 'cipher', cipher)
  const exportedKey = await exportKey(key)
  console.log(logPrefix, 'exportedKey', exportedKey)
  const importedKey = await importKey(exportedKey)
  console.log(logPrefix, 'importedKey', importedKey)
  const decrypted = await decrypt(importedKey, cipher)
  console.log(logPrefix, 'decrypted', decrypted)
}

Final Note

This post is part of a series of posts about encryption. The full posts list is below.

• Using ECDSA in GO to Sign and Verify messages
• A-Symmetic Encrypt and Decrypt on Golang
• Symmetric Encrypt and Decrypt in Golang
• Using ECDSA in JavaScript to Sign and Verify messages
• A-Symmetic Encrypt and Decrypt on JavaScript
• Symmetric Encrypt and Decrypt in Javascript
• Using ECDSA in Python to Sign and Verify messages

XMLHttpRequest Proxy

 

In a previous post about XMLHttpRequest we've describe how to capture requests so that we can examine the URLs, and possibly add our own headers. In this post, we will display a more intrusive method, where we create a proxy for the XMLHttpRequest, so that we can replace the request, and the response, as well as sending the data to a different URL.

The proxy structure is as follows:

const originalXhrClass = XMLHttpRequest

XMLHttpRequest = function () {
  // the proxy code goes here
}

We replace the XMLHttpRequest with our own code, and keep an internal reference to the original XMLHttpRequest. Let's examine the proxy code. Let declare some fields that we will later use.

const originalXhrObject = new originalXhrClass()
const self = this
self.onreadystatechange = null
const doneEventHandlers = []
const requestHeaders = []
let lastDoneEvent
let changeResponseDone = false
let response

In the open method call we can replace the target URL:

Object.defineProperty(self, 'open', {
  value: function () {
    const url = arguments[1]
    arguments[1] = `https://my.example.com/my/modified/url?url=${url}`
    return originalXhrObject['open'].apply(originalXhrObject, arguments)
  },
})

The addRequestHeader calls are kept for later use.

Object.defineProperty(self, 'setRequestHeader', {
  value: function () {
    const [name, value] = arguments
    requestHeaders.push([name, value])
  },
})

The methods/getters/setters that we do not want to change are passed through to the original object.

const getters = ['status', 'statusText', 'readyState', 'responseXML', 'upload']
getters.forEach(function (property) {
  Object.defineProperty(self, property, {
    get: function () {
      return originalXhrObject[property]
    },
  })
})

const getterAndSetters = ['ontimeout, timeout', 'responseType', 'withCredentials', 'onload', 'onerror', 'onprogress']
getterAndSetters.forEach(function (property) {
  Object.defineProperty(self, property, {
    get: function () {
      return originalXhrObject[property]
    },
    set: function (val) {
      originalXhrObject[property] = val
    },
  })
})

const standardMethods = ['removeEventListener', 'abort', 'getAllResponseHeaders', 'getResponseHeader', 'overrideMimeType']
standardMethods.forEach(function (method) {
  Object.defineProperty(self, method, {
    value: function () {
      return originalXhrObject[method].apply(originalXhrObject, arguments)
    },
  })
})

Upon send of the data, we can update the request body.

Object.defineProperty(self, 'send', {
  value: async function () {
    originalXhrObject.setRequestHeader('Content-Type', 'application/json')
    const body = arguments[0]
    const newBody = {
      body,
      requestHeaders,
    }
    arguments[0] = JSON.stringify(newBody)

    return originalXhrObject['send'].apply(originalXhrObject, arguments)
  },
})

The response is modified once it arrives.

async function modifyResponse() {
  if (changeResponseDone) {
    return
  }
  response = `my modified response${originalXhrObject.responseText}`
  changeResponseDone = true
}

originalXhrObject.onreadystatechange = async function () {
  if (originalXhrObject.readyState === 4) {
    await modifyResponse()
  }
  if (self.onreadystatechange) {
    return self.onreadystatechange()
  }
  if (lastDoneEvent) {
    doneEventHandlers.forEach((handler) => {
      handler(lastDoneEvent)
    })
  }
}

we supply response getters.

const responseGetters = ['response', 'responseText']

responseGetters.forEach(function (property) {
  Object.defineProperty(self, property, {
    get: function () {
      return response
    },
  })
})

and we handle the notification per the addEventListener, and for the onload event.

Object.defineProperty(self, 'addEventListener', {
  value: function () {
    const eventType = arguments[0]
    if (eventType === 'load') {
      const handler = arguments[1]
      doneEventHandlers.push(handler)
      return
    }
    return originalXhrObject['addEventListener'].apply(originalXhrObject, arguments)
  },
})

originalXhrObject.addEventListener('load', (event) => {
  if (changeResponseDone) {
    doneEventHandlers.forEach((handler) => {
      setTimeout(() => {
        handler(event)
      }, 100)

    })
  } else {
    lastDoneEvent = event
  }
})

Final Note

We have shown how to proxy the XMLHttpRequest. A different simpler approach could be by using a service worker, where proxy will not be required, and hence there is no need to handle the various event listeners, and getters/setters.

A-Symmetric Encrypt and Decrypt on Golang

 

In this post we will review the steps to use RSA encryption/decryption on GO. 

This post is about using a-symmetric encryption for encrypt/decrypt. In case of need of a-symmetric encryption for sign/verify, check this post.

We start by creation a struct to represent the encryption:

package encryption

import (
   "bytes"
   "crypto/rand"
   "crypto/rsa"
   "crypto/sha256"
   "crypto/x509"
   "encoding/base64"
   "encoding/pem"
   "fmt"
   "hash"
   "io"
   "os"
   "radware.com/euda/commons/global/log"
)

type Key struct {
   privateKey *rsa.PrivateKey
   hash       hash.Hash
   random     io.Reader
}

func (k *Key) init() {
   k.hash = sha256.New()
   k.random = rand.Reader
}

We can generate a new key, which takes about 5 seconds on my desktop machine

func GenerateKey() (*Key, error) {
   privateKey, err := rsa.GenerateKey(rand.Reader, 4096)
   if err != nil {
      return nil, fmt.Errorf("generate key failed: %v", err)
   }

   k := Key{privateKey: privateKey}
   k.init()
   return &k, nil
}

To save the key to a file:

func (k *Key) SavePrivateKey() error {
   privateKeyBytes := x509.MarshalPKCS1PrivateKey(k.privateKey)
   privateKeyBlock := pem.Block{
      Type:  "PRIVATE KEY",
      Bytes: privateKeyBytes,
   }

   var writerBuffer bytes.Buffer

   err := pem.Encode(&writerBuffer, &privateKeyBlock)
   if err != nil {
      return fmt.Errorf("encode private key failed: %v", err)
   }

   fileData := writerBuffer.String()
   err = os.WriteFile(Config.PrivateKeyPath, []byte(fileData), 0644)
   if err != nil {
      return fmt.Errorf("write file failed: %v", err)
   }
   return nil
}

To load the key from a file:

func LoadPrivateKey() (*Key, error) {
   pemFile, err := os.ReadFile(Config.PrivateKeyPath)
   if err != nil {
      return nil, fmt.Errorf("read file failed: %v", err)
   }
   pemBlock, _ := pem.Decode(pemFile)
   privateKey, err := x509.ParsePKCS8PrivateKey(pemBlock.Bytes)
   if err != nil {
      return nil, fmt.Errorf("unmarshal key failed: %v", err)
   }

   rsaPrivateKey,ok :=privateKey.(*rsa.PrivateKey)
   if !ok{
      return nil, fmt.Errorf("convert failed: %v", err)
   }

   k := Key{privateKey: rsaPrivateKey}
   k.init()

   log.Info("private key loaded %v", Config.PrivateKeyPath)
   return &k, nil
}

To get the public key for sending it to other parties:

func (k *Key) GetPublicKeyPem() (string, error) {
   publicKeyBytes, err := x509.MarshalPKIXPublicKey(k.privateKey.Public())
   if err != nil {
      return "", fmt.Errorf("marshal public key failed: %v", err)
   }

   publicKeyBlock := pem.Block{
      Type:  "PUBLIC KEY",
      Bytes: publicKeyBytes,
   }

   var writerBuffer bytes.Buffer

   err = pem.Encode(&writerBuffer, &publicKeyBlock)
   if err != nil {
      return "", fmt.Errorf("encode public key failed: %v", err)
   }

   return writerBuffer.String(), nil
}

And finally, we can encrypt and decrypt:

func (k *Key) EncryptString(clearText string) (string, error) {
   encryptedBytes, err := rsa.EncryptOAEP(
      k.hash,
      k.random,
      &k.privateKey.PublicKey,
      []byte(clearText),
      nil,
   )

   if err != nil {
      return "", fmt.Errorf("encrypt failed: %v", err)
   }
   encryptedBase64 := base64.StdEncoding.EncodeToString(encryptedBytes)
   return encryptedBase64, nil
}

func (k *Key) DecryptString(base64Cipher string) (string, error) {
   encryptedBytes, err := base64.StdEncoding.DecodeString(base64Cipher)
   if err != nil {
      return "", fmt.Errorf("base64 decode failed: %v", err)
   }

   clearTextBytes, err := rsa.DecryptOAEP(
      k.hash,
      k.random,
      k.privateKey,
      encryptedBytes,
      nil,
   )
   if err != nil {
      return "", fmt.Errorf("decrypt failed: %v", err)
   }

   clearText := string(clearTextBytes)
   return clearText, nil
}

Final Note

This post is part of a series of posts about encryption. The full posts list is below.

• Using ECDSA in GO to Sign and Verify messages
• A-Symmetic Encrypt and Decrypt on Golang
• Symmetric Encrypt and Decrypt in Golang
• Using ECDSA in JavaScript to Sign and Verify messages
• A-Symmetic Encrypt and Decrypt on JavaScript
• Symmetric Encrypt and Decrypt in Javascript
• Using ECDSA in Python to Sign and Verify messages