Stochastic Gradient Descent (SGD) Optimizer

Classical SGD optimizer with optional momentum and weight decay.

Variants

• Standard SGD: θ = θ - lr * ∇θ
• SGD with Momentum: Uses exponential moving average of gradients
• SGD with Nesterov: Lookahead momentum variant

Usage Example

-- Create optimizer
let opt := SGDConfig.default.withLearningRate 0.01 |>.withMomentum 0.9

-- Optimization step
let (newParams, newState) := opt.step params grads state

structure Hesper.Optimizer.SGD.SGDConfig : Type

SGD optimizer configuration

• learningRate : Float

  Learning rate (step size)

• momentum : Float

  Momentum coefficient (0 = no momentum, typical: 0.9)

• weightDecay : Float

  Weight decay (L2 regularization coefficient)

• nesterov : Bool

  Use Nesterov momentum

• dampening : Float

  Dampening for momentum (typical: 0 for standard momentum)

instance Hesper.Optimizer.SGD.instInhabitedSGDConfig : Inhabited SGDConfig

Equations

• Hesper.Optimizer.SGD.instInhabitedSGDConfig = { default := Hesper.Optimizer.SGD.instInhabitedSGDConfig.default }

def Hesper.Optimizer.SGD.instInhabitedSGDConfig.default : SGDConfig

Equations

• Hesper.Optimizer.SGD.instInhabitedSGDConfig.default = { learningRate := default, momentum := default, weightDecay := default, nesterov := default, dampening := default }

instance Hesper.Optimizer.SGD.instReprSGDConfig : Repr SGDConfig

Equations

• Hesper.Optimizer.SGD.instReprSGDConfig = { reprPrec := Hesper.Optimizer.SGD.instReprSGDConfig.repr }

def Hesper.Optimizer.SGD.instReprSGDConfig.repr : SGDConfig → Nat → Std.Format

Equations

• One or more equations did not get rendered due to their size.

def Hesper.Optimizer.SGD.SGDConfig.default : SGDConfig

Default SGD configuration (no momentum, lr=0.01)

Equations

• Hesper.Optimizer.SGD.SGDConfig.default = { }

def Hesper.Optimizer.SGD.SGDConfig.withLearningRate (config : SGDConfig) (lr : Float) : SGDConfig

Create SGD with specified learning rate

Equations

• config.withLearningRate lr = { learningRate := lr, momentum := config.momentum, weightDecay := config.weightDecay, nesterov := config.nesterov, dampening := config.dampening }

def Hesper.Optimizer.SGD.SGDConfig.withMomentum (config : SGDConfig) (m : Float) : SGDConfig

Create SGD with momentum

Equations

• config.withMomentum m = { learningRate := config.learningRate, momentum := m, weightDecay := config.weightDecay, nesterov := config.nesterov, dampening := config.dampening }

def Hesper.Optimizer.SGD.SGDConfig.withWeightDecay (config : SGDConfig) (wd : Float) : SGDConfig

Create SGD with weight decay (L2 regularization)

Equations

• config.withWeightDecay wd = { learningRate := config.learningRate, momentum := config.momentum, weightDecay := wd, nesterov := config.nesterov, dampening := config.dampening }

def Hesper.Optimizer.SGD.SGDConfig.withNesterov (config : SGDConfig) : SGDConfig

Enable Nesterov momentum

Equations

• config.withNesterov = { learningRate := config.learningRate, momentum := config.momentum, weightDecay := config.weightDecay, nesterov := true, dampening := config.dampening }

structure Hesper.Optimizer.SGD.SGDState : Type

SGD optimizer state (momentum buffers)

• momentumBuffers : Array (Array Float)

  Momentum buffers for each parameter

• step : Nat

  Number of steps taken

instance Hesper.Optimizer.SGD.instInhabitedSGDState : Inhabited SGDState

Equations

• Hesper.Optimizer.SGD.instInhabitedSGDState = { default := Hesper.Optimizer.SGD.instInhabitedSGDState.default }

def Hesper.Optimizer.SGD.instInhabitedSGDState.default : SGDState

Equations

• Hesper.Optimizer.SGD.instInhabitedSGDState.default = { momentumBuffers := default, step := default }

instance Hesper.Optimizer.SGD.instReprSGDState : Repr SGDState

Equations

• Hesper.Optimizer.SGD.instReprSGDState = { reprPrec := Hesper.Optimizer.SGD.instReprSGDState.repr }

def Hesper.Optimizer.SGD.instReprSGDState.repr : SGDState → Nat → Std.Format

Equations

• One or more equations did not get rendered due to their size.

def Hesper.Optimizer.SGD.SGDState.init (paramShapes : Array Nat) : SGDState

Initialize SGD state for given parameter shapes

Equations

• Hesper.Optimizer.SGD.SGDState.init paramShapes = { momentumBuffers := Array.map (fun (size : Nat) => { toList := List.replicate size 0.0 }) paramShapes, step := 0 }

def Hesper.Optimizer.SGD.SGDState.fromParams (params : Array (Array Float)) : SGDState

Initialize SGD state from parameters

Equations

• Hesper.Optimizer.SGD.SGDState.fromParams params = { momentumBuffers := Array.map (fun (p : Array Float) => { toList := List.replicate p.size 0.0 }) params, step := 0 }

def Hesper.Optimizer.SGD.updateParam (config : SGDConfig) (param grad momentumBuffer : Array Float) : Array Float × Array Float

Update a single parameter with SGD

Equations

• One or more equations did not get rendered due to their size.

def Hesper.Optimizer.SGD.step (config : SGDConfig) (params grads : Array (Array Float)) (state : SGDState) : Array (Array Float) × SGDState

Perform one optimization step for all parameters

Equations

• One or more equations did not get rendered due to their size.

def Hesper.Optimizer.SGD.stepIndexed (config : SGDConfig) (params grads : Array (Array Float)) (state : SGDState) : Array (Array Float) × SGDState

Perform optimization step with proper indexing

Equations

• One or more equations did not get rendered due to their size.

def Hesper.Optimizer.SGD.gradNorm (grads : Array (Array Float)) : Float

Helper: Compute L2 norm of gradients (for monitoring)

Equations

• Hesper.Optimizer.SGD.gradNorm grads = (Array.foldl (fun (acc : Float) (grad : Array Float) => acc + Array.foldl (fun (acc2 g : Float) => acc2 + g * g) 0.0 grad) 0.0 grads).sqrt

def Hesper.Optimizer.SGD.paramNorm (params : Array (Array Float)) : Float

Helper: Compute parameter L2 norm

Equations

• Hesper.Optimizer.SGD.paramNorm params = (Array.foldl (fun (acc : Float) (param : Array Float) => acc + Array.foldl (fun (acc2 p : Float) => acc2 + p * p) 0.0 param) 0.0 params).sqrt