Activation Functions

1/**2 * Example 15: Activation Functions3 *4 * Explore different activation functions used in neural networks.5 * Learn when to use each activation function.6 */78import { mkdirSync, writeFileSync } from "node:fs";9import {10  elu,11  gelu,12  leakyRelu,13  linspace,14  mish,15  relu,16  sigmoid,17  softmax,18  softplus,19  swish,20  tensor,21} from "deepbox/ndarray";22import { Figure } from "deepbox/plot";2324console.log("=== Activation Functions ===\n");2526mkdirSync("docs/examples/15-activation-functions/output", { recursive: true });2728// Generate input range29const x = linspace(-5, 5, 100);3031console.log("1. ReLU (Rectified Linear Unit):");32console.log("-".repeat(50));33const relu_out = relu(x);34console.log("f(x) = max(0, x)");35console.log("Use: Hidden layers, fast computation");36console.log("Range: [0, ∞)\n");3738console.log("2. Sigmoid:");39console.log("-".repeat(50));40const sigmoid_out = sigmoid(x);41console.log("f(x) = 1 / (1 + e^(-x))");42console.log("Use: Binary classification output");43console.log("Range: (0, 1)\n");4445console.log("3. Softmax:");46console.log("-".repeat(50));47const sample = tensor([1.0, 2.0, 3.0, 4.0]);48const softmax_out = softmax(sample);49console.log("Input:", sample.toString());50console.log("Output:", softmax_out.toString());51console.log("Use: Multi-class classification output");52console.log("Properties: Outputs sum to 1.0\n");5354console.log("4. GELU (Gaussian Error Linear Unit):");55console.log("-".repeat(50));56const gelu_out = gelu(x);57console.log("f(x) = x * Φ(x), where Φ is CDF of normal distribution");58console.log("Use: Transformers, modern architectures");59console.log("Smoother than ReLU\n");6061console.log("5. Leaky ReLU:");62console.log("-".repeat(50));63leakyRelu(x, 0.01);64console.log("f(x) = max(αx, x), α = 0.01");65console.log("Use: Prevents dying ReLU problem");66console.log("Allows small negative values\n");6768console.log("6. ELU (Exponential Linear Unit):");69console.log("-".repeat(50));70elu(x, 1.0);71console.log("f(x) = x if x > 0, else α(e^x - 1)");72console.log("Use: Can produce negative outputs");73console.log("Smoother than ReLU\n");7475console.log("7. Mish:");76console.log("-".repeat(50));77mish(x);78console.log("f(x) = x * tanh(softplus(x))");79console.log("Use: State-of-the-art in some tasks");80console.log("Self-regularizing, smooth\n");8182console.log("8. Swish (SiLU):");83console.log("-".repeat(50));84swish(x);85console.log("f(x) = x * sigmoid(x)");86console.log("Use: Discovered by neural architecture search");87console.log("Non-monotonic, smooth\n");8889console.log("9. Softplus:");90console.log("-".repeat(50));91softplus(x);92console.log("f(x) = log(1 + e^x)");93console.log("Use: Smooth approximation of ReLU");94console.log("Always positive\n");9596// Visualize activations97console.log("Creating visualization...");98const fig = new Figure({ width: 800, height: 600 });99const ax = fig.addAxes();100101ax.plot(x, relu_out, { color: "#1f77b4", linewidth: 2 });102ax.plot(x, sigmoid_out, { color: "#ff7f0e", linewidth: 2 });103ax.plot(x, gelu_out, { color: "#2ca02c", linewidth: 2 });104ax.setTitle("Activation Functions Comparison");105ax.setXLabel("Input");106ax.setYLabel("Output");107108const svg = fig.renderSVG();109writeFileSync("docs/examples/15-activation-functions/output/activations.svg", svg.svg);110console.log("✓ Saved: output/activations.svg\n");111112console.log("Selection Guide:");113console.log("• ReLU: Default choice, fast and effective");114console.log("• Sigmoid: Binary classification output layer");115console.log("• Softmax: Multi-class classification output layer");116console.log("• GELU/Mish/Swish: Modern alternatives, often better performance");117console.log("• Leaky ReLU/ELU: When dying ReLU is a problem");118119console.log("\n✓ Activation functions complete!");120