Model Selection & Pipeline

1/**2 * Example 37: Model Selection & Pipeline3 *4 * Pipeline, crossValidate, GridSearchCV and RandomizedSearchCV: chain a scaler5 * and a model, score models with k-fold cross-validation, and tune6 * hyperparameters, including those of a pipeline step.7 */8 9import { makeClassification } from "deepbox/datasets";10import { accuracy } from "deepbox/metrics";11import {12  crossValidate,13  GridSearchCV,14  KNeighborsClassifier,15  LogisticRegression,16  Pipeline,17  RandomForestClassifier,18  RandomizedSearchCV,19  SVC,20} from "deepbox/ml";21import { StandardScaler, trainTestSplit } from "deepbox/preprocess";22 23console.log("=".repeat(60));24console.log("Example 37: Model Selection & Pipeline");25console.log("=".repeat(60));26 27// ============================================================================28// Generate dataset29// ============================================================================30 31const [X, y] = makeClassification({32  nSamples: 200,33  nFeatures: 8,34  nInformative: 5,35  nClasses: 2,36  randomState: 42,37});38 39const [XTrain, XTest, yTrain, yTest] = trainTestSplit(X, y, {40  testSize: 0.25,41  randomState: 42,42});43 44console.log(`Dataset: ${X.shape[0]} samples, ${X.shape[1]} features`);45console.log(`Train: ${XTrain.shape[0]}, Test: ${XTest.shape[0]}`);46 47// ============================================================================48// Part 1: Pipeline (chain Preprocessing + Model)49// ============================================================================50console.log("\nPart 1: Pipeline");51console.log("-".repeat(60));52 53// A Pipeline chains transformers and a final estimator into one object.54// Steps are [name, estimator] pairs. Every step except the last must be a transformer.55const pipe = new Pipeline([56  ["scaler", new StandardScaler()],57  ["classifier", new LogisticRegression({ maxIter: 200 })],58]);59 60// fit() calls fitTransform on each intermediate step, then fit on the final step61pipe.fit(XTrain, yTrain);62 63// predict() calls transform on each intermediate step, then predict on the final step64const pipePred = pipe.predict(XTest);65const pipeAcc = accuracy(yTest, pipePred);66 67console.log("Pipeline (StandardScaler, LogisticRegression):");68console.log(`  Accuracy: ${(Number(pipeAcc) * 100).toFixed(2)}%`);69 70// The same scaler in front of a different model71const pipe2 = new Pipeline([72  ["scaler", new StandardScaler()],73  ["classifier", new KNeighborsClassifier({ nNeighbors: 5 })],74]);75pipe2.fit(XTrain, yTrain);76 77const pipe2Pred = pipe2.predict(XTest);78const pipe2Acc = accuracy(yTest, pipe2Pred);79 80console.log("Pipeline (StandardScaler, KNN):");81console.log(`  Accuracy: ${(Number(pipe2Acc) * 100).toFixed(2)}%`);82 83// ============================================================================84// Part 2: Cross-Validation85// ============================================================================86console.log("\nPart 2: Cross-Validation");87console.log("-".repeat(60));88 89// crossValidate fits a fresh copy of the model on each fold and scores it on the held-out part90const lr = new LogisticRegression({ maxIter: 200 });91const cvResult = crossValidate(lr, XTrain, yTrain, { cv: 5 });92 93console.log("LogisticRegression 5-fold Cross-Validation:");94console.log(95  `  Test scores: [${cvResult.testScores["score"]?.map((s) => s.toFixed(4)).join(", ")}]`96);97const meanScore =98  (cvResult.testScores["score"]?.reduce((a, b) => a + b, 0) ?? 0) /99  (cvResult.testScores["score"]?.length ?? 1);100console.log(`  Mean score: ${meanScore.toFixed(4)}`);101 102// Cross-validate different models for comparison103const models = [104  {105    name: "LogisticRegression",106    model: new LogisticRegression({ maxIter: 200 }),107  },108  { name: "KNN (k=5)", model: new KNeighborsClassifier({ nNeighbors: 5 }) },109  {110    name: "RandomForest (20)",111    model: new RandomForestClassifier({ nEstimators: 20, randomState: 42 }),112  },113  { name: "SVC (RBF)", model: new SVC({ kernel: "rbf", gamma: "scale" }) },114];115 116console.log("\nModel Comparison (5-fold CV):");117for (const { name, model } of models) {118  const cv = crossValidate(model, XTrain, yTrain, { cv: 5 });119  const scores = cv.testScores["score"] ?? [];120  const mean = scores.reduce((a, b) => a + b, 0) / scores.length;121  const std = Math.sqrt(scores.reduce((a, b) => a + (b - mean) ** 2, 0) / scores.length);122  console.log(`  ${name.padEnd(22)} Mean: ${mean.toFixed(4)} ± ${std.toFixed(4)}`);123}124 125// A Pipeline can be cross-validated directly. The scaler is refit on each training fold,126// so no statistics from the held-out fold leak into the scaling.127const pipeCv = crossValidate(pipe, XTrain, yTrain, { cv: 5 });128const pipeScores = pipeCv.testScores["score"] ?? [];129const pipeMean = pipeScores.reduce((a, b) => a + b, 0) / pipeScores.length;130console.log(`  ${"Scaler + LogReg".padEnd(22)} Mean: ${pipeMean.toFixed(4)}`);131 132// ============================================================================133// Part 3: GridSearchCV (exhaustive hyperparameter search)134// ============================================================================135console.log("\nPart 3: GridSearchCV");136console.log("-".repeat(60));137 138// GridSearchCV scores every combination in the grid with k-fold cross-validation139const knnGrid = new GridSearchCV(140  new KNeighborsClassifier(),141  {142    nNeighbors: [3, 5, 7, 9, 11],143  },144  { cv: 5 }145);146 147knnGrid.fit(XTrain, yTrain);148 149console.log("GridSearchCV for KNeighborsClassifier:");150console.log(`  Best params: nNeighbors=${knnGrid.bestParams["nNeighbors"]}`);151console.log(`  Best CV score: ${knnGrid.bestScore.toFixed(4)}`);152 153// Evaluate best model on test set154if (knnGrid.bestEstimator) {155  const best = knnGrid.bestEstimator as KNeighborsClassifier;156  const bestPred = best.predict(XTest);157  const bestAcc = accuracy(yTest, bestPred);158  console.log(`  Test accuracy (best model): ${(Number(bestAcc) * 100).toFixed(2)}%`);159}160 161// Show all CV results162console.log("\n  All GridSearch Results:");163for (const result of knnGrid.cvResults) {164  const params = JSON.stringify(result.params);165  const scores = result.scores;166  const mean = result.meanScore;167  const std = Math.sqrt(scores.reduce((a, b) => a + (b - mean) ** 2, 0) / scores.length);168  console.log(`    ${params.padEnd(20)} mean: ${mean.toFixed(4)}, std: ${std.toFixed(4)}`);169}170 171// ============================================================================172// Part 4: RandomizedSearchCV (random hyperparameter sampling)173// ============================================================================174console.log("\nPart 4: RandomizedSearchCV");175console.log("-".repeat(60));176 177// RandomizedSearchCV tries nIter random combinations from the grid, which costs less than the full grid178const rfRandomSearch = new RandomizedSearchCV(179  new RandomForestClassifier({ randomState: 42 }),180  {181    nEstimators: [10, 20, 50, 100],182    maxDepth: [3, 5, 10, 15],183  },184  { cv: 3, nIter: 8, randomState: 42 }185);186 187rfRandomSearch.fit(XTrain, yTrain);188 189console.log("RandomizedSearchCV for RandomForestClassifier (8 iterations):");190console.log(`  Best params: ${JSON.stringify(rfRandomSearch.bestParams)}`);191console.log(`  Best CV score: ${rfRandomSearch.bestScore.toFixed(4)}`);192 193if (rfRandomSearch.bestEstimator) {194  const best = rfRandomSearch.bestEstimator as RandomForestClassifier;195  const bestPred = best.predict(XTest);196  const bestAcc = accuracy(yTest, bestPred);197  console.log(`  Test accuracy (best model): ${(Number(bestAcc) * 100).toFixed(2)}%`);198}199 200// ============================================================================201// Part 5: GridSearchCV with Pipeline202// ============================================================================203console.log("\nPart 5: GridSearchCV with a Pipeline");204console.log("-".repeat(60));205 206// A parameter of a pipeline step is addressed as stepName__parameterName.207// The scaler is refit inside every fold, so the search is free of scaling leakage.208const pipeForSearch = new Pipeline([209  ["scaler", new StandardScaler()],210  ["classifier", new KNeighborsClassifier()],211]);212 213const pipeGrid = new GridSearchCV(214  pipeForSearch,215  {216    classifier__nNeighbors: [3, 5, 7, 9],217  },218  { cv: 5 }219);220 221pipeGrid.fit(XTrain, yTrain);222 223console.log("GridSearchCV over a Pipeline (scaler + KNN):");224console.log(`  Best params: ${JSON.stringify(pipeGrid.bestParams)}`);225console.log(`  Best CV score: ${pipeGrid.bestScore.toFixed(4)}`);226 227if (pipeGrid.bestEstimator) {228  const best = pipeGrid.bestEstimator as Pipeline;229  const bestPred = best.predict(XTest);230  const bestAcc = accuracy(yTest, bestPred);231  console.log(`  Test accuracy: ${(Number(bestAcc) * 100).toFixed(2)}%`);232}233 234// ============================================================================235// Summary236// ============================================================================237console.log("\nKey Takeaways");238console.log("-".repeat(60));239console.log("• Pipeline: transformers and an estimator behind one fit/predict");240console.log("• crossValidate: k-fold scores for a model or a whole pipeline");241console.log("• GridSearchCV: tries every combination in the grid");242console.log("• RandomizedSearchCV: tries nIter random combinations, cheaper for large grids");243console.log("• Pipeline parameters are tuned as stepName__parameterName");244console.log("• Choose models by CV score, and keep the test set for a final check");245console.log("• Scale features before distance-based models (KNN, SVM)");246 247console.log("\nModel Selection & Pipeline Example Complete!");248console.log("=".repeat(60));