Mathew K Analytics

Lesson 12 · ML algorithms deep dive

Comparing Every ML Algorithm: Capstone Project | ML Algorithms #12

The finale: every real algorithm from this series, run head to head on one final real dataset. Real wine chemistry data, three real cultivars, one fair,…

What you'll learn

Datasets used in this lesson

Save these next to the notebook. In Google Colab, upload them with the 📁 icon on the left first.

📓 Full notebook

Download .ipynb

ML Algorithms Deep-Dive, Video 12: Capstone, Comparing Every Algorithm#

  • The finale: every real algorithm from this series, run head to head on one final real dataset.
  • Real wine chemistry data, three real cultivars, one fair, real comparison.
  • Let's get into it.

Before You Start#

  • Open a new Jupyter Notebook in VS Code and select your Python interpreter as the kernel.
  • You'll need pandas, NumPy, Matplotlib, and scikit-learn.
  • Place wine.csv in the same folder as this notebook.
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.svm import SVC
from sklearn.naive_bayes import GaussianNB
from sklearn.model_selection import train_test_split, cross_val_score, StratifiedKFold
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import confusion_matrix

wine = pd.read_csv('wine.csv')
feature_cols = [c for c in wine.columns if c != 'target']
print(f'Real wines in this dataset: {len(wine)}')
print(f'Real chemical features: {len(feature_cols)}')
Real wines in this dataset: 178
Real chemical features: 13

Part 1: A Fair Playing Field#

X = wine[feature_cols].values
y = wine['target'].values
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42, stratify=y)
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
print(f'Real training wines: {len(X_train)}')
print(f'Real test wines: {len(X_test)}')
Real training wines: 133
Real test wines: 45

Part 2: Running Every Real Algorithm#

models = {
    'Logistic Regression': (LogisticRegression(max_iter=1000), True),
    'KNN (k=5)': (KNeighborsClassifier(n_neighbors=5), True),
    'Decision Tree': (DecisionTreeClassifier(random_state=42), False),
    'Random Forest': (RandomForestClassifier(n_estimators=100, random_state=42), False),
    'Gradient Boosting': (GradientBoostingClassifier(random_state=42), False),
    'SVM (RBF)': (SVC(kernel='rbf', random_state=42), True),
    'Naive Bayes': (GaussianNB(), True),
}
print(f'Real algorithms entered into this comparison: {len(models)}')
Real algorithms entered into this comparison: 7
results = {}
for name, (model, needs_scaling) in models.items():
    X_tr = X_train_scaled if needs_scaling else X_train
    X_te = X_test_scaled if needs_scaling else X_test
    cv_scores = cross_val_score(model, X_tr, y_train, cv=skf, scoring='accuracy')
    model.fit(X_tr, y_train)
    test_accuracy = model.score(X_te, y_test)
    results[name] = (cv_scores.mean(), cv_scores.std(), test_accuracy)
    print(f'{name}: real CV mean={cv_scores.mean():.4f}, real CV std={cv_scores.std():.4f}, real test acc={test_accuracy:.4f}')
Logistic Regression: real CV mean=0.9849, real CV std=0.0185, real test acc=1.0000
KNN (k=5): real CV mean=0.9627, real CV std=0.0331, real test acc=0.9333
Decision Tree: real CV mean=0.9026, real CV std=0.0554, real test acc=0.9556
Random Forest: real CV mean=0.9852, real CV std=0.0296, real test acc=1.0000
Gradient Boosting: real CV mean=0.9402, real CV std=0.0504, real test acc=0.9778
SVM (RBF): real CV mean=0.9926, real CV std=0.0148, real test acc=0.9778
Naive Bayes: real CV mean=0.9704, real CV std=0.0363, real test acc=0.9778

Part 3: Visualizing the Real Comparison#

names = list(results.keys())
means = [results[n][0] for n in names]
stds = [results[n][1] for n in names]
plt.figure(figsize=(11, 6))
plt.bar(names, means, yerr=stds, capsize=5, color='steelblue')
plt.title('Real Cross-Validated Accuracy by Algorithm')
plt.xlabel('Algorithm')
plt.ylabel('CV Accuracy')
plt.xticks(rotation=30, ha='right')
plt.ylim(0.8, 1.02)
plt.tight_layout()
plt.savefig('algorithm_comparison.png', dpi=150)
plt.show()
No description has been provided for this image

Part 4: Is the Real Gap at the Top Actually Meaningful?#

svm_folds = cross_val_score(SVC(kernel='rbf', random_state=42), X_train_scaled, y_train, cv=skf, scoring='accuracy')
rf_folds = cross_val_score(RandomForestClassifier(n_estimators=100, random_state=42), X_train, y_train, cv=skf, scoring='accuracy')
logreg_folds = cross_val_score(LogisticRegression(max_iter=1000), X_train_scaled, y_train, cv=skf, scoring='accuracy')
print(f'Real SVM per-fold scores: {[round(s, 4) for s in svm_folds]}')
print(f'Real Random Forest per-fold scores: {[round(s, 4) for s in rf_folds]}')
print(f'Real Logistic Regression per-fold scores: {[round(s, 4) for s in logreg_folds]}')
Real SVM per-fold scores: [np.float64(0.963), np.float64(1.0), np.float64(1.0), np.float64(1.0), np.float64(1.0)]
Real Random Forest per-fold scores: [np.float64(0.9259), np.float64(1.0), np.float64(1.0), np.float64(1.0), np.float64(1.0)]
Real Logistic Regression per-fold scores: [np.float64(0.963), np.float64(1.0), np.float64(1.0), np.float64(0.9615), np.float64(1.0)]

All three real top algorithms are essentially tied, each one trips on the same one or two genuinely hard real training wines, not on any systematic real weakness. A CV mean difference of a percentage point or two, on only 133 real training wines split five ways, is well within the kind of real noise a single hard fold can produce. This is the exact same real caution from video seven's grid search: don't over-read small real differences at the top of a leaderboard.

Part 5: Checking the Real Held-Out Test Set#

svm_final = SVC(kernel='rbf', random_state=42).fit(X_train_scaled, y_train)
svm_pred = svm_final.predict(X_test_scaled)
print('Real SVM test confusion matrix:')
print(confusion_matrix(y_test, svm_pred))
logreg_final = LogisticRegression(max_iter=1000).fit(X_train_scaled, y_train)
logreg_pred = logreg_final.predict(X_test_scaled)
print('Real logistic regression test confusion matrix:')
print(confusion_matrix(y_test, logreg_pred))
Real SVM test confusion matrix:
[[15  0  0]
 [ 0 18  0]
 [ 0  1 11]]
Real logistic regression test confusion matrix:
[[15  0  0]
 [ 0 18  0]
 [ 0  0 12]]

Part 6: The Real Final Recommendation#

Recommendation: logistic regression. It matched SVM and Random Forest's real accuracy, including a real perfect score on the held-out test set, while training faster and, per video two, offering real coefficients that a human can actually read and explain. SVM remains a completely legitimate real alternative if a genuinely non-linear boundary were ever needed, and Random Forest if real feature importance rankings mattered more than raw coefficients. The real, honest takeaway from all twelve videos: reach for the simplest real algorithm that gets the job done, and only add complexity once a real, validated need for it actually shows up.

Part 7: Saving Your Work#

import joblib
joblib.dump(logreg_final, 'capstone_final_model.joblib')
reloaded_model = joblib.load('capstone_final_model.joblib')
print(f'Real original model test accuracy: {logreg_final.score(X_test_scaled, y_test):.4f}')
print(f'Real reloaded model test accuracy: {reloaded_model.score(X_test_scaled, y_test):.4f}')
Real original model test accuracy: 1.0000
Real reloaded model test accuracy: 1.0000
results_df = pd.DataFrame([
    {'Algorithm': n, 'CV_Mean_Accuracy': round(v[0], 4), 'CV_Std': round(v[1], 4), 'Test_Accuracy': round(v[2], 4)}
    for n, v in results.items()
])
results_df.to_csv('algorithm_comparison_results.csv', index=False)
print('Real comparison table saved to algorithm_comparison_results.csv')
print(results_df)
Real comparison table saved to algorithm_comparison_results.csv
             Algorithm  CV_Mean_Accuracy  CV_Std  Test_Accuracy
0  Logistic Regression            0.9849  0.0185         1.0000
1            KNN (k=5)            0.9627  0.0331         0.9333
2        Decision Tree            0.9026  0.0554         0.9556
3        Random Forest            0.9852  0.0296         1.0000
4    Gradient Boosting            0.9402  0.0504         0.9778
5            SVM (RBF)            0.9926  0.0148         0.9778
6          Naive Bayes            0.9704  0.0363         0.9778

Wrap-Up: The Series, End to End#

  • Twelve videos, one real algorithm each, tested on real cars, real tumors, real flowers, real mushrooms, real wine, real patients, real customers, and real handwritten digits.
  • Regression: linear, then gradient-boosted, with an honest case where simplicity won.
  • Classification: logistic regression, KNN, decision trees, random forests, SVMs, and Naive Bayes, each rematched against a real neighbor for a genuine, direct comparison.
  • Unsupervised learning: K-Means and hierarchical clustering, independently converging on the same real customer segments.
  • PCA, compressing real high-dimensional data without losing what actually mattered.
  • This capstone, running every real classifier head to head and choosing the simplest real winner among a genuine statistical tie.
  • That's the full ML Algorithms Deep-Dive series. Thanks for real building all twelve of these with me. Subscribe for what's coming next see you there.

🧠 Test yourself

Pick an answer to check it. Your best score is saved in this browser.

  1. Why does the capstone's models dictionary store a True or False flag next to each model, for example 'KNN (k=5)': (KNeighborsClassifier(n_neighbors=5), True)?

  2. What is the point of using the same train/test split and the same StratifiedKFold for every algorithm?

  3. Look at this loop from the lesson:

    cv_scores = cross_val_score(model, X_tr, y_train, cv=skf, scoring='accuracy')
    model.fit(X_tr, y_train)
    test_accuracy = model.score(X_te, y_test)
    

    What do cv_scores and test_accuracy each measure?

  4. The top three models (SVM, Random Forest, logistic regression) differ by only a point or two of CV accuracy on 133 training wines. How should you read that?

  5. Why did the capstone recommend logistic regression among the tied top models?

💻 Practice exercises

Try each one in the notebook or in your own editor before opening the solution.

  1. Load the wine data, split it 75/25 with stratify, scale it, and compare LogisticRegression and KNeighborsClassifier(n_neighbors=5) on test accuracy.

    💡 Hint

    Fit the scaler on the training split only.

    ✅ Show solution
    import matplotlib
    matplotlib.use('Agg')
    from sklearn.datasets import load_wine
    from sklearn.model_selection import train_test_split
    from sklearn.preprocessing import StandardScaler
    from sklearn.linear_model import LogisticRegression
    from sklearn.neighbors import KNeighborsClassifier
    
    X, y = load_wine(return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.25, random_state=42, stratify=y)
    scaler = StandardScaler()
    X_train_s = scaler.fit_transform(X_train)
    X_test_s = scaler.transform(X_test)
    
    for name, model in [('Logistic Regression', LogisticRegression(max_iter=1000)),
                        ('KNN (k=5)', KNeighborsClassifier(n_neighbors=5))]:
        model.fit(X_train_s, y_train)
        print(name, 'test accuracy:', round(model.score(X_test_s, y_test), 4))
    
  2. Build a models dictionary with the lesson's (model, needs_scaling) pairs for all seven algorithms. Run 5-fold stratified cross_val_score for each and print a table of CV mean and standard deviation.

    💡 Hint

    Choose X_train_scaled or X_train depending on the flag.

    ✅ Show solution
    import matplotlib
    matplotlib.use('Agg')
    from sklearn.datasets import load_wine
    from sklearn.linear_model import LogisticRegression
    from sklearn.neighbors import KNeighborsClassifier
    from sklearn.tree import DecisionTreeClassifier
    from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
    from sklearn.svm import SVC
    from sklearn.naive_bayes import GaussianNB
    from sklearn.model_selection import train_test_split, cross_val_score, StratifiedKFold
    from sklearn.preprocessing import StandardScaler
    
    X, y = load_wine(return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.25, random_state=42, stratify=y)
    X_train_scaled = StandardScaler().fit_transform(X_train)
    skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
    
    models = {
        'Logistic Regression': (LogisticRegression(max_iter=1000), True),
        'KNN (k=5)': (KNeighborsClassifier(n_neighbors=5), True),
        'Decision Tree': (DecisionTreeClassifier(random_state=42), False),
        'Random Forest': (RandomForestClassifier(n_estimators=100, random_state=42), False),
        'Gradient Boosting': (GradientBoostingClassifier(random_state=42), False),
        'SVM (RBF)': (SVC(kernel='rbf', random_state=42), True),
        'Naive Bayes': (GaussianNB(), True),
    }
    for name, (model, needs_scaling) in models.items():
        X_tr = X_train_scaled if needs_scaling else X_train
        scores = cross_val_score(model, X_tr, y_train, cv=skf, scoring='accuracy')
        print(f'{name:20s} mean={scores.mean():.4f} std={scores.std():.4f}')
    
  3. Compare the seven algorithms using a leak-free approach: wrap scaled models in make_pipeline so the scaler is refit in each fold. Save a bar chart of CV means with error bars to a PNG (Agg backend), and print the leaderboard sorted from best to worst along with which models are within one standard deviation of the leader.

    💡 Hint

    A model is tied with the leader if its mean is at least leader_mean - leader_std.

    ✅ Show solution
    import matplotlib
    matplotlib.use('Agg')
    import numpy as np
    import matplotlib.pyplot as plt
    from sklearn.datasets import load_wine
    from sklearn.pipeline import make_pipeline
    from sklearn.preprocessing import StandardScaler
    from sklearn.linear_model import LogisticRegression
    from sklearn.neighbors import KNeighborsClassifier
    from sklearn.tree import DecisionTreeClassifier
    from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
    from sklearn.svm import SVC
    from sklearn.naive_bayes import GaussianNB
    from sklearn.model_selection import train_test_split, cross_val_score, StratifiedKFold
    
    X, y = load_wine(return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.25, random_state=42, stratify=y)
    skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
    
    models = {
        'Logistic Regression': make_pipeline(StandardScaler(), LogisticRegression(max_iter=1000)),
        'KNN (k=5)': make_pipeline(StandardScaler(), KNeighborsClassifier(n_neighbors=5)),
        'Decision Tree': DecisionTreeClassifier(random_state=42),
        'Random Forest': RandomForestClassifier(n_estimators=100, random_state=42),
        'Gradient Boosting': GradientBoostingClassifier(random_state=42),
        'SVM (RBF)': make_pipeline(StandardScaler(), SVC(kernel='rbf', random_state=42)),
        'Naive Bayes': make_pipeline(StandardScaler(), GaussianNB()),
    }
    results = {}
    for name, model in models.items():
        s = cross_val_score(model, X_train, y_train, cv=skf, scoring='accuracy')
        results[name] = (s.mean(), s.std())
    
    ranked = sorted(results.items(), key=lambda kv: kv[1][0], reverse=True)
    lead_mean, lead_std = ranked[0][1]
    for name, (m, sd) in ranked:
        tied = m >= lead_mean - lead_std
        print(f'{name:20s} {m:.4f} +/- {sd:.4f}  {"(tied with leader)" if tied else ""}')
    
    names = [n for n, _ in ranked]
    plt.figure(figsize=(10, 5))
    plt.bar(names, [results[n][0] for n in names], yerr=[results[n][1] for n in names],
            capsize=5, color='steelblue')
    plt.xticks(rotation=30, ha='right')
    plt.ylim(0.8, 1.02)
    plt.ylabel('CV accuracy')
    plt.tight_layout()
    plt.savefig('algorithm_comparison.png', dpi=100)
    print('saved algorithm_comparison.png')
    

Found this useful?

All lessons, notebooks and datasets here are free. If they helped you, a coffee keeps new lessons coming.