dev/notebooks/auto_examples/applications/plot_prediction_latency.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "\n# Prediction Latency\n\n\nThis is an example showing the prediction latency of various scikit-learn\nestimators.\n\nThe goal is to measure the latency one can expect when doing predictions\neither in bulk or atomic (i.e. one by one) mode.\n\nThe plots represent the distribution of the prediction latency as a boxplot.\n\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "# Authors: Eustache Diemert <eustache@diemert.fr>\n# License: BSD 3 clause\n\nfrom __future__ import print_function\nfrom collections import defaultdict\n\nimport time\nimport gc\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import train_test_split\nfrom scipy.stats import scoreatpercentile\nfrom sklearn.datasets.samples_generator import make_regression\nfrom sklearn.ensemble.forest import RandomForestRegressor\nfrom sklearn.linear_model.ridge import Ridge\nfrom sklearn.linear_model.stochastic_gradient import SGDRegressor\nfrom sklearn.svm.classes import SVR\nfrom sklearn.utils import shuffle\n\n\ndef _not_in_sphinx():\n    # Hack to detect whether we are running by the sphinx builder\n    return '__file__' in globals()\n\n\ndef atomic_benchmark_estimator(estimator, X_test, verbose=False):\n    \"\"\"Measure runtime prediction of each instance.\"\"\"\n    n_instances = X_test.shape[0]\n    runtimes = np.zeros(n_instances, dtype=np.float)\n    for i in range(n_instances):\n        instance = X_test[[i], :]\n        start = time.time()\n        estimator.predict(instance)\n        runtimes[i] = time.time() - start\n    if verbose:\n        print(\"atomic_benchmark runtimes:\", min(runtimes), scoreatpercentile(\n            runtimes, 50), max(runtimes))\n    return runtimes\n\n\ndef bulk_benchmark_estimator(estimator, X_test, n_bulk_repeats, verbose):\n    \"\"\"Measure runtime prediction of the whole input.\"\"\"\n    n_instances = X_test.shape[0]\n    runtimes = np.zeros(n_bulk_repeats, dtype=np.float)\n    for i in range(n_bulk_repeats):\n        start = time.time()\n        estimator.predict(X_test)\n        runtimes[i] = time.time() - start\n    runtimes = np.array(list(map(lambda x: x / float(n_instances), runtimes)))\n    if verbose:\n        print(\"bulk_benchmark runtimes:\", min(runtimes), scoreatpercentile(\n            runtimes, 50), max(runtimes))\n    return runtimes\n\n\ndef benchmark_estimator(estimator, X_test, n_bulk_repeats=30, verbose=False):\n    \"\"\"\n    Measure runtimes of prediction in both atomic and bulk mode.\n\n    Parameters\n    ----------\n    estimator : already trained estimator supporting `predict()`\n    X_test : test input\n    n_bulk_repeats : how many times to repeat when evaluating bulk mode\n\n    Returns\n    -------\n    atomic_runtimes, bulk_runtimes : a pair of `np.array` which contain the\n    runtimes in seconds.\n\n    \"\"\"\n    atomic_runtimes = atomic_benchmark_estimator(estimator, X_test, verbose)\n    bulk_runtimes = bulk_benchmark_estimator(estimator, X_test, n_bulk_repeats,\n                                             verbose)\n    return atomic_runtimes, bulk_runtimes\n\n\ndef generate_dataset(n_train, n_test, n_features, noise=0.1, verbose=False):\n    \"\"\"Generate a regression dataset with the given parameters.\"\"\"\n    if verbose:\n        print(\"generating dataset...\")\n\n    X, y, coef = make_regression(n_samples=n_train + n_test,\n                                 n_features=n_features, noise=noise, coef=True)\n\n    random_seed = 13\n    X_train, X_test, y_train, y_test = train_test_split(\n        X, y, train_size=n_train, random_state=random_seed)\n    X_train, y_train = shuffle(X_train, y_train, random_state=random_seed)\n\n    X_scaler = StandardScaler()\n    X_train = X_scaler.fit_transform(X_train)\n    X_test = X_scaler.transform(X_test)\n\n    y_scaler = StandardScaler()\n    y_train = y_scaler.fit_transform(y_train[:, None])[:, 0]\n    y_test = y_scaler.transform(y_test[:, None])[:, 0]\n\n    gc.collect()\n    if verbose:\n        print(\"ok\")\n    return X_train, y_train, X_test, y_test\n\n\ndef boxplot_runtimes(runtimes, pred_type, configuration):\n    \"\"\"\n    Plot a new `Figure` with boxplots of prediction runtimes.\n\n    Parameters\n    ----------\n    runtimes : list of `np.array` of latencies in micro-seconds\n    cls_names : list of estimator class names that generated the runtimes\n    pred_type : 'bulk' or 'atomic'\n\n    \"\"\"\n\n    fig, ax1 = plt.subplots(figsize=(10, 6))\n    bp = plt.boxplot(runtimes, )\n\n    cls_infos = ['%s\\n(%d %s)' % (estimator_conf['name'],\n                                  estimator_conf['complexity_computer'](\n                                      estimator_conf['instance']),\n                                  estimator_conf['complexity_label']) for\n                 estimator_conf in configuration['estimators']]\n    plt.setp(ax1, xticklabels=cls_infos)\n    plt.setp(bp['boxes'], color='black')\n    plt.setp(bp['whiskers'], color='black')\n    plt.setp(bp['fliers'], color='red', marker='+')\n\n    ax1.yaxis.grid(True, linestyle='-', which='major', color='lightgrey',\n                   alpha=0.5)\n\n    ax1.set_axisbelow(True)\n    ax1.set_title('Prediction Time per Instance - %s, %d feats.' % (\n        pred_type.capitalize(),\n        configuration['n_features']))\n    ax1.set_ylabel('Prediction Time (us)')\n\n    plt.show()\n\n\ndef benchmark(configuration):\n    \"\"\"Run the whole benchmark.\"\"\"\n    X_train, y_train, X_test, y_test = generate_dataset(\n        configuration['n_train'], configuration['n_test'],\n        configuration['n_features'])\n\n    stats = {}\n    for estimator_conf in configuration['estimators']:\n        print(\"Benchmarking\", estimator_conf['instance'])\n        estimator_conf['instance'].fit(X_train, y_train)\n        gc.collect()\n        a, b = benchmark_estimator(estimator_conf['instance'], X_test)\n        stats[estimator_conf['name']] = {'atomic': a, 'bulk': b}\n\n    cls_names = [estimator_conf['name'] for estimator_conf in configuration[\n        'estimators']]\n    runtimes = [1e6 * stats[clf_name]['atomic'] for clf_name in cls_names]\n    boxplot_runtimes(runtimes, 'atomic', configuration)\n    runtimes = [1e6 * stats[clf_name]['bulk'] for clf_name in cls_names]\n    boxplot_runtimes(runtimes, 'bulk (%d)' % configuration['n_test'],\n                     configuration)\n\n\ndef n_feature_influence(estimators, n_train, n_test, n_features, percentile):\n    \"\"\"\n    Estimate influence of the number of features on prediction time.\n\n    Parameters\n    ----------\n\n    estimators : dict of (name (str), estimator) to benchmark\n    n_train : nber of training instances (int)\n    n_test : nber of testing instances (int)\n    n_features : list of feature-space dimensionality to test (int)\n    percentile : percentile at which to measure the speed (int [0-100])\n\n    Returns:\n    --------\n\n    percentiles : dict(estimator_name,\n                       dict(n_features, percentile_perf_in_us))\n\n    \"\"\"\n    percentiles = defaultdict(defaultdict)\n    for n in n_features:\n        print(\"benchmarking with %d features\" % n)\n        X_train, y_train, X_test, y_test = generate_dataset(n_train, n_test, n)\n        for cls_name, estimator in estimators.items():\n            estimator.fit(X_train, y_train)\n            gc.collect()\n            runtimes = bulk_benchmark_estimator(estimator, X_test, 30, False)\n            percentiles[cls_name][n] = 1e6 * scoreatpercentile(runtimes,\n                                                               percentile)\n    return percentiles\n\n\ndef plot_n_features_influence(percentiles, percentile):\n    fig, ax1 = plt.subplots(figsize=(10, 6))\n    colors = ['r', 'g', 'b']\n    for i, cls_name in enumerate(percentiles.keys()):\n        x = np.array(sorted([n for n in percentiles[cls_name].keys()]))\n        y = np.array([percentiles[cls_name][n] for n in x])\n        plt.plot(x, y, color=colors[i], )\n    ax1.yaxis.grid(True, linestyle='-', which='major', color='lightgrey',\n                   alpha=0.5)\n    ax1.set_axisbelow(True)\n    ax1.set_title('Evolution of Prediction Time with #Features')\n    ax1.set_xlabel('#Features')\n    ax1.set_ylabel('Prediction Time at %d%%-ile (us)' % percentile)\n    plt.show()\n\n\ndef benchmark_throughputs(configuration, duration_secs=0.1):\n    \"\"\"benchmark throughput for different estimators.\"\"\"\n    X_train, y_train, X_test, y_test = generate_dataset(\n        configuration['n_train'], configuration['n_test'],\n        configuration['n_features'])\n    throughputs = dict()\n    for estimator_config in configuration['estimators']:\n        estimator_config['instance'].fit(X_train, y_train)\n        start_time = time.time()\n        n_predictions = 0\n        while (time.time() - start_time) < duration_secs:\n            estimator_config['instance'].predict(X_test[[0]])\n            n_predictions += 1\n        throughputs[estimator_config['name']] = n_predictions / duration_secs\n    return throughputs\n\n\ndef plot_benchmark_throughput(throughputs, configuration):\n    fig, ax = plt.subplots(figsize=(10, 6))\n    colors = ['r', 'g', 'b']\n    cls_infos = ['%s\\n(%d %s)' % (estimator_conf['name'],\n                                  estimator_conf['complexity_computer'](\n                                      estimator_conf['instance']),\n                                  estimator_conf['complexity_label']) for\n                 estimator_conf in configuration['estimators']]\n    cls_values = [throughputs[estimator_conf['name']] for estimator_conf in\n                  configuration['estimators']]\n    plt.bar(range(len(throughputs)), cls_values, width=0.5, color=colors)\n    ax.set_xticks(np.linspace(0.25, len(throughputs) - 0.75, len(throughputs)))\n    ax.set_xticklabels(cls_infos, fontsize=10)\n    ymax = max(cls_values) * 1.2\n    ax.set_ylim((0, ymax))\n    ax.set_ylabel('Throughput (predictions/sec)')\n    ax.set_title('Prediction Throughput for different estimators (%d '\n                 'features)' % configuration['n_features'])\n    plt.show()\n\n\n# #############################################################################\n# Main code\n\nstart_time = time.time()\n\n# #############################################################################\n# Benchmark bulk/atomic prediction speed for various regressors\nconfiguration = {\n    'n_train': int(1e3),\n    'n_test': int(1e2),\n    'n_features': int(1e2),\n    'estimators': [\n        {'name': 'Linear Model',\n         'instance': SGDRegressor(penalty='elasticnet', alpha=0.01,\n                                  l1_ratio=0.25, fit_intercept=True,\n                                  tol=1e-4),\n         'complexity_label': 'non-zero coefficients',\n         'complexity_computer': lambda clf: np.count_nonzero(clf.coef_)},\n        {'name': 'RandomForest',\n         'instance': RandomForestRegressor(),\n         'complexity_label': 'estimators',\n         'complexity_computer': lambda clf: clf.n_estimators},\n        {'name': 'SVR',\n         'instance': SVR(kernel='rbf'),\n         'complexity_label': 'support vectors',\n         'complexity_computer': lambda clf: len(clf.support_vectors_)},\n    ]\n}\nbenchmark(configuration)\n\n# benchmark n_features influence on prediction speed\npercentile = 90\npercentiles = n_feature_influence({'ridge': Ridge()},\n                                  configuration['n_train'],\n                                  configuration['n_test'],\n                                  [100, 250, 500], percentile)\nplot_n_features_influence(percentiles, percentile)\n\n# benchmark throughput\nthroughputs = benchmark_throughputs(configuration)\nplot_benchmark_throughput(throughputs, configuration)\n\nstop_time = time.time()\nprint(\"example run in %.2fs\" % (stop_time - start_time))"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.5"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}