<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>Getting started &mdash; mlens 0.2.3 documentation</title>
  

  
  
  
  

  

  
  
    

  

  
  
    <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  

  
    <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Lato" type="text/css" />
  
    <link rel="stylesheet" href="../_static/css/pygments.css" type="text/css" />
  
    <link rel="stylesheet" href="../_static/css/mlens-theme.css" type="text/css" />
  
    <link rel="stylesheet" href="../_static/css/sphinx-glr.css" type="text/css" />
  

  
        <link rel="index" title="Index"
              href="../genindex.html"/>
        <link rel="search" title="Search" href="../search.html"/>
    <link rel="top" title="mlens 0.2.3 documentation" href="../index.html"/>
        <link rel="next" title="Advanced features tutorial" href="advanced.html"/>
        <link rel="prev" title="Install" href="../start/install.html"/> 

  
  <script src="../_static/js/modernizr.min.js"></script>

</head>

<body class="wy-body-for-nav" role="document">

   
  <div class="wy-grid-for-nav">

    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search">
          

          
            <a href="../index.html">
          

          
            
            <img src="../_static/logo.png" class="logo" />
          
          </a>

          
            
            
              <div class="version">
                0.2.3
              </div>
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption"><span class="caption-text">Installation</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../start/install.html">Install</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../start/install.html#bleeding-edge">Bleeding edge</a></li>
<li class="toctree-l2"><a class="reference internal" href="../start/install.html#dependencies">Dependencies</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../start/install.html#test-build">Test build</a></li>
</ul>
<p class="caption"><span class="caption-text">High-level API</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="current reference internal" href="#">Getting started</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#preliminaries">Preliminaries</a></li>
<li class="toctree-l2"><a class="reference internal" href="#ensemble-guide">Ensemble guide</a><ul>
<li class="toctree-l3"><a class="reference internal" href="#building-an-ensemble">Building an ensemble</a></li>
<li class="toctree-l3"><a class="reference internal" href="#multi-layer-ensembles">Multi-layer ensembles</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="#model-selection-guide">Model selection guide</a><ul>
<li class="toctree-l3"><a class="reference internal" href="#the-scoring-function">The scoring function</a></li>
<li class="toctree-l3"><a class="reference internal" href="#a-simple-evaluation">A simple evaluation</a></li>
<li class="toctree-l3"><a class="reference internal" href="#preprocessing">Preprocessing</a></li>
<li class="toctree-l3"><a class="reference internal" href="#model-selection-across-preprocessing-pipelines">Model Selection across preprocessing pipelines</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="#visualization-guide">Visualization guide</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="advanced.html">Advanced features tutorial</a><ul>
<li class="toctree-l2"><a class="reference internal" href="advanced.html#propagating-input-features">Propagating input features</a></li>
<li class="toctree-l2"><a class="reference internal" href="advanced.html#probabilistic-ensemble-learning">Probabilistic ensemble learning</a></li>
<li class="toctree-l2"><a class="reference internal" href="advanced.html#advanced-subsemble-techniques">Advanced Subsemble techniques</a></li>
<li class="toctree-l2"><a class="reference internal" href="advanced.html#general-multi-layer-ensemble-learning">General multi-layer ensemble learning</a></li>
<li class="toctree-l2"><a class="reference internal" href="advanced.html#passing-file-paths-as-data-input">Passing file paths as data input</a></li>
<li class="toctree-l2"><a class="reference internal" href="advanced.html#ensemble-model-selection">Ensemble model selection</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../start/ensembles.html">Ready-made ensemble classes</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../start/ensembles.html#super-learner">Super Learner</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../start/ensembles.html#references">References</a></li>
<li class="toctree-l3"><a class="reference internal" href="../start/ensembles.html#notes">Notes</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../start/ensembles.html#subsemble">Subsemble</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../start/ensembles.html#id8">References</a></li>
<li class="toctree-l3"><a class="reference internal" href="../start/ensembles.html#id10">Notes</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../start/ensembles.html#blend-ensemble">Blend Ensemble</a></li>
<li class="toctree-l2"><a class="reference internal" href="../start/ensembles.html#temporal-ensemble">Temporal Ensemble</a></li>
<li class="toctree-l2"><a class="reference internal" href="../start/ensembles.html#sequential-ensemble">Sequential Ensemble</a></li>
</ul>
</li>
</ul>
<p class="caption"><span class="caption-text">Mechanics</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../overview.html">Overview</a></li>
<li class="toctree-l1"><a class="reference internal" href="learner.html">Learner Mechanics</a><ul>
<li class="toctree-l2"><a class="reference internal" href="learner.html#the-learner-api">The Learner API</a><ul>
<li class="toctree-l3"><a class="reference internal" href="learner.html#basics">Basics</a></li>
<li class="toctree-l3"><a class="reference internal" href="learner.html#partitioning">Partitioning</a></li>
<li class="toctree-l3"><a class="reference internal" href="learner.html#preprocessing">Preprocessing</a></li>
<li class="toctree-l3"><a class="reference internal" href="learner.html#parallel-estimation">Parallel estimation</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="layer.html">Layer Mechanics</a><ul>
<li class="toctree-l2"><a class="reference internal" href="layer.html#the-layer-api">The Layer API</a><ul>
<li class="toctree-l3"><a class="reference internal" href="layer.html#basics">Basics</a></li>
<li class="toctree-l3"><a class="reference internal" href="layer.html#multitasking">Multitasking</a></li>
<li class="toctree-l3"><a class="reference internal" href="layer.html#layer-features">Layer features</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="parallel.html">Parallel Mechanics</a><ul>
<li class="toctree-l2"><a class="reference internal" href="parallel.html#parallelprocessing-api">ParallelProcessing API</a><ul>
<li class="toctree-l3"><a class="reference internal" href="parallel.html#stacking-a-set-of-parallel-jobs">Stacking a set of parallel jobs</a></li>
<li class="toctree-l3"><a class="reference internal" href="parallel.html#manual-initialization-and-processing">Manual initialization and processing</a></li>
<li class="toctree-l3"><a class="reference internal" href="parallel.html#memory-management">Memory management</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="sequential.html">Sequential Mechanics</a></li>
</ul>
<p class="caption"><span class="caption-text">Details</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../benchmarks/memory.html">Memory consumption</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../benchmarks/memory.html#memory-mapping">Memory mapping</a></li>
<li class="toctree-l2"><a class="reference internal" href="../benchmarks/memory.html#ml-ensemble-memory-profiling">ML-Ensemble memory profiling</a></li>
<li class="toctree-l2"><a class="reference internal" href="../benchmarks/memory.html#memory-performance-benchmark">Memory performance benchmark</a></li>
<li class="toctree-l2"><a class="reference internal" href="../benchmarks/memory.html#gotcha-s">Gotcha's</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../benchmarks/benchmarks.html">Performance benchmarks</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../benchmarks/benchmarks.html#mnist">MNIST</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../benchmarks/benchmarks.html#benchmark">Benchmark</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../benchmarks/benchmarks.html#the-friedman-regression-problem-1">The Friedman Regression Problem 1</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../benchmarks/benchmarks.html#id5">Benchmark</a></li>
<li class="toctree-l3"><a class="reference internal" href="../benchmarks/benchmarks.html#references">References</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../benchmarks/scaling.html">Scale benchmarks</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../benchmarks/scaling.html#single-process-vs-multi-process">Single process vs multi-process</a></li>
<li class="toctree-l2"><a class="reference internal" href="../benchmarks/scaling.html#ensemble-comparison">Ensemble comparison</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../deep/troubleshooting.html">Troubleshooting</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../deep/troubleshooting.html#bad-interaction-with-third-party-packages">Bad interaction with third-party packages</a></li>
<li class="toctree-l2"><a class="reference internal" href="../deep/troubleshooting.html#array-copying-during-fitting">Array copying during fitting</a></li>
</ul>
</li>
</ul>
<p class="caption"><span class="caption-text">Additional Information</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../misc/license.html">License</a></li>
<li class="toctree-l1"><a class="reference internal" href="../misc/updates.html">Change log</a></li>
</ul>

            
          
        </div>
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
        
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="../index.html">mlens</a>
        
      </nav>


      
      <div class="wy-nav-content">
        <div class="rst-content">
          















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../index.html">Docs</a> &raquo;</li>
        
      <li>Getting started</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
            
            <a href="../_sources/tutorials/start.rst.txt" rel="nofollow"> View page source</a>
          
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="getting-started">
<span id="sphx-glr-tutorials-start-py"></span><span id="id1"></span><h1>Getting started<a class="headerlink" href="#getting-started" title="Permalink to this headline">¶</a></h1>
<p>This tutorial highlights the basics of the
high-level API for ensemble classes, the model selection suite and
features visualization.</p>
<table border="1" class="docutils">
<colgroup>
<col width="36%" />
<col width="64%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Tutorials</th>
<th class="head">Content</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><a class="reference internal" href="#ensemble-guide"><span class="std std-ref">Ensemble guide</span></a></td>
<td>How to build, fit and predict with an ensemble</td>
</tr>
<tr class="row-odd"><td><a class="reference internal" href="#model-selection-guide"><span class="std std-ref">Model selection guide</span></a></td>
<td>How to compare several estimators in one go</td>
</tr>
<tr class="row-even"><td><a class="reference internal" href="#visualization-guide"><span class="std std-ref">Visualization guide</span></a></td>
<td>Plotting functionality</td>
</tr>
</tbody>
</table>
<p>The <a class="reference internal" href="advanced.html#ensemble-tutorial"><span class="std std-ref">advanced high-level API tutorials</span></a> shows how to
leverage advanced features such as probabilistic layers, feature propagation
etc. For tutorials on low-level mechanics, see
<a class="reference internal" href="learner.html#learner-tutorial"><span class="std std-ref">the mechanics guides</span></a>.</p>
<div class="section" id="preliminaries">
<h2>Preliminaries<a class="headerlink" href="#preliminaries" title="Permalink to this headline">¶</a></h2>
<p>We use the following setup throughout:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="kn">as</span> <span class="nn">np</span>
<span class="kn">from</span> <span class="nn">pandas</span> <span class="kn">import</span> <span class="n">DataFrame</span>
<span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">accuracy_score</span>
<span class="kn">from</span> <span class="nn">sklearn.datasets</span> <span class="kn">import</span> <span class="n">load_iris</span>

<span class="n">seed</span> <span class="o">=</span> <span class="mi">2017</span>
<span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">seed</span><span class="p">(</span><span class="n">seed</span><span class="p">)</span>

<span class="n">data</span> <span class="o">=</span> <span class="n">load_iris</span><span class="p">()</span>
<span class="n">idx</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">permutation</span><span class="p">(</span><span class="mi">150</span><span class="p">)</span>
<span class="n">X</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="n">idx</span><span class="p">]</span>
<span class="n">y</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">target</span><span class="p">[</span><span class="n">idx</span><span class="p">]</span>
</pre></div>
</div>
</div>
<div class="section" id="ensemble-guide">
<span id="id2"></span><h2>Ensemble guide<a class="headerlink" href="#ensemble-guide" title="Permalink to this headline">¶</a></h2>
<div class="section" id="building-an-ensemble">
<h3>Building an ensemble<a class="headerlink" href="#building-an-ensemble" title="Permalink to this headline">¶</a></h3>
<p>Instantiating a fully specified ensemble is straightforward and requires
three steps: first create the instance, second add the intermediate layers,
and finally the meta estimator.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">mlens.ensemble</span> <span class="kn">import</span> <span class="n">SuperLearner</span>
<span class="kn">from</span> <span class="nn">sklearn.linear_model</span> <span class="kn">import</span> <span class="n">LogisticRegression</span>
<span class="kn">from</span> <span class="nn">sklearn.ensemble</span> <span class="kn">import</span> <span class="n">RandomForestClassifier</span>
<span class="kn">from</span> <span class="nn">sklearn.svm</span> <span class="kn">import</span> <span class="n">SVC</span>

<span class="c1"># --- Build ---</span>
<span class="c1"># Passing a scoring function will create cv scores during fitting</span>
<span class="c1"># the scorer should be a simple function accepting to vectors and returning a scalar</span>
<span class="n">ensemble</span> <span class="o">=</span> <span class="n">SuperLearner</span><span class="p">(</span><span class="n">scorer</span><span class="o">=</span><span class="n">accuracy_score</span><span class="p">,</span> <span class="n">random_state</span><span class="o">=</span><span class="n">seed</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>

<span class="c1"># Build the first layer</span>
<span class="n">ensemble</span><span class="o">.</span><span class="n">add</span><span class="p">([</span><span class="n">RandomForestClassifier</span><span class="p">(</span><span class="n">random_state</span><span class="o">=</span><span class="n">seed</span><span class="p">),</span> <span class="n">SVC</span><span class="p">()])</span>

<span class="c1"># Attach the final meta estimator</span>
<span class="n">ensemble</span><span class="o">.</span><span class="n">add_meta</span><span class="p">(</span><span class="n">LogisticRegression</span><span class="p">())</span>

<span class="c1"># --- Use ---</span>

<span class="c1"># Fit ensemble</span>
<span class="n">ensemble</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">[:</span><span class="mi">75</span><span class="p">],</span> <span class="n">y</span><span class="p">[:</span><span class="mi">75</span><span class="p">])</span>

<span class="c1"># Predict</span>
<span class="n">preds</span> <span class="o">=</span> <span class="n">ensemble</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X</span><span class="p">[</span><span class="mi">75</span><span class="p">:])</span>
</pre></div>
</div>
<p class="sphx-glr-script-out">Out:</p>
<div class="sphx-glr-script-out highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">Fitting</span> <span class="mi">2</span> <span class="n">layers</span>
<span class="n">Processing</span> <span class="n">layer</span><span class="o">-</span><span class="mi">1</span>             <span class="n">done</span> <span class="o">|</span> <span class="mi">00</span><span class="p">:</span><span class="mi">00</span><span class="p">:</span><span class="mi">00</span>
<span class="n">Processing</span> <span class="n">layer</span><span class="o">-</span><span class="mi">2</span>             <span class="n">done</span> <span class="o">|</span> <span class="mi">00</span><span class="p">:</span><span class="mi">00</span><span class="p">:</span><span class="mi">00</span>
<span class="n">Fit</span> <span class="n">complete</span>                        <span class="o">|</span> <span class="mi">00</span><span class="p">:</span><span class="mi">00</span><span class="p">:</span><span class="mi">00</span>

<span class="n">Predicting</span> <span class="mi">2</span> <span class="n">layers</span>
<span class="n">Processing</span> <span class="n">layer</span><span class="o">-</span><span class="mi">1</span>             <span class="n">done</span> <span class="o">|</span> <span class="mi">00</span><span class="p">:</span><span class="mi">00</span><span class="p">:</span><span class="mi">00</span>
<span class="n">Processing</span> <span class="n">layer</span><span class="o">-</span><span class="mi">2</span>             <span class="n">done</span> <span class="o">|</span> <span class="mi">00</span><span class="p">:</span><span class="mi">00</span><span class="p">:</span><span class="mi">00</span>
<span class="n">Predict</span> <span class="n">complete</span>                    <span class="o">|</span> <span class="mi">00</span><span class="p">:</span><span class="mi">00</span><span class="p">:</span><span class="mi">00</span>
</pre></div>
</div>
<p>To check the performance of estimator in the layers, call the <code class="docutils literal notranslate"><span class="pre">data</span></code>
attribute. The attribute can be wrapped in a <a class="reference external" href="http://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html#pandas.DataFrame" title="(in pandas v0.24.1)"><code class="xref py py-class docutils literal notranslate"><span class="pre">pandas.DataFrame</span></code></a>,
but prints in a tabular format as is.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">print</span><span class="p">(</span><span class="s2">&quot;Fit data:</span><span class="se">\n</span><span class="si">%r</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">ensemble</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
</pre></div>
</div>
<p class="sphx-glr-script-out">Out:</p>
<div class="sphx-glr-script-out highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">Fit</span> <span class="n">data</span><span class="p">:</span>
                                   <span class="n">score</span><span class="o">-</span><span class="n">m</span>  <span class="n">score</span><span class="o">-</span><span class="n">s</span>  <span class="n">ft</span><span class="o">-</span><span class="n">m</span>  <span class="n">ft</span><span class="o">-</span><span class="n">s</span>  <span class="n">pt</span><span class="o">-</span><span class="n">m</span>  <span class="n">pt</span><span class="o">-</span><span class="n">s</span>
<span class="n">layer</span><span class="o">-</span><span class="mi">1</span>  <span class="n">randomforestclassifier</span>       <span class="mf">0.84</span>     <span class="mf">0.06</span>  <span class="mf">0.05</span>  <span class="mf">0.00</span>  <span class="mf">0.00</span>  <span class="mf">0.00</span>
<span class="n">layer</span><span class="o">-</span><span class="mi">1</span>  <span class="n">svc</span>                          <span class="mf">0.89</span>     <span class="mf">0.05</span>  <span class="mf">0.01</span>  <span class="mf">0.00</span>  <span class="mf">0.00</span>  <span class="mf">0.00</span>
</pre></div>
</div>
<p>To round off, let's see how the ensemble as a whole fared.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">print</span><span class="p">(</span><span class="s2">&quot;Prediction score: </span><span class="si">%.3f</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">accuracy_score</span><span class="p">(</span><span class="n">preds</span><span class="p">,</span> <span class="n">y</span><span class="p">[</span><span class="mi">75</span><span class="p">:]))</span>
</pre></div>
</div>
<p class="sphx-glr-script-out">Out:</p>
<div class="sphx-glr-script-out highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">Prediction</span> <span class="n">score</span><span class="p">:</span> <span class="mf">0.960</span>
</pre></div>
</div>
</div>
<div class="section" id="multi-layer-ensembles">
<h3>Multi-layer ensembles<a class="headerlink" href="#multi-layer-ensembles" title="Permalink to this headline">¶</a></h3>
<p>With each call to the <code class="docutils literal notranslate"><span class="pre">add</span></code> method, another layer is added to the ensemble.
Note that all ensembles are <em>sequential</em> in the order layers are added. For
instance, in the above example, we could add a second layer as follows.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">ensemble</span> <span class="o">=</span> <span class="n">SuperLearner</span><span class="p">(</span><span class="n">scorer</span><span class="o">=</span><span class="n">accuracy_score</span><span class="p">,</span> <span class="n">random_state</span><span class="o">=</span><span class="n">seed</span><span class="p">)</span>

<span class="c1"># Build the first layer</span>
<span class="n">ensemble</span><span class="o">.</span><span class="n">add</span><span class="p">([</span><span class="n">RandomForestClassifier</span><span class="p">(</span><span class="n">random_state</span><span class="o">=</span><span class="n">seed</span><span class="p">),</span> <span class="n">LogisticRegression</span><span class="p">()])</span>

<span class="c1"># Build the second layer</span>
<span class="n">ensemble</span><span class="o">.</span><span class="n">add</span><span class="p">([</span><span class="n">LogisticRegression</span><span class="p">(),</span> <span class="n">SVC</span><span class="p">()])</span>

<span class="c1"># Attach the final meta estimator</span>
<span class="n">ensemble</span><span class="o">.</span><span class="n">add_meta</span><span class="p">(</span><span class="n">SVC</span><span class="p">())</span>
</pre></div>
</div>
<p>We now fit this ensemble in the same manner as before:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">ensemble</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">[:</span><span class="mi">75</span><span class="p">],</span> <span class="n">y</span><span class="p">[:</span><span class="mi">75</span><span class="p">])</span>
<span class="n">preds</span> <span class="o">=</span> <span class="n">ensemble</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X</span><span class="p">[</span><span class="mi">75</span><span class="p">:])</span>
<span class="k">print</span><span class="p">(</span><span class="s2">&quot;Fit data:</span><span class="se">\n</span><span class="si">%r</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">ensemble</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
</pre></div>
</div>
<p class="sphx-glr-script-out">Out:</p>
<div class="sphx-glr-script-out highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">Fit</span> <span class="n">data</span><span class="p">:</span>
                                   <span class="n">score</span><span class="o">-</span><span class="n">m</span>  <span class="n">score</span><span class="o">-</span><span class="n">s</span>  <span class="n">ft</span><span class="o">-</span><span class="n">m</span>  <span class="n">ft</span><span class="o">-</span><span class="n">s</span>  <span class="n">pt</span><span class="o">-</span><span class="n">m</span>  <span class="n">pt</span><span class="o">-</span><span class="n">s</span>
<span class="n">layer</span><span class="o">-</span><span class="mi">1</span>  <span class="n">logisticregression</span>           <span class="mf">0.75</span>     <span class="mf">0.14</span>  <span class="mf">0.01</span>  <span class="mf">0.01</span>  <span class="mf">0.00</span>  <span class="mf">0.00</span>
<span class="n">layer</span><span class="o">-</span><span class="mi">1</span>  <span class="n">randomforestclassifier</span>       <span class="mf">0.84</span>     <span class="mf">0.06</span>  <span class="mf">0.03</span>  <span class="mf">0.00</span>  <span class="mf">0.00</span>  <span class="mf">0.00</span>
<span class="n">layer</span><span class="o">-</span><span class="mi">2</span>  <span class="n">logisticregression</span>           <span class="mf">0.67</span>     <span class="mf">0.12</span>  <span class="mf">0.01</span>  <span class="mf">0.00</span>  <span class="mf">0.00</span>  <span class="mf">0.00</span>
<span class="n">layer</span><span class="o">-</span><span class="mi">2</span>  <span class="n">svc</span>                          <span class="mf">0.89</span>     <span class="mf">0.00</span>  <span class="mf">0.00</span>  <span class="mf">0.00</span>  <span class="mf">0.00</span>  <span class="mf">0.00</span>
</pre></div>
</div>
</div>
</div>
<div class="section" id="model-selection-guide">
<span id="id3"></span><h2>Model selection guide<a class="headerlink" href="#model-selection-guide" title="Permalink to this headline">¶</a></h2>
<p>The work horse class is the <a class="reference external" href="http://ml-ensemble.com/docs/model_selection.html#mlens.model_selection.Evaluator" title="(in mlens v0.2.2)"><code class="xref py py-class docutils literal notranslate"><span class="pre">Evaluator</span></code></a>, which allows you to
grid search several models in one go across several preprocessing pipelines.
The evaluator class pre-fits transformers, thus avoiding fitting the same
preprocessing pipelines on the same data repeatedly. Also, by fitting all
models over all parameter draws in one operation, parallelization is
maximized.
The following example evaluates a <a class="reference external" href="https://en.wikipedia.org/wiki/Naive_Bayes_classifier">Naive Bayes</a> estimator and a
<a class="reference external" href="https://en.wikipedia.org/wiki/K-nearest_neighbors_algorithm">K-Nearest-Neighbor</a> estimator under three different preprocessing scenarios:
no preprocessing, standard scaling, and subset selection.
In the latter case, preprocessing is constituted by selecting a subset of
features.</p>
<div class="section" id="the-scoring-function">
<h3>The scoring function<a class="headerlink" href="#the-scoring-function" title="Permalink to this headline">¶</a></h3>
<p>An important note is that the scoring function must be wrapped by
<a class="reference external" href="http://ml-ensemble.com/docs/metrics.html#mlens.metrics.make_scorer" title="(in mlens v0.2.2)"><code class="xref py py-func docutils literal notranslate"><span class="pre">make_scorer()</span></code></a>, to ensure all scoring functions behave similarly regardless
of whether they measure accuracy or errors. To wrap a function, simple do:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">mlens.metrics</span> <span class="kn">import</span> <span class="n">make_scorer</span>
<span class="n">accuracy_scorer</span> <span class="o">=</span> <span class="n">make_scorer</span><span class="p">(</span><span class="n">accuracy_score</span><span class="p">,</span> <span class="n">greater_is_better</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
</pre></div>
</div>
<p>The <code class="docutils literal notranslate"><span class="pre">make_scorer</span></code> wrapper
is a copy of the Scikit-learn's <a class="reference external" href="https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html#sklearn.metrics.make_scorer" title="(in scikit-learn v0.20.2)"><code class="xref py py-func docutils literal notranslate"><span class="pre">sklearn.metrics.make_scorer()</span></code></a>, and you
can import the Scikit-learn version as well.
Note however that to pickle the <a class="reference external" href="http://ml-ensemble.com/docs/model_selection.html#mlens.model_selection.Evaluator" title="(in mlens v0.2.2)"><code class="xref py py-class docutils literal notranslate"><span class="pre">Evaluator</span></code></a>, you <strong>must</strong> import
<code class="docutils literal notranslate"><span class="pre">make_scorer</span></code> from <code class="docutils literal notranslate"><span class="pre">mlens</span></code>.</p>
</div>
<div class="section" id="a-simple-evaluation">
<h3>A simple evaluation<a class="headerlink" href="#a-simple-evaluation" title="Permalink to this headline">¶</a></h3>
<p>Before throwing preprocessing into the mix, let's see how to evaluate a set of
estimator. First, we need a list of estimator and a dictionary of parameter
distributions that maps to each estimator. The estimators should be put in a
list, either as is or as a named tuple (<code class="docutils literal notranslate"><span class="pre">(name,</span> <span class="pre">est)</span></code>). If you don't name
the estimator, the <a class="reference external" href="http://ml-ensemble.com/docs/model_selection.html#mlens.model_selection.Evaluator" title="(in mlens v0.2.2)"><code class="xref py py-class docutils literal notranslate"><span class="pre">Evaluator</span></code></a> will automatically name the model as the
class name in lower case. This name must be the key in the parameter
dictionary.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">mlens.model_selection</span> <span class="kn">import</span> <span class="n">Evaluator</span>
<span class="kn">from</span> <span class="nn">sklearn.naive_bayes</span> <span class="kn">import</span> <span class="n">GaussianNB</span>
<span class="kn">from</span> <span class="nn">sklearn.neighbors</span> <span class="kn">import</span> <span class="n">KNeighborsClassifier</span>
<span class="kn">from</span> <span class="nn">scipy.stats</span> <span class="kn">import</span> <span class="n">randint</span>

<span class="c1"># Here we name the estimators ourselves</span>
<span class="n">ests</span> <span class="o">=</span> <span class="p">[(</span><span class="s1">&#39;gnb&#39;</span><span class="p">,</span> <span class="n">GaussianNB</span><span class="p">()),</span> <span class="p">(</span><span class="s1">&#39;knn&#39;</span><span class="p">,</span> <span class="n">KNeighborsClassifier</span><span class="p">())]</span>

<span class="c1"># Now we map parameters to these</span>
<span class="c1"># The gnb doesn&#39;t have any parameters so we can skip it</span>
<span class="n">pars</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;n_neighbors&#39;</span><span class="p">:</span> <span class="n">randint</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">20</span><span class="p">)}</span>
<span class="n">params</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;knn&#39;</span><span class="p">:</span> <span class="n">pars</span><span class="p">}</span>
</pre></div>
</div>
<p>We can now run an evaluation over these estimators and parameter distributions
by calling the <code class="docutils literal notranslate"><span class="pre">fit</span></code> method.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">evaluator</span> <span class="o">=</span> <span class="n">Evaluator</span><span class="p">(</span><span class="n">accuracy_scorer</span><span class="p">,</span> <span class="n">cv</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">random_state</span><span class="o">=</span><span class="n">seed</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
<span class="n">evaluator</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">ests</span><span class="p">,</span> <span class="n">params</span><span class="p">,</span> <span class="n">n_iter</span><span class="o">=</span><span class="mi">10</span><span class="p">)</span>
</pre></div>
</div>
<p class="sphx-glr-script-out">Out:</p>
<div class="sphx-glr-script-out highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">Launching</span> <span class="n">job</span>
<span class="n">Job</span>           <span class="n">done</span> <span class="o">|</span> <span class="mi">00</span><span class="p">:</span><span class="mi">00</span><span class="p">:</span><span class="mi">00</span>
</pre></div>
</div>
<p>The full history of the evaluation can be found in <code class="docutils literal notranslate"><span class="pre">cv_results</span></code>. To compare
models with their best parameters, we can pass the <code class="docutils literal notranslate"><span class="pre">results</span></code> attribute to
a <a class="reference external" href="http://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html#pandas.DataFrame" title="(in pandas v0.24.1)"><code class="xref py py-obj docutils literal notranslate"><span class="pre">pandas.DataFrame</span></code></a> or print it as a table. We use <code class="docutils literal notranslate"><span class="pre">m</span></code> to denote
mean values and <code class="docutils literal notranslate"><span class="pre">s</span></code> to denote standard deviation across folds for brevity.
Note that the timed prediction is for the training set, for comparability with
training time.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">print</span><span class="p">(</span><span class="s2">&quot;Score comparison with best params founds:</span><span class="se">\n\n</span><span class="si">%r</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">evaluator</span><span class="o">.</span><span class="n">results</span><span class="p">)</span>
</pre></div>
</div>
<p class="sphx-glr-script-out">Out:</p>
<div class="sphx-glr-script-out highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">Score</span> <span class="n">comparison</span> <span class="k">with</span> <span class="n">best</span> <span class="n">params</span> <span class="n">founds</span><span class="p">:</span>

       <span class="n">test_score</span><span class="o">-</span><span class="n">m</span>  <span class="n">test_score</span><span class="o">-</span><span class="n">s</span>  <span class="n">train_score</span><span class="o">-</span><span class="n">m</span>  <span class="n">train_score</span><span class="o">-</span><span class="n">s</span>  <span class="n">fit_time</span><span class="o">-</span><span class="n">m</span>  <span class="n">fit_time</span><span class="o">-</span><span class="n">s</span>  <span class="n">pred_time</span><span class="o">-</span><span class="n">m</span>  <span class="n">pred_time</span><span class="o">-</span><span class="n">s</span>               <span class="n">params</span>
<span class="n">gnb</span>           <span class="mf">0.960</span>         <span class="mf">0.033</span>          <span class="mf">0.957</span>          <span class="mf">0.006</span>       <span class="mf">0.006</span>       <span class="mf">0.002</span>        <span class="mf">0.004</span>        <span class="mf">0.002</span>
<span class="n">knn</span>           <span class="mf">0.967</span>         <span class="mf">0.033</span>          <span class="mf">0.980</span>          <span class="mf">0.005</span>       <span class="mf">0.002</span>       <span class="mf">0.001</span>        <span class="mf">0.010</span>        <span class="mf">0.004</span>  <span class="p">{</span><span class="s1">&#39;n_neighbors&#39;</span><span class="p">:</span> <span class="mi">15</span><span class="p">}</span>
</pre></div>
</div>
</div>
<div class="section" id="preprocessing">
<h3>Preprocessing<a class="headerlink" href="#preprocessing" title="Permalink to this headline">¶</a></h3>
<p>Next, suppose we want to compare the models across a set of preprocessing pipelines.
To do this, we first need to specify a dictionary of preprocessing pipelines to
run through. Each entry in the dictionary should be a list of transformers to apply sequentially.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">mlens.preprocessing</span> <span class="kn">import</span> <span class="n">Subset</span>
<span class="kn">from</span> <span class="nn">sklearn.preprocessing</span> <span class="kn">import</span> <span class="n">StandardScaler</span>

<span class="c1"># Map preprocessing cases through a dictionary</span>
<span class="n">preprocess_cases</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;none&#39;</span><span class="p">:</span> <span class="p">[],</span>
                    <span class="s1">&#39;sc&#39;</span><span class="p">:</span> <span class="p">[</span><span class="n">StandardScaler</span><span class="p">()],</span>
                    <span class="s1">&#39;sub&#39;</span><span class="p">:</span> <span class="p">[</span><span class="n">Subset</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">])]</span>
                    <span class="p">}</span>
</pre></div>
</div>
<p>The <code class="docutils literal notranslate"><span class="pre">fit</span></code> methods determines automatically whether there is any preprocessing
or any estimator jobs to run, so all we need to do is specify the arguments
we want to be processed. If a previous preprocessing job was fitted, those
pipelines are stored and will be used for subsequent estimator fits.</p>
<p>This can be helpful if the preprocessing is time-consuming, for instance if
the preprocessing pipeline is an ensemble itself. All ensembles implement
a <code class="docutils literal notranslate"><span class="pre">transform</span></code> method that, in contrast to the <code class="docutils literal notranslate"><span class="pre">predict</span></code> method, regenerates
the predictions made during the <code class="docutils literal notranslate"><span class="pre">fit``call.</span> <span class="pre">More</span> <span class="pre">precisely,</span> <span class="pre">the</span> <span class="pre">``transform</span></code>
method uses the estimators fitted with cross-validation to construct predictions,
whereas the <code class="docutils literal notranslate"><span class="pre">predict</span></code> method uses the final estimators fitted on all data.
This allows us use ensembles as preprocessing steps that mimicks how that ensemble
would produce predictions for a subsequent meta learner or layer. Since fitting
large ensembles is highly time-consuming, fixing the lower layers as preprocessing
input is highly valuable for tuning the higher layers and / or the final meta learner.
See the <a class="reference internal" href="advanced.html#model-selection-tutorial"><span class="std std-ref">Ensemble model selection</span></a> tutorial for
an example. To fit only the preprocessing pipelines, we omit any estimators in the
<code class="docutils literal notranslate"><span class="pre">fit</span></code> call.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">evaluator</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">preprocessing</span><span class="o">=</span><span class="n">preprocess_cases</span><span class="p">)</span>
</pre></div>
</div>
<p class="sphx-glr-script-out">Out:</p>
<div class="sphx-glr-script-out highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">Launching</span> <span class="n">job</span>
<span class="n">Job</span>           <span class="n">done</span> <span class="o">|</span> <span class="mi">00</span><span class="p">:</span><span class="mi">00</span><span class="p">:</span><span class="mi">00</span>
</pre></div>
</div>
</div>
<div class="section" id="model-selection-across-preprocessing-pipelines">
<h3>Model Selection across preprocessing pipelines<a class="headerlink" href="#model-selection-across-preprocessing-pipelines" title="Permalink to this headline">¶</a></h3>
<p>To evaluate the same set of estimators across all pipelines with the same
parameter distributions, there is no need to take any heed of the preprocessing
pipeline, just carry on as in the simple case:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">evaluator</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">ests</span><span class="p">,</span> <span class="n">params</span><span class="p">,</span> <span class="n">n_iter</span><span class="o">=</span><span class="mi">10</span><span class="p">)</span>
<span class="k">print</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">Comparison across preprocessing pipelines:</span><span class="se">\n\n</span><span class="si">%r</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">evaluator</span><span class="o">.</span><span class="n">results</span><span class="p">)</span>
</pre></div>
</div>
<p class="sphx-glr-script-out">Out:</p>
<div class="sphx-glr-script-out highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">Launching</span> <span class="n">job</span>
<span class="n">Job</span>           <span class="n">done</span> <span class="o">|</span> <span class="mi">00</span><span class="p">:</span><span class="mi">00</span><span class="p">:</span><span class="mi">01</span>

<span class="n">Comparison</span> <span class="n">across</span> <span class="n">preprocessing</span> <span class="n">pipelines</span><span class="p">:</span>

             <span class="n">test_score</span><span class="o">-</span><span class="n">m</span>  <span class="n">test_score</span><span class="o">-</span><span class="n">s</span>  <span class="n">train_score</span><span class="o">-</span><span class="n">m</span>  <span class="n">train_score</span><span class="o">-</span><span class="n">s</span>  <span class="n">fit_time</span><span class="o">-</span><span class="n">m</span>  <span class="n">fit_time</span><span class="o">-</span><span class="n">s</span>  <span class="n">pred_time</span><span class="o">-</span><span class="n">m</span>  <span class="n">pred_time</span><span class="o">-</span><span class="n">s</span>               <span class="n">params</span>
<span class="n">none</span>  <span class="n">gnb</span>           <span class="mf">0.960</span>         <span class="mf">0.033</span>          <span class="mf">0.957</span>          <span class="mf">0.006</span>       <span class="mf">0.004</span>       <span class="mf">0.001</span>        <span class="mf">0.003</span>        <span class="mf">0.001</span>
<span class="n">none</span>  <span class="n">knn</span>           <span class="mf">0.967</span>         <span class="mf">0.033</span>          <span class="mf">0.980</span>          <span class="mf">0.005</span>       <span class="mf">0.002</span>       <span class="mf">0.001</span>        <span class="mf">0.006</span>        <span class="mf">0.001</span>  <span class="p">{</span><span class="s1">&#39;n_neighbors&#39;</span><span class="p">:</span> <span class="mi">15</span><span class="p">}</span>
<span class="n">sc</span>    <span class="n">gnb</span>           <span class="mf">0.960</span>         <span class="mf">0.033</span>          <span class="mf">0.957</span>          <span class="mf">0.006</span>       <span class="mf">0.004</span>       <span class="mf">0.001</span>        <span class="mf">0.003</span>        <span class="mf">0.001</span>
<span class="n">sc</span>    <span class="n">knn</span>           <span class="mf">0.960</span>         <span class="mf">0.044</span>          <span class="mf">0.965</span>          <span class="mf">0.003</span>       <span class="mf">0.001</span>       <span class="mf">0.001</span>        <span class="mf">0.006</span>        <span class="mf">0.001</span>   <span class="p">{</span><span class="s1">&#39;n_neighbors&#39;</span><span class="p">:</span> <span class="mi">8</span><span class="p">}</span>
<span class="n">sub</span>   <span class="n">gnb</span>           <span class="mf">0.780</span>         <span class="mf">0.133</span>          <span class="mf">0.791</span>          <span class="mf">0.020</span>       <span class="mf">0.004</span>       <span class="mf">0.001</span>        <span class="mf">0.004</span>        <span class="mf">0.001</span>
<span class="n">sub</span>   <span class="n">knn</span>           <span class="mf">0.800</span>         <span class="mf">0.126</span>          <span class="mf">0.837</span>          <span class="mf">0.015</span>       <span class="mf">0.002</span>       <span class="mf">0.001</span>        <span class="mf">0.005</span>        <span class="mf">0.002</span>   <span class="p">{</span><span class="s1">&#39;n_neighbors&#39;</span><span class="p">:</span> <span class="mi">9</span><span class="p">}</span>
</pre></div>
</div>
<p>You can also map different estimators to different preprocessing folds, and
map different parameter distribution to each case. We will map two different
parameter distributions</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">pars_1</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;n_neighbors&#39;</span><span class="p">:</span> <span class="n">randint</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">30</span><span class="p">)}</span>
<span class="n">pars_2</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;n_neighbors&#39;</span><span class="p">:</span> <span class="n">randint</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">10</span><span class="p">)}</span>
<span class="n">params</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;sc.knn&#39;</span><span class="p">:</span> <span class="n">pars_1</span><span class="p">,</span>
          <span class="s1">&#39;none.knn&#39;</span><span class="p">:</span> <span class="n">pars_2</span><span class="p">,</span>
          <span class="s1">&#39;sub.knn&#39;</span><span class="p">:</span> <span class="n">pars_2</span><span class="p">}</span>

<span class="c1"># We can map different estimators to different cases</span>
<span class="n">ests_1</span> <span class="o">=</span> <span class="p">[(</span><span class="s1">&#39;gnb&#39;</span><span class="p">,</span> <span class="n">GaussianNB</span><span class="p">()),</span> <span class="p">(</span><span class="s1">&#39;knn&#39;</span><span class="p">,</span> <span class="n">KNeighborsClassifier</span><span class="p">())]</span>
<span class="n">ests_2</span> <span class="o">=</span> <span class="p">[(</span><span class="s1">&#39;knn&#39;</span><span class="p">,</span> <span class="n">KNeighborsClassifier</span><span class="p">())]</span>
<span class="n">estimators</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;sc&#39;</span><span class="p">:</span> <span class="n">ests_1</span><span class="p">,</span>
              <span class="s1">&#39;none&#39;</span><span class="p">:</span> <span class="n">ests_2</span><span class="p">,</span>
              <span class="s1">&#39;sub&#39;</span><span class="p">:</span> <span class="n">ests_1</span><span class="p">}</span>

<span class="n">evaluator</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">estimators</span><span class="p">,</span> <span class="n">params</span><span class="p">,</span> <span class="n">n_iter</span><span class="o">=</span><span class="mi">10</span><span class="p">)</span>
<span class="k">print</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">Comparison with different parameter dists:</span><span class="se">\n\n</span><span class="si">%r</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">evaluator</span><span class="o">.</span><span class="n">results</span><span class="p">)</span>
</pre></div>
</div>
<p class="sphx-glr-script-out">Out:</p>
<div class="sphx-glr-script-out highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">Launching</span> <span class="n">job</span>
<span class="n">Job</span>           <span class="n">done</span> <span class="o">|</span> <span class="mi">00</span><span class="p">:</span><span class="mi">00</span><span class="p">:</span><span class="mi">01</span>

<span class="n">Comparison</span> <span class="k">with</span> <span class="n">different</span> <span class="n">parameter</span> <span class="n">dists</span><span class="p">:</span>

             <span class="n">test_score</span><span class="o">-</span><span class="n">m</span>  <span class="n">test_score</span><span class="o">-</span><span class="n">s</span>  <span class="n">train_score</span><span class="o">-</span><span class="n">m</span>  <span class="n">train_score</span><span class="o">-</span><span class="n">s</span>  <span class="n">fit_time</span><span class="o">-</span><span class="n">m</span>  <span class="n">fit_time</span><span class="o">-</span><span class="n">s</span>  <span class="n">pred_time</span><span class="o">-</span><span class="n">m</span>  <span class="n">pred_time</span><span class="o">-</span><span class="n">s</span>               <span class="n">params</span>
<span class="n">none</span>  <span class="n">knn</span>           <span class="mf">0.967</span>         <span class="mf">0.045</span>          <span class="mf">0.961</span>          <span class="mf">0.007</span>       <span class="mf">0.002</span>       <span class="mf">0.001</span>        <span class="mf">0.003</span>        <span class="mf">0.001</span>   <span class="p">{</span><span class="s1">&#39;n_neighbors&#39;</span><span class="p">:</span> <span class="mi">3</span><span class="p">}</span>
<span class="n">sc</span>    <span class="n">gnb</span>           <span class="mf">0.960</span>         <span class="mf">0.033</span>          <span class="mf">0.957</span>          <span class="mf">0.006</span>       <span class="mf">0.004</span>       <span class="mf">0.001</span>        <span class="mf">0.002</span>        <span class="mf">0.001</span>
<span class="n">sc</span>    <span class="n">knn</span>           <span class="mf">0.940</span>         <span class="mf">0.055</span>          <span class="mf">0.959</span>          <span class="mf">0.008</span>       <span class="mf">0.002</span>       <span class="mf">0.000</span>        <span class="mf">0.008</span>        <span class="mf">0.001</span>  <span class="p">{</span><span class="s1">&#39;n_neighbors&#39;</span><span class="p">:</span> <span class="mi">22</span><span class="p">}</span>
<span class="n">sub</span>   <span class="n">gnb</span>           <span class="mf">0.780</span>         <span class="mf">0.133</span>          <span class="mf">0.791</span>          <span class="mf">0.020</span>       <span class="mf">0.003</span>       <span class="mf">0.001</span>        <span class="mf">0.003</span>        <span class="mf">0.001</span>
<span class="n">sub</span>   <span class="n">knn</span>           <span class="mf">0.800</span>         <span class="mf">0.126</span>          <span class="mf">0.837</span>          <span class="mf">0.015</span>       <span class="mf">0.002</span>       <span class="mf">0.001</span>        <span class="mf">0.004</span>        <span class="mf">0.002</span>   <span class="p">{</span><span class="s1">&#39;n_neighbors&#39;</span><span class="p">:</span> <span class="mi">9</span><span class="p">}</span>
</pre></div>
</div>
</div>
</div>
<div class="section" id="visualization-guide">
<span id="id4"></span><h2>Visualization guide<a class="headerlink" href="#visualization-guide" title="Permalink to this headline">¶</a></h2>
<p><strong>Explained variance plot</strong></p>
<p>The <code class="xref py py-class docutils literal notranslate"><span class="pre">exp_var_plot</span></code> function
plots the explained variance from mapping a matrix <code class="docutils literal notranslate"><span class="pre">X</span></code> onto a smaller
dimension using a user-supplied transformer, such as the Scikit-learn
<a class="reference external" href="https://scikit-learn.org/stable/modules/generated/sklearn.decomposition.PCA.html#sklearn.decomposition.PCA" title="(in scikit-learn v0.20.2)"><code class="xref py py-class docutils literal notranslate"><span class="pre">sklearn.decomposition.PCA</span></code></a> transformer for
Principal Components Analysis.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="kn">as</span> <span class="nn">plt</span>
<span class="kn">from</span> <span class="nn">mlens.visualization</span> <span class="kn">import</span> <span class="n">exp_var_plot</span>
<span class="kn">from</span> <span class="nn">sklearn.decomposition</span> <span class="kn">import</span> <span class="n">PCA</span>
<span class="n">exp_var_plot</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">PCA</span><span class="p">(),</span> <span class="n">marker</span><span class="o">=</span><span class="s1">&#39;s&#39;</span><span class="p">,</span> <span class="n">where</span><span class="o">=</span><span class="s1">&#39;post&#39;</span><span class="p">)</span>
</pre></div>
</div>
<img alt="../_images/sphx_glr_start_001.png" class="align-center" src="../_images/sphx_glr_start_001.png" />
<p><strong>Principal Components Analysis plot</strong></p>
<p>The <code class="xref py py-class docutils literal notranslate"><span class="pre">pca_plot</span></code> function
plots a PCA analysis or similar if <code class="docutils literal notranslate"><span class="pre">n_components</span></code> is one of <code class="docutils literal notranslate"><span class="pre">[1,</span> <span class="pre">2,</span> <span class="pre">3]</span></code>.
By passing a class labels, the plot shows how well separated different classes
are.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">mlens.visualization</span> <span class="kn">import</span> <span class="n">pca_plot</span>
<span class="kn">from</span> <span class="nn">sklearn.decomposition</span> <span class="kn">import</span> <span class="n">PCA</span>
<span class="n">pca_plot</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">PCA</span><span class="p">(</span><span class="n">n_components</span><span class="o">=</span><span class="mi">2</span><span class="p">),</span> <span class="n">y</span><span class="o">=</span><span class="n">y</span><span class="p">)</span>
</pre></div>
</div>
<img alt="../_images/sphx_glr_start_002.png" class="align-center" src="../_images/sphx_glr_start_002.png" />
<p><strong>Principal Components Comparison plot</strong></p>
<p>The <code class="xref py py-class docutils literal notranslate"><span class="pre">pca_comp_plot</span></code> function
plots a matrix of PCA analyses, one for each combination of
<code class="docutils literal notranslate"><span class="pre">n_components=2,</span> <span class="pre">3</span></code> and <code class="docutils literal notranslate"><span class="pre">kernel='linear',</span> <span class="pre">'rbf'</span></code>.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">mlens.visualization</span> <span class="kn">import</span> <span class="n">pca_comp_plot</span>
<span class="n">pca_comp_plot</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
</pre></div>
</div>
<img alt="../_images/sphx_glr_start_003.png" class="align-center" src="../_images/sphx_glr_start_003.png" />
<p><strong>Correlation matrix plot</strong></p>
<p>The <code class="xref py py-class docutils literal notranslate"><span class="pre">corrmat</span></code> function plots the lower triangle of
a correlation matrix and is adapted the <a class="reference external" href="http://seaborn.pydata.org/examples/many_pairwise_correlations.html">Seaborn</a> correlation matrix.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">mlens.visualization</span> <span class="kn">import</span> <span class="n">corrmat</span>
<span class="c1"># Generate som different predictions to correlate</span>
<span class="n">params</span> <span class="o">=</span> <span class="p">[</span><span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.3</span><span class="p">,</span> <span class="mf">1.0</span><span class="p">,</span> <span class="mf">3.0</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="mi">30</span><span class="p">]</span>
<span class="n">preds</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="mi">150</span><span class="p">,</span> <span class="mi">6</span><span class="p">))</span>
<span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">c</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">params</span><span class="p">):</span>
    <span class="n">preds</span><span class="p">[:,</span> <span class="n">i</span><span class="p">]</span> <span class="o">=</span> <span class="n">LogisticRegression</span><span class="p">(</span><span class="n">C</span><span class="o">=</span><span class="n">c</span><span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>

<span class="n">corr</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="p">(</span><span class="n">preds</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="p">[</span><span class="s1">&#39;C=</span><span class="si">%.1f</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="n">i</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">params</span><span class="p">])</span><span class="o">.</span><span class="n">corr</span><span class="p">()</span>
<span class="n">corrmat</span><span class="p">(</span><span class="n">corr</span><span class="p">)</span>
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
</pre></div>
</div>
<img alt="../_images/sphx_glr_start_004.png" class="align-center" src="../_images/sphx_glr_start_004.png" />
<p><strong>Clustered correlation heatmap plot</strong></p>
<p>The <code class="xref py py-class docutils literal notranslate"><span class="pre">clustered_corrmap</span></code> function is similar to <code class="xref py py-class docutils literal notranslate"><span class="pre">corrmat</span></code>,
but differs in two respects. First, and most importantly, it uses a user
supplied clustering estimator to cluster the correlation matrix on similar
features, which can often help visualize whether there are blocks of highly
correlated features. Secondly, it plots the full matrix (as opposed to the
lower triangle).</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">mlens.visualization</span> <span class="kn">import</span> <span class="n">clustered_corrmap</span>
<span class="kn">from</span> <span class="nn">sklearn.cluster</span> <span class="kn">import</span> <span class="n">KMeans</span>
<span class="n">Z</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="p">[</span><span class="s1">&#39;f_</span><span class="si">%i</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="n">i</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">5</span><span class="p">)])</span>
</pre></div>
</div>
<p>We duplicate all features, note that the heatmap orders features
as duplicate pairs, and thus fully pick up on this duplication.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">corr</span> <span class="o">=</span> <span class="n">Z</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">Z</span><span class="p">,</span> <span class="n">lsuffix</span><span class="o">=</span><span class="s1">&#39;L&#39;</span><span class="p">,</span> <span class="n">rsuffix</span><span class="o">=</span><span class="s1">&#39;R&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">corr</span><span class="p">()</span>
<span class="n">clustered_corrmap</span><span class="p">(</span><span class="n">corr</span><span class="p">,</span> <span class="n">KMeans</span><span class="p">())</span>
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
</pre></div>
</div>
<img alt="../_images/sphx_glr_start_005.png" class="align-center" src="../_images/sphx_glr_start_005.png" />
<p><strong>Input-Output correlations</strong></p>
<p>The <code class="xref py py-class docutils literal notranslate"><span class="pre">corr_X_y</span></code> function gives a dashboard of
pairwise correlations between the input data (<code class="docutils literal notranslate"><span class="pre">X</span></code>) and the labels to be
predicted (<code class="docutils literal notranslate"><span class="pre">y</span></code>). If the number of features is large, it is advised to set
the <code class="docutils literal notranslate"><span class="pre">no_ticks</span></code> parameter to <code class="docutils literal notranslate"><span class="pre">True</span></code>, to avoid rendering an illegible
x-axis. Note that <code class="docutils literal notranslate"><span class="pre">X</span></code> must be a <a class="reference external" href="http://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html#pandas.DataFrame" title="(in pandas v0.24.1)"><code class="xref py py-class docutils literal notranslate"><span class="pre">pandas.DataFrame</span></code></a>.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">mlens.visualization</span> <span class="kn">import</span> <span class="n">corr_X_y</span>
<span class="n">Z</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="p">[</span><span class="s1">&#39;feature_</span><span class="si">%i</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="n">i</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">5</span><span class="p">)])</span>
<span class="n">corr_X_y</span><span class="p">(</span><span class="n">Z</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="n">no_ticks</span><span class="o">=</span><span class="bp">False</span><span class="p">)</span>
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
</pre></div>
</div>
<img alt="../_images/sphx_glr_start_006.png" class="align-center" src="../_images/sphx_glr_start_006.png" />
<p><strong>Total running time of the script:</strong> ( 0 minutes  5.928 seconds)</p>
<div class="sphx-glr-footer docutils container">
<div class="sphx-glr-download docutils container">
<a class="reference download internal" href="../_downloads/start.py" download=""><code class="xref download docutils literal notranslate"><span class="pre">Download</span> <span class="pre">Python</span> <span class="pre">source</span> <span class="pre">code:</span> <span class="pre">start.py</span></code></a></div>
<div class="sphx-glr-download docutils container">
<a class="reference download internal" href="../_downloads/start.ipynb" download=""><code class="xref download docutils literal notranslate"><span class="pre">Download</span> <span class="pre">Jupyter</span> <span class="pre">notebook:</span> <span class="pre">start.ipynb</span></code></a></div>
</div>
<p class="sphx-glr-signature"><a class="reference external" href="https://sphinx-gallery.readthedocs.io">Generated by Sphinx-Gallery</a></p>
</div>
</div>


           </div>
           <div class="articleComments">
            
           </div>
          </div>
          <footer>
  

  <hr/>

  <div role="contentinfo">
    <p>
        &copy; Copyright 2017, Sebastian Flennerhag.

    </p>
  </div>
  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>

        </div>
      </div>

    </section>

  </div>
  


  

    <script type="text/javascript">
        var DOCUMENTATION_OPTIONS = {
            URL_ROOT:'../',
            VERSION:'0.2.3',
            COLLAPSE_INDEX:false,
            FILE_SUFFIX:'.html',
            HAS_SOURCE:  true,
            SOURCELINK_SUFFIX: '.txt'
        };
    </script>
      <script type="text/javascript" src="../_static/jquery.js"></script>
      <script type="text/javascript" src="../_static/underscore.js"></script>
      <script type="text/javascript" src="../_static/doctools.js"></script>
      <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>

  

  
  
    <script type="text/javascript" src="../_static/js/theme.js"></script>
  

  
  
  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.StickyNav.enable();
      });
  </script>
   

</body>
</html>