<NME> test_bootstrap_controller.py
<BEF> """
Tests for the bootstrap.py (formerly bootstrap_controller.py) file.
"""
import unittest
from collections import OrderedDict

import numpy as np
import numpy.testing as npt
import pandas as pd
from scipy.sparse import csr_matrix, eye

import pylogit.bootstrap as bc
import pylogit.asym_logit as asym
import pylogit.mixed_logit_calcs as mlc
import pylogit.mixed_logit as mixed_logit
import pylogit.nested_logit as nested_logit
from pylogit.conditional_logit import MNL

try:
    # Python 3.x does not natively support xrange
    from past.builtins import xrange
except ImportError:
    pass


class BootstrapTests(unittest.TestCase):
    def setUp(self):
        # The set up being used is one where there are two choice situations,
        # The first having three alternatives, and the second having only two
        # alternatives. There is one generic variable. Two alternative
        # specific constants and all three shape parameters are used.

        # Create the betas to be used during the tests
        self.fake_betas = np.array([-0.6])

        # Create the fake outside intercepts to be used during the tests
        self.fake_intercepts = np.array([1, 0.5])

        # Create names for the intercept parameters
        self.fake_intercept_names = ["ASC 1", "ASC 2"]

        # Record the position of the intercept that is not being estimated
        self.fake_intercept_ref_pos = 2

        # Create the shape parameters to be used during the tests. Note that
        # these are the reparameterized shape parameters, thus they will be
        # exponentiated in the fit_mle process and various calculations.
        self.fake_shapes = np.array([-1, 1])

        # Create names for the intercept parameters
        self.fake_shape_names = ["Shape 1", "Shape 2"]

        # Record the position of the shape parameter that is being constrained
        self.fake_shape_ref_pos = 2

        # Calculate the 'natural' shape parameters
        self.natural_shapes = asym._convert_eta_to_c(self.fake_shapes,
                                                     self.fake_shape_ref_pos)

        # Create an array of all model parameters
        self.fake_all_params = np.concatenate((self.fake_shapes,
                                               self.fake_intercepts,
                                               self.fake_betas))

        # The mapping between rows and alternatives is given below.
        self.fake_rows_to_alts = csr_matrix(np.array([[1, 0, 0],
                                                      [0, 1, 0],
                                                      [0, 0, 1],
                                                      [1, 0, 0],
                                                      [0, 0, 1],
                                                      [1, 0, 0],
                                                      [0, 1, 0],
                                                      [0, 0, 1],
                                                      [0, 1, 0],
                                                      [0, 0, 1],
                                                      [1, 0, 0],
                                                      [0, 1, 0],
                                                      [0, 0, 1],
                                                      [1, 0, 0],
                                                      [0, 1, 0],
                                                      [0, 0, 1]]))

        # Get the mappping between rows and observations
        self.fake_rows_to_obs = csr_matrix(np.array([[1, 0, 0, 0, 0, 0],
                                                     [1, 0, 0, 0, 0, 0],
                                                     [1, 0, 0, 0, 0, 0],
                                                     [0, 1, 0, 0, 0, 0],
                                                     [0, 1, 0, 0, 0, 0],
                                                     [0, 0, 1, 0, 0, 0],
                                                     [0, 0, 1, 0, 0, 0],
                                                     [0, 0, 1, 0, 0, 0],
                                                     [0, 0, 0, 1, 0, 0],
                                                     [0, 0, 0, 1, 0, 0],
                                                     [0, 0, 0, 0, 1, 0],
                                                     [0, 0, 0, 0, 1, 0],
                                                     [0, 0, 0, 0, 1, 0],
                                                     [0, 0, 0, 0, 0, 1],
                                                     [0, 0, 0, 0, 0, 1],
                                                     [0, 0, 0, 0, 0, 1]]))

        # Create the fake design matrix with columns denoting X
        # The intercepts are not included because they are kept outside the
        # index in the scobit model.
        self.fake_design = np.array([[1],
                                     [2],
                                     [3],
                                     [1.5],
                                     [3.5],
                                     [0.78],
                                     [0.23],
                                     [1.04],
                                     [2.52],
                                     [1.49],
                                     [0.85],
                                     [1.37],
                                     [1.17],
                                     [2.03],
                                     [1.62],
                                     [1.94]])

        # Create the index array for this set of choice situations
        self.fake_index = self.fake_design.dot(self.fake_betas)

        # Create the needed dataframe for the Asymmetric Logit constructor
        self.fake_df = pd.DataFrame({"obs_id": [1, 1, 1, 2, 2, 3, 3, 3,
                                                4, 4, 5, 5, 5, 6, 6, 6],
                                     "alt_id": [1, 2, 3, 1, 3, 1, 2, 3,
                                                2, 3, 1, 2, 3, 1, 2, 3],
                                     "choice": [0, 1, 0, 0, 1, 1, 0, 0,
                                                1, 0, 1, 0, 0, 0, 0, 1],
                                     "x": self.fake_design[:, 0],
                                     "intercept":
                                        np.ones(self.fake_design.shape[0])})

        # Record the various column names
        self.alt_id_col = "alt_id"
        self.obs_id_col = "obs_id"
        self.choice_col = "choice"

        # Create the index specification  and name dictionaryfor the model
        self.fake_specification = OrderedDict()
        self.fake_names = OrderedDict()
        self.fake_specification["x"] = [[1, 2, 3]]
        self.fake_names["x"] = ["x (generic coefficient)"]

        # Bundle args and kwargs used to construct the Asymmetric Logit model.
        self.constructor_args = [self.fake_df,
                                 self.alt_id_col,
                                 self.obs_id_col,
                                 self.choice_col,
                                 self.fake_specification]

        # Create a variable for the kwargs being passed to the constructor
        self.constructor_kwargs = {"intercept_ref_pos":
                                   self.fake_intercept_ref_pos,
                                   "shape_ref_pos": self.fake_shape_ref_pos,
                                   "names": self.fake_names,
                                   "intercept_names":
                                   self.fake_intercept_names,
                                   "shape_names": self.fake_shape_names}

        # Initialize a basic Asymmetric Logit model whose coefficients will be
        # estimated.
        self.asym_model_obj = asym.MNAL(*self.constructor_args,
                                   **self.constructor_kwargs)

        self.asym_model_obj.coefs = pd.Series(self.fake_betas)
        self.asym_model_obj.intercepts =\
            pd.Series(self.fake_intercepts, index=self.fake_intercept_names)
        self.asym_model_obj.shapes =\
            pd.Series(self.fake_shapes, index=self.fake_shape_names)
        self.asym_model_obj.params =\
            pd.Series(np.concatenate([self.fake_shapes,
                                      self.fake_intercepts,
                                      self.fake_betas]),
                      index=self.fake_shape_names +
                            self.fake_intercept_names +
                            self.fake_names["x"])
        self.asym_model_obj.nests = None

        #####
        # Initialize a basic MNL model
        #####
        # Create the MNL specification and name dictionaries.
        self.mnl_spec, self.mnl_names = OrderedDict(), OrderedDict()
        self.mnl_spec["intercept"] = [1, 2]
        self.mnl_names["intercept"] = self.fake_intercept_names

        self.mnl_spec.update(self.fake_specification)
        self.mnl_names.update(self.fake_names)

        mnl_construct_args = self.constructor_args[:-1] + [self.mnl_spec]
        mnl_kwargs = {"names": self.mnl_names}
        self.mnl_model_obj = MNL(*mnl_construct_args, **mnl_kwargs)

        return None

    def test_get_param_names(self):
        # Alias the function being tested.
        func = bc.get_param_names

        # Get the function results
        func_results = func(self.asym_model_obj)

        # Get the expected results
        expected_results = self.asym_model_obj.params.index.tolist()

        # Test the function results
        self.assertIsInstance(func_results, list)
        self.assertEqual(func_results, expected_results)

        # Set the nest names and re-test the function.
        self.asym_model_obj.nest_names = ["No Nest"]
        expected_results_2 = self.asym_model_obj.nest_names + expected_results
        func_results_2 = func(self.asym_model_obj)
        self.assertIsInstance(func_results_2, list)
        self.assertEqual(func_results_2, expected_results_2)

        return None

    def test_get_param_list_for_prediction(self):
        self.assertEqual(self.asym_model_obj.params.index.tolist(),
                         boot_obj.mle_params.index.tolist())
        expected_attrs =\
            ["point_samples", "conf_intervals", "conf_alpha", "summary"]
        for current_attr in expected_attrs:
            self.assertTrue(hasattr(boot_obj, current_attr))
            self.assertIsNone(getattr(boot_obj, current_attr))

        return None

    def test_bootstrap_params(self):
        # Create the bootstrap object.
        boot_obj =\
            bc.Boot(self.asym_model_obj, self.asym_model_obj.params.values)
        # Create the expected result
        expected_param_list = [4 * np.ones((5, num_replicates)),
                               3 * np.ones((1, num_replicates)),
                               2 * np.ones((3, num_replicates)),
                               np.ones((2, num_replicates))]

        # Alias the function being tested
        func = bc.get_param_list_for_prediction

        # Calculate the function result
        func_result = func(fake_model_obj, fake_replicates)

        # Perform the desired tests with a full set of parameters
        self.assertIsInstance(func_result, list)
        self.assertEqual(len(func_result), 4)
        for pos, func_array in enumerate(func_result):
                            "constrained_pos": [0],
                            "boot_seed": 1988}

        # Get the function results
        func_results =\
            boot_obj.bootstrap_params(num_samples,
                                      mnl_obj=self.mnl_model_obj,
                                      mnl_init_vals=mnl_init_vals,
                                      mnl_fit_kwargs=mnl_kwargs,
                                      constrained_pos=[0],
                                      boot_seed=1988)

        # Perform the requisite tests
        self.assertIsNone(func_results)
        self.assertIsInstance(boot_obj.point_samples, pd.DataFrame)
        self.assertEqual(boot_obj.point_samples.ndim, 2)

        expected_shape = (3, self.asym_model_obj.params.size)
        self.assertEqual(boot_obj.point_samples.shape, expected_shape)
        self.assertEqual(boot_obj.point_samples.iloc[:, 0].unique().size, 1)
        self.assertEqual(boot_obj.point_samples.iloc[:, 0].unique()[0], 0)
        self.assertTrue(boot_obj.point_samples.iloc[:, 1].unique().size > 1)
        return None
        expected_error_msg =\
            "`replicates` MUST be either 'bootstrap' or 'jackknife'."
        # Perform the desired tests
        for good_arg in good_args:
            self.assertIsNone(func(good_arg))
        for bad_arg in bad_args:
            self.assertRaisesRegexp(ValueError,
                                    expected_error_msg,
                                    func,
                                    bad_arg)
        return None

    def test_boot_initialization(self):
        # Create the bootstrap object
        boot_obj =\
            bc.Boot(self.asym_model_obj, self.asym_model_obj.params.values)

        # Test the bootstrap object.
        self.assertIsInstance(boot_obj, bc.Boot)
        self.assertEqual(id(boot_obj.model_obj), id(self.asym_model_obj))
        self.assertEqual(self.asym_model_obj.params.index.tolist(),
                         boot_obj.mle_params.index.tolist())
        expected_attrs =\
            ["bootstrap_replicates", "jackknife_replicates",
             "percentile_interval", "bca_interval",
             "abc_interval", "all_intervals",
             "jackknife_log_likehoods",
             "bootstrap_log_likelihoods"]
        for current_attr in expected_attrs:
            self.assertTrue(hasattr(boot_obj, current_attr))
            self.assertIsNone(getattr(boot_obj, current_attr))

        return None

    def test_generate_bootstrap_replicates(self):
        # Create the bootstrap object.
        boot_obj =\
            bc.Boot(self.asym_model_obj, self.asym_model_obj.params.values)

        # Determine the number of bootstrap samples that we wish to take
        num_samples = 3

        # Create the necessary keyword arguments.
        mnl_init_vals =\
            np.zeros(len(self.fake_intercept_names) +
                     sum([len(x) for x in self.fake_names.values()]))

        mnl_kwargs = {"ridge": 0.01,
                      "maxiter": 1200,
                      "method": "bfgs"}

        bootstrap_kwargs = {"mnl_obj": self.mnl_model_obj,
                            "mnl_init_vals": mnl_init_vals,
                            "mnl_fit_kwargs": mnl_kwargs,
                            "constrained_pos": [0],
                            "boot_seed": 1988}

        # Alias the needed function
        func = boot_obj.generate_bootstrap_replicates

        # Get the function results
        func_results =\
            func(num_samples,
                 mnl_obj=self.mnl_model_obj,
                 mnl_init_vals=mnl_init_vals,
                 mnl_fit_kwargs=mnl_kwargs,
                 constrained_pos=[0],
                 boot_seed=1988)

        # Perform the requisite tests
        self.assertIsNone(func_results)
        self.assertIsInstance(boot_obj.bootstrap_replicates, pd.DataFrame)
        self.assertEqual(boot_obj.bootstrap_replicates.ndim, 2)

        expected_shape = (num_samples, self.asym_model_obj.params.size)
        self.assertEqual(boot_obj.bootstrap_replicates.shape, expected_shape)
        self.assertEqual(boot_obj.bootstrap_replicates
                                 .iloc[:, 0].unique().size, 1)
        self.assertEqual(boot_obj.bootstrap_replicates
                                 .iloc[:, 0].unique()[0], 0)
        self.assertTrue(boot_obj.bootstrap_replicates
                                .iloc[:, 1].unique().size > 1)
        return None

    def test_generate_jackknife_replicates(self):
        # Create the bootstrap object.
        boot_obj =\
            bc.Boot(self.asym_model_obj, self.asym_model_obj.params.values)



        # Create the necessary keyword arguments.
        mnl_init_vals =\
            np.zeros(len(self.fake_intercept_names) +
                     sum([len(x) for x in self.fake_names.values()]))

        mnl_kwargs = {"ridge": 0.01,
                      "maxiter": 1200,
                      "method": "bfgs"}

        bootstrap_kwargs = {"mnl_obj": self.mnl_model_obj,
                            "mnl_init_vals": mnl_init_vals,
                            "mnl_fit_kwargs": mnl_kwargs,
                            "constrained_pos": [0],
                            "boot_seed": 1988}

        # Alias the needed function
        func = boot_obj.generate_jackknife_replicates

        # Get the function results
        func_results =\
            func(mnl_obj=self.mnl_model_obj,
                 mnl_init_vals=mnl_init_vals,
                 mnl_fit_kwargs=mnl_kwargs,
                 constrained_pos=[0])

        # Perform the requisite tests
        self.assertIsNone(func_results)
        self.assertIsInstance(boot_obj.jackknife_replicates, pd.DataFrame)
        self.assertEqual(boot_obj.jackknife_replicates.ndim, 2)

        expected_shape =\
            (self.fake_rows_to_obs.shape[1], self.asym_model_obj.params.size)
        self.assertEqual(boot_obj.jackknife_replicates.shape, expected_shape)
        self.assertEqual(boot_obj.jackknife_replicates
                                 .iloc[:, 0].unique().size, 1)
        self.assertEqual(boot_obj.jackknife_replicates
                                 .iloc[:, 0].unique()[0], 0)
        self.assertTrue(boot_obj.jackknife_replicates
                                .iloc[:, 1].unique().size > 1)
        return None


class IntervalTests(unittest.TestCase):
    """
    References
    ----------
    Efron, Bradley, and Robert J. Tibshirani. An Introduction to the
        Bootstrap. CRC press, 1994. Chapter 14.

    Notes
    -----
    The data and tests used in the `IntervalTests` test suite come from the
    Efron & Tibshirani reference cited above.
    """
    def setUp(self):
        # Store the spatial test data from Efron and Tibshirani (1994)
        self.test_data =\
            np.array([48, 36, 20, 29, 42, 42, 20, 42, 22, 41, 45, 14, 6,
                      0, 33, 28, 34, 4, 32, 24, 47, 41, 24, 26, 30, 41])

        # Note how many test data observations there are.
        self.num_test_obs = self.test_data.size

        # Store the MLE estimate
        self.test_theta_hat = self.calc_theta(self.test_data)
        # Create a pandas series of the data. Allows for easy case deletion.
        self.raw_series = pd.Series(self.test_data)
        # Create the array of jackknife replicates
        self.jackknife_replicates =\
            np.empty((self.num_test_obs, 1), dtype=float)
        for obs in xrange(self.num_test_obs):
            current_data = self.raw_series[self.raw_series.index != obs].values
            self.jackknife_replicates[obs] = self.calc_theta(current_data)[0]

        # Create the bootstrap replicates
        num_test_reps = 5000
        test_indices = np.arange(self.num_test_obs)
        boot_indx_shape = (num_test_reps, self.num_test_obs)
        np.random.seed(8292017)
        boot_indices =\
            np.random.choice(test_indices,
                             replace=True,
                             size=self.num_test_obs*num_test_reps)
        self.bootstrap_replicates =\
            np.fromiter((self.calc_theta(self.test_data[x])[0] for x in
                         boot_indices.reshape(boot_indx_shape)),
                        dtype=float)[:, None]

        self.rows_to_obs = eye(self.test_data.size, format='csr', dtype=int)

        # Create a fake model object and a fake model class that will implement the
        # T(P) function through it's fit_mle method.
        test_data = self.test_data
        fake_rows_to_obs = self.rows_to_obs
        calc_theta = self.calc_theta

        class FakeModel(object):
            def __init__(self):
                # Create needed attributes to successfully mock an MNDC_Model
                #instance in this test
                self.data = pd.Series([pos for pos, x in enumerate(test_data)])
                self.obs_id_col = np.arange(self.data.size, dtype=int)

                needed_names = ['ind_var_names', 'intercept_names',
                                'shape_names', 'nest_names']
                for name in needed_names:
                    setattr(self, name, None)
                self.ind_var_names = ['variance']

            # Create a get_mappings_for_fit function that will allow for
            # successful mocking in this test
            def get_mappings_for_fit(self):
                return {"rows_to_obs": fake_rows_to_obs}

            # Use the T(P) function from the spatial test data example.
            def fit_mle(self,
                        init_vals,
                        weights=None,
                        **kwargs):
                return {'x': calc_theta(test_data, weights=weights)}
        self.fake_model_obj = FakeModel()

        # Create the bootstrap object
        self.boot =\
            bc.Boot(self.fake_model_obj,
                    pd.Series(self.test_theta_hat, index=["variance"]))

        self.boot.bootstrap_replicates =\
            pd.DataFrame(self.bootstrap_replicates, columns=['variance'])
        self.boot.jackknife_replicates =\
            pd.DataFrame(self.jackknife_replicates, columns=['variance'])

        # Store the confidence percentage that will be used for the test
        self.conf_percentage = 90
        return None

    # Create the function to calculate the objective function.
    def calc_theta(self, array, weights=None):
        if weights is None:
            result = ((array - array.mean())**2).sum() / float(array.size)
        else:
            a_mean = weights.dot(array)
            differences = (array - a_mean)
            squared_diffs = differences**2
            result = weights.dot(squared_diffs)
        return np.array([result])

    def test_calc_percentile_interval(self):
        # Alias the function being tested
        func = self.boot.calc_percentile_interval

        # Perform the first test
        self.assertIsNone(self.boot.percentile_interval)

        # Calculate the function result
        func(self.conf_percentage)

        # Note the expected result is from Table 14.2 on page 183 of
        # Efron & Tibshirani (1994)
        expected_result = np.array([100.8, 233.9])
        expected_columns = ['5%', '95%']

        # Perform the remaining tests
        self.assertIsInstance(self.boot.percentile_interval, pd.DataFrame)
        self.assertEqual(expected_columns,
                         self.boot.percentile_interval.columns.tolist())
        self.assertIn("variance", self.boot.percentile_interval.index)
        self.assertEqual(self.boot.percentile_interval.shape, (1, 2))
        npt.assert_allclose(self.boot.percentile_interval.iloc[0, :],
                            expected_result, rtol=0.02)

        # Set the percentile interval back to none.
        self.boot.percentile_interval = None
        self.assertIsNone(self.boot.percentile_interval)
        return None

    def test_calc_bca_interval(self):
        # Alias the function being tested
        func = self.boot.calc_bca_interval

        # Perform the first test
        self.assertIsNone(self.boot.bca_interval)

        # Calculate the function result
        func(self.conf_percentage)

        # Note the expected result is from Table 14.2 on page 183 of
        # Efron & Tibshirani (1994)
        expected_result = np.array([115.8, 259.6])
        expected_columns = ['5%', '95%']

        # Perform the remaining tests
        self.assertIsInstance(self.boot.bca_interval, pd.DataFrame)
        self.assertEqual(expected_columns,
                         self.boot.bca_interval.columns.tolist())
        self.assertIn("variance", self.boot.bca_interval.index)
        self.assertEqual(self.boot.bca_interval.shape, (1, 2))
        npt.assert_allclose(self.boot.bca_interval.iloc[0, :],
                            expected_result, rtol=0.01)

        # Set the percentile interval back to none.
        self.boot.bca_interval = None
        self.assertIsNone(self.boot.bca_interval)
        return None

    def test_calc_abc_interval(self):
        # Alias the function being tested
        func = self.boot.calc_abc_interval

        # Perform the first test
        self.assertIsNone(self.boot.abc_interval)

        # Calculate the function result
        func(self.conf_percentage, self.test_theta_hat, epsilon=0.001)

        # Note the expected result, from Table 14.2 on page 183 of
        # Efron & Tibshirani (1994)
        expected_result = np.array([116.7, 260.9])
        expected_columns = ['5%', '95%']

        # Perform the remaining tests
        self.assertIsInstance(self.boot.abc_interval, pd.DataFrame)
        self.assertEqual(expected_columns,
                         self.boot.abc_interval.columns.tolist())
        self.assertIn("variance", self.boot.abc_interval.index)
        self.assertEqual(self.boot.abc_interval.shape, (1, 2))
        npt.assert_allclose(self.boot.abc_interval.iloc[0, :],
                            expected_result, rtol=0.01)

        # Set the percentile interval back to none.
        self.boot.abc_interval = None
        self.assertIsNone(self.boot.abc_interval)
        return None

    def test_calc_conf_intervals_except_all(self):
        kwargs = {"init_vals": self.test_theta_hat,
                  "epsilon": 0.001}

        # Alias the function being tested
        func = self.boot.calc_conf_intervals

        # Create the list of attributes to be tested
        tested_attrs = ['percentile_interval', 'bca_interval', 'abc_interval']
        interval_types = ['pi', 'bca', 'abc']

        # Note the expected result, from Table 14.2 on page 183 of
        # Efron & Tibshirani (1994)
        expected_result =\
            np.array([[100.8, 233.9], [115.8, 259.6], [116.7, 260.9]])
        expected_columns = ['5%', '95%']

        # Perform the desired tests
        for pos, i_type in enumerate(interval_types):
            desired_attr = getattr(self.boot, tested_attrs[pos])
            self.assertIsNone(desired_attr)
            # Calculate the function result
            kwargs['interval_type'] = i_type
            func(self.conf_percentage, **kwargs)
            # Perform the remaining tests
            desired_attr = getattr(self.boot, tested_attrs[pos])
            self.assertIsInstance(desired_attr, pd.DataFrame)
            self.assertEqual(expected_columns,
                             desired_attr.columns.tolist())
            self.assertIn("variance", desired_attr.index)
            self.assertEqual(desired_attr.shape, (1, 2))
            npt.assert_allclose(desired_attr.iloc[0, :],
                                expected_result[pos], rtol=0.02)

            # Perform clean-up activities after the test
            setattr(self.boot, tested_attrs[pos], None)
        return None

    def test_calc_conf_intervals_all(self):
        kwargs = {"interval_type": 'all',
                  "init_vals": self.test_theta_hat,
                  "epsilon": 0.001}

        # Alias the function being tested
        func = self.boot.calc_conf_intervals

        # Create the list of attributes to be tested
        tested_attrs = ['percentile_interval', 'bca_interval', 'abc_interval']

        # Note the expected result, from Table 14.2 on page 183 of
        # Efron & Tibshirani (1994)
        expected_result =\
            np.array([[100.8, 233.9], [115.8, 259.6], [116.7, 260.9]])

        # Note the expected MultiIndex columns
        expected_columns_all = [("percentile_interval", "5%"),
                                ("percentile_interval", "95%"),
                                ("BCa_interval", "5%"),
                                ("BCa_interval", "95%"),
                                ("ABC_interval", "5%"),
                                ("ABC_interval", "95%")]
        expected_columns_single = ["5%", "95%"]

        # Perform the expected tests before running the function
        for attr in tested_attrs:
            self.assertIsNone(getattr(self.boot, attr))

        # Calculate the function results
        func(self.conf_percentage, **kwargs)

        # Perform the remaining tests
        for pos, attr in enumerate(tested_attrs):
            desired_attr = getattr(self.boot, attr)
            self.assertEqual(expected_columns_single,
                             desired_attr.columns.tolist())
            self.assertIn("variance", desired_attr.index)
            self.assertEqual(desired_attr.shape, (1, 2))
            npt.assert_allclose(desired_attr.iloc[0, :],
                                expected_result[pos], rtol=0.02)

        # Test the 'all_intervals' attribute.
        self.assertIsInstance(self.boot.all_intervals, pd.DataFrame)
        self.assertEqual(expected_columns_all,
                         self.boot.all_intervals.columns.tolist())
        self.assertIn("variance", self.boot.all_intervals.index)
        self.assertEqual(self.boot.all_intervals.shape, (1, 6))
        npt.assert_allclose(self.boot.all_intervals.values,
                            expected_result.reshape((1, 6)), rtol=0.02)

        # Set the various intervals back to None.
        for attr in tested_attrs + ['all_intervals']:
            setattr(self.boot, attr, None)
            self.assertIsNone(getattr(self.boot, attr))
        return None

    def test_interval_type_error_in_calc_conf_intervals(self):
        # Alias the function being tested
        func = self.boot.calc_conf_intervals

        # Create kwargs for the function to be tested
        kwargs = {"interval_type": 'bad_type',
                  "init_vals": self.test_theta_hat,
                  "epsilon": 0.001}

        # Note the expected error message.
        expected_error_msg =\
            "interval_type MUST be in `\['pi', 'bca', 'abc', 'all'\]`"

        # Ensure that the appropriate errors are raised.
        self.assertRaisesRegexp(ValueError,
                                expected_error_msg,
                                func,
                                self.conf_percentage,
                                **kwargs)
        return None


class AnalysisTests(unittest.TestCase):
    def make_mnl_model(self):
        # The set up being used is one where there are two choice situations,
        # The first having three alternatives, and the second having only two
        # alternatives. There is one generic variable. Two alternative
        # specific constants and all three shape parameters are used.

        # Create the betas to be used during the tests
        fake_betas = np.array([-0.6])

        # Create the fake design matrix with columns denoting X
        # The intercepts are not included because they are kept outside the
        # index in the scobit model.
        fake_design = np.array([[1],
                                [2],
                                [3],
                                [1.5],
                                [3.5]])

        # Create the index array for this set of choice situations
        fake_index = fake_design.dot(fake_betas)

        # Create the needed dataframe for the model constructor
        fake_df = pd.DataFrame({"obs_id": [1, 1, 1, 2, 2],
                                "alt_id": [1, 2, 3, 1, 3],
                                "choice": [0, 1, 0, 0, 1],
                                "x": fake_design[:, 0]})

        # Record the various column names
        alt_id_col = "alt_id"
        obs_id_col = "obs_id"
        choice_col = "choice"

        # Create the index specification  and name dictionaryfor the model
        mnl_spec = OrderedDict()
        mnl_names = OrderedDict()
        mnl_spec["x"] = [[1, 2, 3]]
        mnl_names["x"] = ["x (generic coefficient)"]

        # Bundle args and kwargs used to construct the Asymmetric Logit model.
        mnl_args = [fake_df, alt_id_col, obs_id_col, choice_col, mnl_spec]

        # Create a variable for the kwargs being passed to the constructor
        mnl_kwargs = {"names": mnl_names}

        # Initialize a basic choice model.
        mnl_obj = MNL(*mnl_args, **mnl_kwargs)

        # Create the desired model attributes for the clog log model
        mnl_obj.coefs = pd.Series(fake_betas, index=mnl_names["x"])
        mnl_obj.intercepts = None
        mnl_obj.shapes = None
        mnl_obj.nests = None
        mnl_obj.params = mnl_obj.coefs.copy()
        return mnl_obj

    def make_asym_model(self):
        # The set up being used is one where there are two choice situations,
        # The first having three alternatives, and the second having only two
        # alternatives. There is one generic variable. Two alternative
        # specific constants and all three shape parameters are used.

        # Create the betas to be used during the tests
        fake_betas = np.array([-0.6])

        # Create the fake outside intercepts to be used during the tests
        fake_intercepts = np.array([1, 0.5])

        # Create names for the intercept parameters
        fake_intercept_names = ["ASC 1", "ASC 2"]

        # Record the position of the intercept that is not being estimated
        fake_intercept_ref_pos = 2

        # Create the shape parameters to be used during the tests. Note that
        # these are the reparameterized shape parameters, thus they will be
        # exponentiated in the fit_mle process and various calculations.
        fake_shapes = np.array([-1, 1])

        # Create names for the intercept parameters
        fake_shape_names = ["Shape 1", "Shape 2"]

        # Record the position of the shape parameter that is being constrained
        fake_shape_ref_pos = 2

        # Calculate the 'natural' shape parameters
        natural_shapes = asym._convert_eta_to_c(fake_shapes,
                                                fake_shape_ref_pos)

        # Create an array of all model parameters
        fake_all_params = np.concatenate((fake_shapes,
                                          fake_intercepts,
                                          fake_betas))

        # The mapping between rows and alternatives is given below.
        fake_rows_to_alts = csr_matrix(np.array([[1, 0, 0],
                                                 [0, 1, 0],
                                                 [0, 0, 1],
                                                 [1, 0, 0],
                                                 [0, 0, 1]]))

        # Get the mappping between rows and observations
        fake_rows_to_obs = csr_matrix(np.array([[1, 0],
                                                [1, 0],
                                                [1, 0],
                                                [0, 1],
                                                [0, 1]]))

        # Create the fake design matrix with columns denoting X
        # The intercepts are not included because they are kept outside the
        # index in the scobit model.
        fake_design = np.array([[1],
                                [2],
                                [3],
                                [1.5],
                                [3.5]])

        # Create the index array for this set of choice situations
        fake_index = fake_design.dot(fake_betas)

        # Create the needed dataframe for the Asymmetric Logit constructor
        fake_df = pd.DataFrame({"obs_id": [1, 1, 1, 2, 2],
                                "alt_id": [1, 2, 3, 1, 3],
                                "choice": [0, 1, 0, 0, 1],
                                "x": fake_design[:, 0],
                                "intercept": [1 for i in range(5)]})

        # Record the various column names
        alt_id_col = "alt_id"
        obs_id_col = "obs_id"
        choice_col = "choice"

        # Create the index specification  and name dictionaryfor the model
        fake_specification = OrderedDict()
        fake_names = OrderedDict()
        fake_specification["x"] = [[1, 2, 3]]
        fake_names["x"] = ["x (generic coefficient)"]

        # Bundle args and kwargs used to construct the Asymmetric Logit model.
        constructor_args = [fake_df,
                            alt_id_col,
                            obs_id_col,
                            choice_col,
                            fake_specification]

        # Create a variable for the kwargs being passed to the constructor
        constructor_kwargs = {"intercept_ref_pos": fake_intercept_ref_pos,
                              "shape_ref_pos": fake_shape_ref_pos,
                              "names": fake_names,
                              "intercept_names": fake_intercept_names,
                               "shape_names": fake_shape_names}

        # Initialize a basic Asymmetric Logit model whose coefficients will be
        # estimated.
        model_obj = asym.MNAL(*constructor_args, **constructor_kwargs)

        # Get the fitted probabilities for this model and dataset
        # Note this relies on the calc_probabilities function being functional.
        # args = [fake_betas,
        #         fake_design,
        #         fake_df[alt_id_col].values,
        #         fake_rows_to_obs,
        #         fake_rows_to_alts,
        #         model_obj.utility_transform]
        # kwargs = {"intercept_params": fake_intercepts,
        #           "shape_params": fake_shapes,
        #           "return_long_probs": True}
        # model_obj.prob_array =\
        #     choice_calcs.calc_probabilities(*args, **kwargs)

        model_obj.coefs = pd.Series(fake_betas, index=fake_names["x"])
        model_obj.intercepts =\
            pd.Series(fake_intercepts, index=fake_intercept_names)
        model_obj.shapes = pd.Series(fake_shapes, index=fake_shape_names)
        model_obj.nests = None
        model_obj.params =\
            pd.concat([model_obj.shapes,
                       model_obj.intercepts,
                       model_obj.coefs],
                      axis=0, ignore_index=False)
        return model_obj

    def make_mixed_model(self):
        # Fake random draws where Row 1 is for observation 1 and row 2 is
        # for observation 2. Column 1 is for draw 1 and column 2 is for draw 2
        fake_draws = mlc.get_normal_draws(2, 2, 1, seed=1)[0]
        # Create the betas to be used during the tests
        fake_betas = np.array([0.3, -0.6, 0.2])
        fake_std = 1
        fake_betas_ext = np.concatenate((fake_betas,
                                         np.array([fake_std])),
                                        axis=0)

        # Create the fake design matrix with columns denoting ASC_1, ASC_2, X
        fake_design = np.array([[1, 0, 1],
                               [0, 1, 2],
                               [0, 0, 3],
                               [1, 0, 1.5],
                               [0, 1, 2.5],
                               [0, 0, 3.5],
                               [1, 0, 0.5],
                               [0, 1, 1.0],
                               [0, 0, 1.5]])
        # Record what positions in the design matrix are being mixed over
        mixing_pos = [2]

        # Create the arrays that specify the choice situation, individual id
        # and alternative ids
        situation_ids = np.array([1, 1, 1, 2, 2, 2, 3, 3, 3])
        individual_ids = np.array([1, 1, 1, 1, 1, 1, 2, 2, 2])
        alternative_ids = np.array([1, 2, 3, 1, 2, 3, 1, 2, 3])
        # Create a fake array of choices
        choice_array = np.array([0, 1, 0, 0, 0, 1, 1, 0, 0])

        # Create the 'rows_to_mixers' sparse array for this dataset
        # Denote the rows that correspond to observation 1 and observation 2
        obs_1_rows = np.ones(fake_design.shape[0])
        # Make sure the rows for observation 2 are given a zero in obs_1_rows
        obs_1_rows[-3:] = 0
        obs_2_rows = 1 - obs_1_rows
        # Create the row_to_mixers scipy.sparse matrix
        fake_rows_to_mixers = csr_matrix(obs_1_rows[:, None] ==
                                         np.array([1, 0])[None, :])
        # Create the rows_to_obs scipy.sparse matrix
        fake_rows_to_obs = csr_matrix(situation_ids[:, None] ==
                                      np.arange(1, 4)[None, :])
        # Create the rows_to_alts scipy.sparse matrix
        fake_rows_to_alts = csr_matrix(alternative_ids[:, None] ==
                                       np.arange(1, 4)[None, :])

        # Create the design matrix that we should see for draw 1 and draw 2
        arrays_to_join = (fake_design.copy(),
                          fake_design.copy()[:, -1][:, None])
        fake_design_draw_1 = np.concatenate(arrays_to_join, axis=1)
        fake_design_draw_2 = fake_design_draw_1.copy()

        # Multiply the 'random' coefficient draws by the corresponding variable
        fake_design_draw_1[:, -1] *= (obs_1_rows *
                                      fake_draws[0, 0] +
                                      obs_2_rows *
                                      fake_draws[1, 0])
        fake_design_draw_2[:, -1] *= (obs_1_rows *
                                      fake_draws[0, 1] +
                                      obs_2_rows *
                                      fake_draws[1, 1])
        extended_design_draw_1 = fake_design_draw_1[:, None, :]
        extended_design_draw_2 = fake_design_draw_2[:, None, :]
        fake_design_3d = np.concatenate((extended_design_draw_1,
                                         extended_design_draw_2),
                                        axis=1)

        # Create the fake systematic utility values
        sys_utilities_draw_1 = fake_design_draw_1.dot(fake_betas_ext)
        sys_utilities_draw_2 = fake_design_draw_2.dot(fake_betas_ext)

        #####
        # Calculate the probabilities of each alternatve in each choice
        # situation
        #####
        long_exp_draw_1 = np.exp(sys_utilities_draw_1)
        long_exp_draw_2 = np.exp(sys_utilities_draw_2)
        ind_exp_sums_draw_1 = fake_rows_to_obs.T.dot(long_exp_draw_1)
        ind_exp_sums_draw_2 = fake_rows_to_obs.T.dot(long_exp_draw_2)
        long_exp_sum_draw_1 = fake_rows_to_obs.dot(ind_exp_sums_draw_1)
        long_exp_sum_draw_2 = fake_rows_to_obs.dot(ind_exp_sums_draw_2)
        long_probs_draw_1 = long_exp_draw_1 / long_exp_sum_draw_1
        long_probs_draw_2 = long_exp_draw_2 / long_exp_sum_draw_2
        prob_array = np.concatenate((long_probs_draw_1[:, None],
                                     long_probs_draw_2[:, None]),
                                    axis=1)

        ###########
        # Create a mixed logit object for later use.
        ##########
        # Create a fake old long format dataframe for mixed logit model object
        alt_id_column = "alt_id"
        situation_id_column = "situation_id"
        obs_id_column = "observation_id"
        choice_column = "choice"

        data = {"x": fake_design[:, 2],
                alt_id_column: alternative_ids,
                situation_id_column: situation_ids,
                obs_id_column: individual_ids,
                choice_column: choice_array}
        fake_old_df = pd.DataFrame(data)
        fake_old_df["intercept"] = 1

        # Create a fake specification
        fake_spec = OrderedDict()
        fake_names = OrderedDict()

        fake_spec["intercept"] = [1, 2]
        fake_names["intercept"] = ["ASC 1", "ASC 2"]

        fake_spec["x"] = [[1, 2, 3]]
        fake_names["x"] = ["beta_x"]

        # Specify the mixing variable
        fake_mixing_vars = ["beta_x"]

        # Create a fake version of a mixed logit model object
        args = [fake_old_df,
                alt_id_column,
                situation_id_column,
                choice_column,
                fake_spec]
        kwargs = {"names": fake_names,
                  "mixing_id_col": obs_id_column,
                  "mixing_vars": fake_mixing_vars}
        mixl_obj = mixed_logit.MixedLogit(*args, **kwargs)

        # Set all the necessary attributes for prediction:
        # design_3d, coefs, intercepts, shapes, nests, mixing_pos
        mixl_obj.design_3d = fake_design_3d
        mixl_obj.ind_var_names += ["Sigma X"]
        mixl_obj.coefs =\
            pd.Series(fake_betas_ext, index=mixl_obj.ind_var_names)
        mixl_obj.intercepts = None
        mixl_obj.shapes = None
        mixl_obj.nests = None
        mixl_obj.params = mixl_obj.coefs.copy()
        return mixl_obj

    def make_nested_model(self):
        # Create the betas to be used during the tests
        fake_betas = np.array([0.3, -0.6, 0.2])
        # Create the fake nest coefficients to be used during the tests
        # Note that these are the 'natural' nest coefficients, i.e. the
        # inverse of the scale parameters for each nest. They should be bigger
        # than or equal to 1.
        natural_nest_coefs = np.array([1 - 1e-16, 0.5])
        # Create an array of all model parameters
        fake_all_params = np.concatenate((natural_nest_coefs,
                                          fake_betas))
        # The set up being used is one where there are two choice situations,
        # The first having three alternatives, and the second having only two.
        # The nest memberships of these alternatives are given below.
        fake_rows_to_nests = csr_matrix(np.array([[1, 0],
                                                  [1, 0],
                                                  [0, 1],
                                                  [1, 0],
                                                  [0, 1]]))

        # Create a sparse matrix that maps the rows of the design matrix to the
        # observatins
        fake_rows_to_obs = csr_matrix(np.array([[1, 0],
                                                [1, 0],
                                                [1, 0],
                                                [0, 1],
                                                [0, 1]]))

        # Create the fake design matrix with columns denoting ASC_1, ASC_2, X
        fake_design = np.array([[1, 0, 1],
                                [0, 1, 2],
                                [0, 0, 3],
                                [1, 0, 1.5],
                                [0, 0, 3.5]])

        # Create fake versions of the needed arguments for the MNL constructor
        fake_df = pd.DataFrame({"obs_id": [1, 1, 1, 2, 2],
                                "alt_id": [1, 2, 3, 1, 3],
                                "choice": [0, 1, 0, 0, 1],
                                "x": range(5),
                                "intercept": [1 for i in range(5)]})

        # Record the various column names
        alt_id_col = "alt_id"
        obs_id_col = "obs_id"
        choice_col = "choice"

        # Store the choice array
        choice_array = fake_df[choice_col].values

        # Create a sparse matrix that maps the chosen rows of the design
        # matrix to the observatins
        fake_chosen_rows_to_obs = csr_matrix(np.array([[0, 0],
                                                       [1, 0],
                                                       [0, 0],
                                                       [0, 0],
                                                       [0, 1]]))

        # Create the index specification  and name dictionaryfor the model
        fake_specification = OrderedDict()
        fake_specification["intercept"] = [1, 2]
        fake_specification["x"] = [[1, 2, 3]]
        fake_names = OrderedDict()
        fake_names["intercept"] = ["ASC 1", "ASC 2"]
        fake_names["x"] = ["x (generic coefficient)"]

        # Create the nesting specification
        fake_nest_spec = OrderedDict()
        fake_nest_spec["Nest 1"] = [1, 2]
        fake_nest_spec["Nest 2"] = [3]

        # Create a nested logit object
        args = [fake_df,
                alt_id_col,
                obs_id_col,
                choice_col,
                fake_specification]
        kwargs = {"names": fake_names,
                  "nest_spec": fake_nest_spec}
        model_obj = nested_logit.NestedLogit(*args, **kwargs)

        model_obj.coefs = pd.Series(fake_betas, index=model_obj.ind_var_names)
        model_obj.intercepts = None
        model_obj.shapes = None

        def logit(x):
            return np.log(x / (1 - x))
        model_obj.nests =\
            pd.Series(logit(natural_nest_coefs), index=fake_nest_spec.keys())
        model_obj.params =\
            pd.concat([model_obj.nests, model_obj.coefs],
                      axis=0, ignore_index=False)

        # Store a ridge parameter
        # ridge = 0.5

        # Gather the arguments needed for the calc_nested_probs function
        # args = [natural_nest_coefs,
        #         fake_betas,
        #         model_obj.design,
        #         fake_rows_to_obs,
        #         fake_rows_to_nests]
        # kwargs = {"return_type": "long_probs"}
        # model_obj.prob_array = nlc.calc_nested_probs(*args, **kwargs)
        return model_obj

    def setUp(self):
        """
        Create the real model objects.
        """
        self.mnl_model = self.make_mnl_model()
        self.asym_model = self.make_asym_model()
        self.mixed_model = self.make_mixed_model()
        self.nested_model = self.make_nested_model()
        return None

    def test_calc_log_likes_for_replicates(self):
        # Create the keyword arguments needed for the test.
        kwargs = {'num_draws': 10, 'seed': 932017}

        # Note the objects that are to be tested
        model_objects = [self.mnl_model,
                         self.asym_model,
                         self.mixed_model,
                         self.nested_model]

        # Iterate over the Asym, MNL, Mixed, and Nested models.
        for model_obj in model_objects:
            # Create the bootstrap object based on the model object.
            boot = bc.Boot(model_obj, model_obj.params.values)

            # Create the bootstrap and jackknife replicate attributes.
            replicates =\
                pd.DataFrame(np.concatenate([model_obj.params.values[None, :],
                                             model_obj.params.values[None, :]],
                                            axis=0))
            boot.bootstrap_replicates = replicates
            boot.jackknife_replicates = replicates

            # Alias the function being tested.
            func = boot.calc_log_likes_for_replicates

            for replicate_type in ['bootstrap', 'jackknife']:
                # Calculate function results using each bootstrap object
                kwargs["replicates"] = replicate_type
                func_result = func(**kwargs)

                # Ensure function results have the expected properties
                self.assertIsInstance(func_result, np.ndarray)
                self.assertEqual(func_result.ndim, 1)
                self.assertEqual(func_result.shape, (replicates.shape[0],))
        return None

    def test_calc_gradient_norm_for_replicates(self):
        # Create the bootstrap object based on the MNL model.
        base_array = self.mnl_model.params.values
        base_array_2d = base_array[None, :]
        boot = bc.Boot(self.mnl_model, base_array)

        # Create the bootstrap and jackknife replicate attributes.
        replicates = pd.DataFrame(np.concatenate((base_array_2d,
                                                  base_array_2d + 1,
                                                  base_array_2d - 1),
                                                 axis=0))
        boot.bootstrap_replicates = replicates
        boot.jackknife_replicates = replicates

        # Alias the function being tested.
        func = boot.calc_gradient_norm_for_replicates

        # Perform the desired tests.
        for replicate_type in ['bootstrap', 'jackknife']:
            func_result = func(replicates=replicate_type)
            self.assertIsInstance(func_result, np.ndarray)
            self.assertEqual(func_result.shape, (replicates.shape[0],))
            self.assertTrue(np.unique(func_result).size == func_result.size)
        return None

<MSG> Made majority of changes needed before defining the confidence interval function on the Boot object. Refactored some modules to decrease the size of the individual files. Tests still need to be written.

<DFF> @@ -222,14 +222,17 @@ class BootstrapTests(unittest.TestCase):
         self.assertEqual(self.asym_model_obj.params.index.tolist(),
                          boot_obj.mle_params.index.tolist())
         expected_attrs =\
-            ["point_samples", "conf_intervals", "conf_alpha", "summary"]
+            ["bootstrap_replicates", "jackknife_replicates",
+             "percentile_interval", "bca_interval",
+             "abc_interval", "conf_intervals",
+             "conf_alpha", "summary"]
         for current_attr in expected_attrs:
             self.assertTrue(hasattr(boot_obj, current_attr))
             self.assertIsNone(getattr(boot_obj, current_attr))
 
         return None
 
-    def test_bootstrap_params(self):
+    def test_generate_bootstrap_replicates(self):
         # Create the bootstrap object.
         boot_obj =\
             bc.Boot(self.asym_model_obj, self.asym_model_obj.params.values)
@@ -252,23 +255,29 @@ class BootstrapTests(unittest.TestCase):
                             "constrained_pos": [0],
                             "boot_seed": 1988}
 
+        # Alias the needed function
+        func = boot_obj.generate_bootstrap_replicates
+
         # Get the function results
         func_results =\
-            boot_obj.bootstrap_params(num_samples,
-                                      mnl_obj=self.mnl_model_obj,
-                                      mnl_init_vals=mnl_init_vals,
-                                      mnl_fit_kwargs=mnl_kwargs,
-                                      constrained_pos=[0],
-                                      boot_seed=1988)
+            func(num_samples,
+                 mnl_obj=self.mnl_model_obj,
+                 mnl_init_vals=mnl_init_vals,
+                 mnl_fit_kwargs=mnl_kwargs,
+                 constrained_pos=[0],
+                 boot_seed=1988)
 
         # Perform the requisite tests
         self.assertIsNone(func_results)
-        self.assertIsInstance(boot_obj.point_samples, pd.DataFrame)
-        self.assertEqual(boot_obj.point_samples.ndim, 2)
+        self.assertIsInstance(boot_obj.bootstrap_replicates, pd.DataFrame)
+        self.assertEqual(boot_obj.bootstrap_replicates.ndim, 2)
 
         expected_shape = (3, self.asym_model_obj.params.size)
-        self.assertEqual(boot_obj.point_samples.shape, expected_shape)
-        self.assertEqual(boot_obj.point_samples.iloc[:, 0].unique().size, 1)
-        self.assertEqual(boot_obj.point_samples.iloc[:, 0].unique()[0], 0)
-        self.assertTrue(boot_obj.point_samples.iloc[:, 1].unique().size > 1)
+        self.assertEqual(boot_obj.bootstrap_replicates.shape, expected_shape)
+        self.assertEqual(boot_obj.bootstrap_replicates
+                                 .iloc[:, 0].unique().size, 1)
+        self.assertEqual(boot_obj.bootstrap_replicates
+                                 .iloc[:, 0].unique()[0], 0)
+        self.assertTrue(boot_obj.bootstrap_replicates
+                                .iloc[:, 1].unique().size > 1)
         return None

Made majority of changes needed before defining the confidence interval function on the Boot object. Refactored some modules to decrease the size of the individual files. Tests still need to be written.