A Recurrent Latent Variable Model \\ for Sequential Data
%David S.~Hippocampus\thanks{ Use footnote for providing further information %about author (webpage
alternative address)---not
In this paper, we explore the inclusion of latent random variables into the hidden state of a recurrent neural network (RNN) by combining the elements of the variational autoencoder. We argue that through the use of high-level latent random variables, the {variational RNN (VRNN)\footnote{Code is available at \url{http://www.github.com/jych/nips2015_vrnn can model the kind of variability observed in highly structured sequential data such as natural speech. We empirically evaluate the proposed model against other related sequential models on four speech datasets and one handwriting dataset. Our results show the important roles that latent random variables can play in the RNN dynamics.
nips2015.tex
\documentclass{article} % For LaTeX2e
\usepackage{nips15submit_e,times}
% Fix bbl issue: see http://tex.stackexchange.com/questions/14938/ualgocforiginalchapter-undefined-when-using-algorithm2e-with-natbib
\let\chapter\section
\usepackage{algorithm2e}
\usepackage{hyperref}
\usepackage{url}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{amssymb}
\usepackage{graphicx}
\usepackage{xspace}
\usepackage{tabularx}
\usepackage{multicol}
\usepackage{multirow}
\usepackage{url}
\usepackage[numbers,comma,square]{natbib}
\usepackage{wrapfig}
\usepackage[utf8]{inputenc}
\usepackage[normalem]{ulem}
\usepackage{xcolor}

\newcommand{\obs}{\text{obs}}
\newcommand{\mis}{\text{mis}}

\newcommand{\qt}[1]{\left<#1\right>}
\newcommand{\ql}[1]{\left[#1\right]}
\newcommand{\hess}{\mathbf{H}}
\newcommand{\jacob}{\mathbf{J}}
\newcommand{\hl}{HL}
\newcommand{\cost}{\mathcal{L}}
\newcommand{\lout}{\mathbf{r}}
\newcommand{\louti}{r}
\newcommand{\outi}{y}
\newcommand{\out}{\mathbf{y}}
\newcommand{\gauss}{\mathbf{G_N}}
\newcommand{\eye}{\mathbf{I}}
\newcommand{\softmax}{\phi}
\newcommand{\targ}{\mathbf{t}}
\newcommand{\metric}{\mathbf{G}}
\newcommand{\sample}{\mathbf{z}}

\newcommand{\bmx}[0]{\begin{bmatrix}}
\newcommand{\emx}[0]{\end{bmatrix}}
\newcommand{\qexp}[1]{\left<#1\right>}
\newcommand{\vect}[1]{\mathbf{#1}}
\newcommand{\vects}[1]{\boldsymbol{#1}}
\newcommand{\matr}[1]{\mathbf{#1}}
\newcommand{\var}[0]{\operatorname{Var}}
\newcommand{\std}[0]{\operatorname{std}}
\newcommand{\cov}[0]{\operatorname{Cov}}
\newcommand{\diag}[0]{\operatorname{diag}}
\newcommand{\matrs}[1]{\boldsymbol{#1}}
\newcommand{\va}[0]{\vect{a}}
\newcommand{\vb}[0]{\vect{b}}
\newcommand{\vc}[0]{\vect{c}}
\newcommand{\vh}[0]{\vect{h}}
\newcommand{\vv}[0]{\vect{v}}
\newcommand{\vx}[0]{\vect{x}}
\newcommand{\vz}[0]{\vect{z}}
\newcommand{\vw}[0]{\vect{w}}
\newcommand{\vs}[0]{\vect{s}}
\newcommand{\vf}[0]{\vect{f}}
\newcommand{\vi}[0]{\vect{i}}
\newcommand{\vo}[0]{\vect{o}}
\newcommand{\vy}[0]{\vect{y}}
\newcommand{\vg}[0]{\vect{g}}
\newcommand{\vm}[0]{\vect{m}}
\newcommand{\vu}[0]{\vect{u}}
\newcommand{\vL}[0]{\vect{L}}
\newcommand{\vr}[0]{\vect{r}}
\newcommand{\mW}[0]{\matr{W}}
\newcommand{\mG}[0]{\matr{G}}
\newcommand{\mX}[0]{\matr{X}}
\newcommand{\mQ}[0]{\matr{Q}}
\newcommand{\mU}[0]{\matr{U}}
\newcommand{\mV}[0]{\matr{V}}
\newcommand{\mA}{\matr{A}}
\newcommand{\mD}{\matr{D}}
\newcommand{\mS}{\matr{S}}
\newcommand{\mI}{\matr{I}}
\newcommand{\td}[0]{\text{d}}
\newcommand{\TT}[0]{\vects{\theta}}
\newcommand{\vepsilon}[0]{\vects{\epsilon}}
\newcommand{\vsigma}[0]{\vects{\sigma}}
\newcommand{\valpha}[0]{\vects{\alpha}}
\newcommand{\vmu}[0]{\vects{\mu}}
\newcommand{\vzero}[0]{\vect{0}}
\newcommand{\tf}[0]{\text{m}}
\newcommand{\tdf}[0]{\text{dm}}
\newcommand{\grad}[0]{\nabla}
\newcommand{\alert}[1]{\textcolor{red}{#1}}
\newcommand{\N}[0]{\mathcal{N}}
\newcommand{\LL}[0]{\mathcal{L}}
\newcommand{\MM}[0]{\mathcal{M}}
\newcommand{\NN}[0]{\mathcal{N}}
\newcommand{\HH}[0]{\mathcal{H}}
\newcommand{\RR}[0]{\mathbb{R}}
\newcommand{\II}[0]{\mathbb{I}}
\newcommand{\Scal}[0]{\mathcal{S}}
\newcommand{\sigmoid}{\sigma}
\newcommand{\E}[0]{\mathbb{E}}
\newcommand{\enabla}[0]{\ensuremath{%
    \overset{\raisebox{-0.3ex}[0.5ex][0ex]{%
    \ensuremath{\scriptscriptstyle e}}}{\nabla}}}
\newcommand{\enhnabla}[0]{\nabla_{\hspace{-0.5mm}e}\,}
\newcommand{\tred}[1]{\textcolor{red}{#1}}
\newcommand{\todo}[1]{{\Large\textcolor{red}{#1}}}
\newcommand{\done}[1]{{\Large\textcolor{green}{#1}}}
\newcommand{\dd}[1]{\ensuremath{\mbox{d}#1}}

\DeclareMathOperator*{\argmax}{\arg \max}
\DeclareMathOperator*{\argmin}{\arg \min}
\newcommand{\newln}{\\&\quad\quad{}}

\newcommand{\Ax}{\mathcal{A}_x}
\newcommand{\Ay}{\mathcal{A}_y}
\newcommand{\ola}{\overleftarrow}
\newcommand{\ora}{\overrightarrow}
\newcommand{\ov}{\overline}
\newcommand{\ts}{\rule{0pt}{2.6ex}}       % Top strut
\newcommand{\ms}{\rule{0pt}{0ex}}         % Middle strut
\newcommand{\bs}{\rule[-1.2ex]{0pt}{0pt}} % Bottom strut
\newcommand{\specialcell}[2][c]{%
  \begin{tabular}[#1]{@{}c@{}}#2\end{tabular}}

\graphicspath{{./figures/}}
\renewcommand{\bibname}{References}

%\setlength{\oddsidemargin}{0in}
%\setlength{\evensidemargin}{0in}
%\setlength{\textwidth}{6.5in}
%\setlength{\topmargin}{-.3in}
%\setlength{\textheight}{9in}
%\setlength{\parindent}{0pt}
%\pagestyle{empty}
%\documentstyle[nips14submit_09,times,art10]{article} % For LaTeX 2.09


\title{A Recurrent Latent Variable Model \\ for Sequential Data}

\author{
%David S.~Hippocampus\thanks{ Use footnote for providing further information
%about author (webpage, alternative address)---\emph{not} for acknowledging
%funding agencies.} \\
%Department of Computer Science\\
%Cranberry-Lemon University\\
%Pittsburgh, PA 15213 \\
%\texttt{hippo@cs.cranberry-lemon.edu} \\
%Junyoung Chung, Kyle Kastner, Laurent Dinh, Kratarth Goel,\\
%{\bf Aaron Courville, Yoshua Bengio$^*$}\\
Junyoung Chung,\; Kyle Kastner,\; Laurent Dinh,\; Kratarth Goel,\\
{\bf Aaron Courville,\; Yoshua Bengio$^*$}\\
Department of Computer Science and Operations Research\\
Universit\'{e} de Montr\'{e}al\\
%University of Montreal\\
$^*$CIFAR Senior Fellow\\
\texttt{\{firstname.lastname\}@umontreal.ca}\\
%\And
%Coauthor \\
%Affiliation \\
%Address \\
%\texttt{email} \\
%\AND
%Coauthor \\
%Affiliation \\
%Address \\
%\texttt{email} \\
%\And
%Coauthor \\
%Affiliation \\
%Address \\
%\texttt{email} \\
%\And
%Coauthor \\
%Affiliation \\
%Address \\
%\texttt{email} \\
%(if needed)\\
}

% The \author macro works with any number of authors. There are two commands
% used to separate the names and addresses of multiple authors: \And and \AND.
%
% Using \And between authors leaves it to \LaTeX{} to determine where to break
% the lines. Using \AND forces a linebreak at that point. So, if \LaTeX{}
% puts 3 of 4 authors names on the first line, and the last on the second
% line, try using \AND instead of \And before the third author name.

\newcommand{\fix}{\marginpar{FIX}}
\newcommand{\new}{\marginpar{NEW}}

\nipsfinalcopy % Uncomment for camera-ready version

\begin{document}


\maketitle

\begin{abstract}
In this paper, we explore the inclusion of latent random variables into the hidden state
of a recurrent neural network (RNN) by combining the elements of the variational autoencoder.
We argue that through the use of high-level latent random variables,
the {\it variational RNN} (VRNN)\footnote{Code is available at \url{http://www.github.com/jych/nips2015_vrnn}}
can model the kind of variability observed in highly structured sequential data such as natural speech.
We empirically evaluate the proposed model against other related sequential models on four speech datasets and one handwriting dataset.
Our results show the important roles that latent random variables can play in the RNN dynamics.
\end{abstract}


\section{Introduction}
\label{sec:intro}

Learning generative models of sequences is a long-standing machine learning
challenge and historically the domain of dynamic Bayesian networks (DBNs)
such as hidden Markov models (HMMs) and Kalman filters.
The dominance of DBN-based approaches has been recently overturned by
a resurgence of interest in recurrent neural network (RNN) based approaches.
An RNN is a special type of neural network that is
able to handle both variable-length input and output. By training an RNN
to predict the next output in a sequence, given all previous outputs,
it can be used to model joint probability distribution over sequences.

Both RNNs and DBNs consist of two parts: (1) a transition function that determines
the evolution of the internal hidden state,
and (2) a mapping from the state to the output.
There are, however, a few important differences between RNNs and DBNs.

DBNs have typically been limited either to relatively simple state transition structures (e.g., linear models in the case of the Kalman filter) or to relatively
simple internal state structure (e.g., the HMM state space consists of a single set of mutually exclusive states). RNNs, on the other hand, typically possess
both a richly distributed internal state representation and flexible non-linear transition functions.
These differences give RNNs extra expressive power in comparison to DBNs.
This expressive power and the ability to train via error backpropagation are
the key reasons why RNNs have gained popularity as generative models for highly structured sequential data.

In this paper, we focus on another important difference between DBNs and RNNs.
While the hidden state in DBNs is expressed in terms of random variables,
the internal transition structure of the standard RNN is entirely deterministic.
The only source of randomness or variability in the RNN is found in the
conditional output probability model. We suggest that this can be an
inappropriate way to model the kind of variability observed in
highly structured data, such as natural speech, which is characterized
by strong and complex dependencies among the output variables at different timesteps.
We argue, as have others \citep{boulanger2012modeling,Bayer-et-al-arXiv-2014},
that these complex dependencies cannot be modelled efficiently by the output probability models used in standard RNNs,
which include either a simple unimodal distribution or a mixture of unimodal distributions.

We propose the use of high-level latent random variables to model the variability observed in the data.
In the context of standard neural network models for non-sequential data, the variational autoencoder (VAE)~\citep{Kingma+Welling-ICLR2014, rezende2014stochastic}
offers an interesting combination of highly flexible non-linear mapping between
the latent random state and the observed output and effective approximate inference.
In this paper, we propose to extend the VAE into a recurrent framework for modelling high-dimensional sequences.
The VAE can model complex multimodal distributions, which will help when the underlying true data distribution
consists of multimodal conditional distributions. We call this model a {\it variational RNN} (VRNN).

A natural question to ask is: how do we encode observed variability via latent random variables?
The answer to this question depends on the nature of the data itself.
In this work, we are mainly interested in highly structured data
that often arises in AI applications. By highly structured,
we mean that the data is characterized by two properties.
Firstly, there is a relatively high signal-to-noise ratio,
meaning that the vast majority of the variability observed in the data is due to the
signal itself and cannot reasonably be considered as noise.
Secondly, there exists a complex relationship between the underlying factors
of variation and the observed data. For example, in speech, the vocal qualities of the speaker
have a strong but complicated influence on the audio waveform,
affecting the waveform in a consistent manner across frames.

With these considerations in mind, we suggest that our model variability should induce \emph{temporal dependencies across timesteps}.
Thus, like DBN models such as HMMs and Kalman filters, we model the dependencies between the latent random variables across timesteps.
While we are not the first to propose integrating random variables into the RNN hidden state~\citep{boulanger2012modeling,Bayer-et-al-arXiv-2014,fabius2014variational,gregor2015draw},
we believe we are the first to integrate the dependencies between the latent random variables at neighboring timesteps.

We evaluate the proposed VRNN model against other RNN-based models
-- including a VRNN model without introducing temporal dependencies between the latent random variables --
on two challenging sequential data types: natural speech and handwriting.
We demonstrate that for the speech modelling tasks, the VRNN-based models significantly outperform the RNN-based models and
the VRNN model that does not integrate temporal dependencies between latent random variables.

\section{Background}
\label{sec:background}
\subsection{Sequence modelling with Recurrent Neural Networks}
An RNN can take as input a variable-length sequence $\vx=(\vx_1, \vx_2, \ldots, \vx_T)$ by recursively
processing each symbol while maintaining its internal hidden state $\vh$. At each
timestep $t$, the RNN reads the symbol $\vx_t \in \RR^d$ and updates its hidden state $\vh_t \in \RR^p$ by:
\begin{align}
    \label{eq:rnn_generic}
    \vh_t =& f_{\theta}\left(\vx_t, \vh_{t-1}\right),
\end{align}
where $f$ is a deterministic non-linear transition function, and $\theta$ is the parameter set of $f$.
The transition function $f$ can be implemented with gated activation functions such as long short-term memory~\citep[LSTM,][]{Hochreiter+Schmidhuber-1997}
or gated recurrent unit~\citep[GRU,][]{cho2014emnlp}.
RNNs model sequences by parameterizing a factorization of
the joint sequence probability distribution as a product of conditional probabilities such that:
\begin{align}
    p(\vx_1, \vx_2, \ldots, \vx_T) &= \prod_{t=1}^T p(\vx_t \mid \vx_{< t}), \nonumber \\
    p(\vx_t \mid \vx_{< t}) &= g_{\tau}(\vh_{t-1}), \label{eq:rnn_output}
\end{align}
where $g$ is a function that maps the RNN hidden state $\vh_{t-1}$ to a probability distribution over possible outputs,
and $\tau$ is the parameter set of $g$.

One of the main factors that determines the representational power of an RNN
is the output function $g$ in Eq.~\eqref{eq:rnn_output}.
With a deterministic transition function $f$, the choice of $g$ effectively defines the
family of joint probability distributions $p(\vx_1, \ldots, \vx_T)$ that can be expressed by the RNN.

We can express the output function $g$ in Eq.~\eqref{eq:rnn_output} as being composed of two parts. The
first part $\varphi_{\tau}$ is a function that returns the parameter set $\phi_t$ given the hidden state
$\vh_{t-1}$, i.e., $\phi_t = \varphi_{\tau}(\vh_{t-1})$, while the second part of $g$ returns the density of $\vx_t$, i.e., $p_{\phi_t}(\vx_t \mid \vx_{< t})$.

When modelling high-dimensional and real-valued sequences, a reasonable choice of an observation model is a Gaussian mixture model (GMM) as used in \citep{graves2013generating}.
For GMM, $\varphi_{\tau}$ returns a set of mixture coefficients $\alpha_t$,
means $\vmu_{\cdot, t}$ and covariances $\Sigma_{\cdot, t}$ of the corresponding mixture components.
The probability of $\vx_t$ under the mixture distribution is:
\begin{align*}
    p_{\valpha_t,\vmu_{\cdot, t},\Sigma_{\cdot, t}}(\vx_t \mid \vx_{<t}) = \sum_{j} \alpha_{j,t} \NN\left(
        \vx_t ; \vmu_{j,t}, \Sigma_{j,t}
    \right).
\end{align*}
With the notable exception of \citep{graves2013generating}, there has been little work
investigating the structured output density model for RNNs with real-valued sequences.

There is potentially a significant issue in the way the RNN models output variability.
Given a deterministic transition function, the only source of variability is in the
conditional output probability density. This can present problems when modelling sequences that are
at once highly variable and highly structured (i.e., with a high signal-to-noise ratio).
To effectively model these types of sequences, the RNN must be capable of mapping very small
variations in $\vx_{t}$ (i.e., the only source of randomness) to potentially very large variations
in the hidden state $\vh_{t}$. Limiting the capacity of the network, as must be done to guard against overfitting, will force a compromise between the generation of
a clean signal and encoding sufficient input variability to capture the high-level variability both within a single observed
sequence and across data examples.

The need for highly structured output functions in an RNN has been previously noted.
\citet{boulanger2012modeling} extensively tested NADE and RBM-based output densities for modelling
sequences of binary vector representations of music.
\citet{Bayer-et-al-arXiv-2014} introduced a sequence of independent latent
variables corresponding to the states of the RNN. Their model, called {\it STORN},
first generates a sequence of samples $\vz=(\vz_1, \ldots, \vz_T)$ from the sequence of independent latent
random variables. At each timestep, the transition function $f$ from
Eq.~\eqref{eq:rnn_generic} computes the next hidden state $\vh_t$ based on
the previous state $\vh_{t-1}$, the previous output $\vx_{t-1}$ and the sampled latent random variables $\vz_t$.
They proposed to train this model based on the VAE principle (see Sec.~\ref{sec:vari}).
Similarly, \citet{Pachitariu2012learning} earlier proposed both a sequence of independent latent random variables and a stochastic hidden state for the RNN.

These approaches are closely related to the approach proposed in this paper.
However, there is a major difference in how the prior distribution over the
latent random variable is modelled. Unlike the aforementioned approaches, our approach makes the prior
%latent random variable $\vz$ is modelled. Unlike the aforementioned approaches, our approach makes the prior
distribution of the latent random variable at timestep $t$ dependent on all the preceding
inputs via the RNN hidden state $\vh_{t-1}$ (see Eq.~\eqref{eq:vrnn_prior}). The introduction of
temporal structure into the prior distribution is expected to improve the representational power of
the model, which we empirically observe in the experiments (See Table~\ref{tab:result1}).
However, it is important to note that any approach based on having stochastic latent
state is orthogonal to having a structured output function, and that these two can be used together to form a single model.


\subsection{Variational Autoencoder}
\label{sec:vari}
For non-sequential data, VAEs~\citep{Kingma+Welling-ICLR2014,rezende2014stochastic} have
recently been shown to be an effective modelling paradigm to recover complex multimodal distributions over the data space.
A VAE introduces a set of latent random variables $\vz$, designed to capture the variations in the observed variables $\vx$.
As an example of a directed graphical model, the joint distribution is defined as:
\begin{align}
    \label{eq:bayes}
    p(\vx, \vz) = p(\vx \mid \vz)p(\vz).
\end{align}
The prior over the latent random variables, $p(\vz)$,
is generally chosen to be a simple Gaussian distribution and the conditional $p(\vx \mid \vz)$
is an arbitrary observation model whose parameters are computed by a parametric function of $\vz$.
Importantly, the VAE typically parameterizes $p(\vx \mid \vz)$
with a highly flexible function approximator such as a neural network.
While latent random variable models of the form given in Eq.~\eqref{eq:bayes} are not uncommon,
endowing the conditional $p(\vx \mid \vz)$ as a potentially highly non-linear
mapping from $\vz$ to $\vx$ is a rather unique feature of the VAE.

However, introducing a highly non-linear mapping from $\vz$ to $\vx$ results
in intractable inference of the posterior $p(\vz \mid \vx)$. Instead, the VAE uses
a variational approximation $q(\vz \mid \vx)$ of the posterior that enables the
use of the lower bound:
\begin{align}
    \label{eq:lowerbound}
    \log p(\vx) &\geq -\mathrm{KL}(q(\vz \mid \vx) \| p(\vz)) + \E_{q(\vz \mid \vx)}\left[\log p(\vx \mid \vz)\right],
\end{align}
where $\mathrm{KL}(Q \| P)$ is Kullback-Leibler divergence between two
distributions $Q$ and $P$.

In \citep{Kingma+Welling-ICLR2014}, the approximate posterior $q(\vz \mid \vx)$
is a Gaussian $\NN(\vmu, \text{diag}(\vsigma^2))$ whose mean $\vmu$ and
variance $\vsigma^2$ are the output of a highly non-linear function of
$\vx$, once again typically a neural network.

The generative model $p(\vx \mid \vz)$ and inference model $q(\vz \mid \vx)$ are
then trained jointly by maximizing the variational lower bound with respect to
their parameters, where the integral with respect to $q(\vz \mid \vx)$
is approximated stochastically. The gradient of this estimate can have a low variance
estimate, by reparametrizing $\vz = \vmu + \vsigma \odot \vepsilon$ and rewriting:
\begin{align*}
    \E_{q(\vz \mid \vx)}\left[\log p(\vx \mid \vz) \right] =
    \E_{p(\boldsymbol\epsilon)}\left[\log p(\vx \mid \vz =
    \boldsymbol\mu + \boldsymbol\sigma \odot \boldsymbol\epsilon) \right],
\end{align*}
where $\vepsilon$ is a vector of standard Gaussian variables.
The inference model can then be trained through standard backpropagation technique
for stochastic gradient descent.

\section{Variational Recurrent Neural Network}
\label{sec:vrnn}

In this section, we introduce a recurrent version of the VAE
for the purpose of modelling sequences. Drawing inspiration from simpler
dynamic Bayesian networks (DBNs) such as HMMs and Kalman filters, the proposed
{\it variational recurrent neural network} (VRNN) explicitly models the dependencies between latent random variables across subsequent timesteps.
However, unlike these simpler DBN models,
the VRNN retains the flexibility to model highly non-linear dynamics.

\paragraph{Generation}
The VRNN contains a VAE at every timestep. However, these
VAEs are conditioned on the state variable $\vh_{t-1}$ of an RNN.
This addition will help the VAE to take into account the temporal
structure of the sequential data. Unlike a standard VAE,
the prior on the latent random variable is no longer a standard Gaussian distribution, but follows the distribution:
\begin{align}
    \label{eq:vrnn_prior}
    \vz_t \sim \NN(\vmu_{0,t}, \text{diag}(\vsigma_{0,t}^2))\text{ , where  }
    [\vmu_{0,t}, \vsigma_{0,t}] = \varphi_{\tau}^{\text{prior}} (\vh_{t-1}),
\end{align}
where $\boldsymbol\mu_{0,t}$ and $\boldsymbol\sigma_{0,t}$ denote the parameters of
the conditional prior distribution.
Moreover, the generating distribution will not only be conditioned on $\vz_t$ but also on $\vh_{t-1}$ such that:
\begin{align}
    \label{eq:vrnn_gen}
    \vx_t \mid \vz_t \sim \NN(\vmu_{x,t}, \text{diag}(\vsigma_{x,t}^2))\text{ , where }
    [\vmu_{x,t}, \vsigma_{x,t}] = \varphi_{\tau}^{\text{dec}}(\varphi_{\tau}^{\vz}(\vz_t), \vh_{t-1}),
\end{align}
where $\boldsymbol\mu_{x,t}$ and $\boldsymbol\sigma_{x,t}$ denote the parameters of
the generating distribution, $\varphi_{\tau}^{\text{prior}}$ and $\varphi_{\tau}^{\text{dec}}$ can be
any highly flexible function such as neural networks.
$\varphi_{\tau}^{\vx}$ and $\varphi_{\tau}^{\vz}$ can also be neural networks, which extract features from $\vx_t$ and $\vz_t$, respectively.
We found that these feature extractors are crucial for learning complex sequences.
The RNN updates its hidden state using the recurrence equation:
\begin{align}
    \label{eq:vrnn_trans}
    \vh_t =& f_{\theta}\left(\varphi_{\tau}^{\vx}(\vx_t), \varphi_{\tau}^{\vz}(\vz_{t}), \vh_{t-1}\right),
\end{align}
where $f$ was originally the transition function from Eq.~\eqref{eq:rnn_generic}.
From Eq.~\eqref{eq:vrnn_trans}, we find that $\vh_{t}$ is a function of $\vx_{\leq t}$ and $\vz_{\leq t}$. Therefore,
Eq.~\eqref{eq:vrnn_prior} and Eq.~\eqref{eq:vrnn_gen} define the distributions $p(\vz_t \mid \vx_{< t}, \vz_{< t})$ and $p(\vx_t \mid \vz_{\leq t}, \vx_{<t})$, respectively.
The parameterization of the generative model results in and -- was motivated by -- the factorization:
\begin{align}
    \label{eq:joint_fact}
    p(\vx_{\leq T}, \vz_{\leq T}) = \prod_{t=1}^{T}{ p(\vx_t \mid \vz_{\leq t}, \vx_{<t})p(\vz_t \mid \vx_{< t}, \vz_{< t}) }.
\end{align}

\paragraph{Inference}
In a similar fashion, the approximate posterior will not only be a function of $\vx_t$ but also of $\vh_{t-1}$ following the equation:
\begin{align}
    \label{eq:vrnn_posterior}
    \vz_t \mid \vx_t \sim \NN(\vmu_{z,t}, \text{diag}(\vsigma_{z,t}^2))\text{ , where }
    [\vmu_{z,t}, \vsigma_{z,t}] = \varphi_{\tau}^{\text{enc}}(\varphi_{\tau}^{\vx}(\vx_t), \vh_{t-1}),
\end{align}
similarly $\boldsymbol\mu_{z,t}$ and $\boldsymbol\sigma_{z,t}$ denote the parameters of the approximate posterior.
We note that the encoding of the approximate posterior and the decoding for generation are tied through the RNN hidden state $\vh_{t-1}$.
We also observe that this conditioning on $\vh_{t-1}$ results in the factorization:
\begin{align}
    \label{eq:q_fact}
    q(\vz_{\leq T} \mid \vx_{\leq T}) = \prod_{t=1}^{T}{q(\vz_{t} \mid \vx_{\leq t}, \vz_{< t})}.
\end{align}

\begin{figure*}
    \centering
    \begin{minipage}{0.16\textwidth}
        \centering
        \includegraphics[width=1.0\columnwidth]{eq_prior.pdf}
    \end{minipage}
    \hfill
    \begin{minipage}{0.20\textwidth}
        \centering
        \includegraphics[width=1.0\columnwidth]{eq_gen.pdf}
    \end{minipage}
    \hfill
    \begin{minipage}{0.16\textwidth}
        \centering
        \includegraphics[width=1.0\columnwidth]{eq_rec.pdf}
    \end{minipage}
    \hfill
    \begin{minipage}{0.16\textwidth}
        \centering
        \includegraphics[width=1.0\columnwidth]{eq_infer.pdf}
    \end{minipage}
    \hfill
    \begin{minipage}{0.20\textwidth}
        \centering
        \includegraphics[width=1.0\columnwidth]{eq_all.pdf}
    \end{minipage}
    \vspace{0.5cm}
    \vfill
    \begin{minipage}{0.16\textwidth}
        \centering
        (a) Prior
    \end{minipage}
    \hfill
    \begin{minipage}{0.20\textwidth}
        \centering
        (b) Generation
    \end{minipage}
    \hfill
    \begin{minipage}{0.16\textwidth}
        \centering
        (c) Recurrence
    \end{minipage}
    \hfill
    \begin{minipage}{0.16\textwidth}
        \centering
        (d) Inference
    \end{minipage}
    \hfill
    \begin{minipage}{0.20\textwidth}
        \centering
        (e) Overall
    \end{minipage}
    \caption{Graphical illustrations of each operation of the VRNN:
             (a) computing the conditional prior using Eq.~\eqref{eq:vrnn_prior};
             (b) generating function using Eq.~\eqref{eq:vrnn_gen};
             (c) updating the RNN hidden state using Eq.~\eqref{eq:vrnn_trans};
             (d) inference of the approximate posterior using Eq.~\eqref{eq:vrnn_posterior};
             (e) overall computational paths of the VRNN.}
    \label{fig:rnn_struct}
\end{figure*}

\paragraph{Learning}
The objective function becomes a timestep-wise variational lower bound using Eq.~\eqref{eq:joint_fact} and Eq.~\eqref{eq:q_fact}:
%This factorization is crucial in breaking the variational lower bound into timestep-wise
\begin{align}
    \label{eq:objective_function} 
    %\int{\log\frac{p(\vx_{\leq T}, \vz_{\leq T})}{q(\vz_{\leq T} \mid \vx_{\leq T})}dq(\vz_{\leq T} \mid \vx_{\leq T})} =&
    %\sum_{t=1}^{T}{-\mathrm{KL}(q(\vz_{t} \mid \vx_{\leq t}, \vz_{< t}) \| p(\vz_{t} \mid \vx_{< t}, \vz_{< t}))} \\
    %&~~~~~~{+ \E_{q(\vz_{t} \mid \vx_{\leq t}, \vz_{< t})}\left[\log p(\vx_{t} \mid \vz_{\leq t}, \vx_{< t})\right]}.
    %\sum_{t=1}^{T}{-\mathrm{KL}(q(\vz_{t} \mid \vx_{\leq t}, \vz_{< t}) \| p(\vz_{t} \mid \vx_{< t}, \vz_{< t}))} + \E_{q(\vz_{t} \mid \vx_{\leq t}, \vz_{< t})}\left[\log p(\vx_{t} \mid \vz_{\leq t}, \vx_{< t})\right].
    \E_{q(\vz_{\leq T}\mid\vx_{\leq T})}\left[\sum_{t=1}^{T}\left({-\mathrm{KL}(q(\vz_{t} \mid \vx_{\leq t}, \vz_{< t}) \| p(\vz_{t} \mid \vx_{< t}, \vz_{< t}))} + \log p(\vx_{t} \mid \vz_{\leq t}, \vx_{< t})\right)\right].
\end{align}
As in the standard VAE, we learn the generative and inference models
jointly by maximizing the variational lower bound with respect to their parameters.
The schematic view of the VRNN is shown in Fig.~\ref{fig:rnn_struct},
operations (a)--(d) correspond to Eqs.~\eqref{eq:vrnn_prior}--\eqref{eq:vrnn_trans}, \eqref{eq:vrnn_posterior}, respectively.
The VRNN applies the operation (a) when computing the conditional prior (see Eq.~\eqref{eq:vrnn_prior}).
If the variant of the VRNN (VRNN-I) does not apply the operation (a), then the prior becomes independent across timesteps.
STORN~\citep{Bayer-et-al-arXiv-2014} can be considered as an instance of the VRNN-I model family.
In fact, STORN puts further restrictions on the dependency structure of the approximate inference model.
We include this version of the model (VRNN-I) in our experimental evaluation in order to directly study the impact
of including the temporal dependency structure in the prior (i.e., conditional prior) over the latent random variables.

\section{Experiment Settings}
\label{sec:settings}

We evaluate the proposed VRNN model on two tasks:
(1) modelling natural speech directly from the raw audio waveforms;
(2) modelling handwriting generation.

\paragraph{Speech modelling}
We train the models to directly model raw audio signals, represented as a sequence of 200-dimensional frames.
Each frame corresponds to the real-valued amplitudes of 200 consecutive raw acoustic samples.
Note that this is unlike the conventional approach for modelling speech, often used in speech synthesis where models are expressed over
representations such as spectral features~\citep[see, e.g.,][]{Tokuda-et-al-2013,bertrand2008unsupervised_b,lee2009unsupervised}.

We evaluate the models on the following four speech datasets:
\begin{enumerate}
    \itemsep 0em
    \item {\bf Blizzard}: This text-to-speech dataset made available by the Blizzard
        Challenge 2013 contains 300 hours of English, spoken by a single female
        speaker~\citep{King2013}.
    \item {\bf TIMIT}: This widely used dataset for benchmarking speech
        recognition systems contains $6,300$ English sentences, read by 630
        speakers.
    \item {\bf Onomatopoeia}\footnote{
            This dataset has been provided by Ubisoft.
        }: This is a set of $6,738$ non-linguistic human-made
        sounds such as coughing, screaming, laughing and shouting, recorded from
        51 voice actors.
    \item {\bf Accent}: This dataset contains English paragraphs read by
        $2,046$ different native and non-native English speakers~\citep{speech2015accent}.
\end{enumerate}
For the Blizzard and Accent datasets, we
process the data so that each sample duration is $0.5s$ (the sampling frequency used is $16\mathrm{kHz}$).
Except the TIMIT dataset, the rest of the datasets do not have predefined train/test splits.
We shuffle and divide the data into train/validation/test splits using a ratio of $0.9/0.05/0.05$.
%We use {\it truncated backpropagation through time} and
%initialize the hidden state of the RNN part with the final hidden state of previous minibatch, resetting to a zero-vector every four updates.
%Excluding TIMIT dataset, the rest of the datasets do not have predefined train/test splits.
%We shuffle and divide the data into train/validation/test splits using ratio of $0.9/0.05/0.05$.

\paragraph{Handwriting generation}
We let each model learn a sequence of $(x,y)$ coordinates together with binary
indicators of pen-up/pen-down, using the IAM-OnDB dataset, which consists of $13,040$ handwritten lines written by $500$
writers~\cite{liwicki2005iam_b}. We preprocess and split the dataset as
done in \citep{graves2013generating}.

\begin{table}[t]
    \caption{Average log-likelihood on the test (or validation) set of each task.}
    \vfill
    \label{tab:result1}
    \centering
    \begin{tabular}{c || c | c | c | c | c}
        \hline
        & \multicolumn{4}{c|}{Speech modelling} & Handwriting \\
        \cline{2-6}
        Models & Blizzard & TIMIT & Onomatopoeia & Accent & IAM-OnDB \\
        \hline
        \hline
        RNN-Gauss    & 3539               & -1900               & -984                & -1293              & 1016  \\
        \hline
        RNN-GMM      & 7413               & 26643               & 18865               & 3453               & 1358  \\
        \hline
        VRNN-I-Gauss & $\geq 8933$        & $\geq 28340$        & $\geq 19053$        & $\geq 3843$        & $\geq 1332$  \\
                     & $\approx 9188$     & $\approx 29639$     & $\approx 19638$     & $\approx 4180$     & $\approx 1353$  \\
        \hline
        VRNN-Gauss   & $\geq 9223$        & $\geq 28805$        & $\geq 20721$        & $\geq 3952$        & $\geq$ 1337  \\
                     & $\approx \bf 9516$ & $\approx \bf 30235$ & $\approx \bf 21332$ & $\approx 4223$     & $\approx$ 1354  \\
        \hline
        VRNN-GMM     & $\geq 9107$        & $\geq 28982$        & $\geq 20849$        & $\geq 4140$        & $\geq$ 1384  \\
                     & $\approx 9392$     & $\approx 29604$     & $\approx 21219$     & $\approx \bf 4319$ & $\approx$ \bf 1384  \\
        \hline
    \end{tabular}
\end{table}

\paragraph{Preprocessing and training}
The only preprocessing used in our experiments is normalizing each sequence using the global mean and standard deviation computed from the entire training set.
We train each model with stochastic gradient descent on the negative log-likelihood using the Adam optimizer~\cite{kingma2015adam},
with a learning rate of $0.001$ for TIMIT and Accent and $0.0003$ for the rest.
We use a minibatch size of $128$ for Blizzard and Accent and $64$ for the rest.
The final model was chosen with early-stopping based on the validation performance.

\paragraph{Models}
We compare the VRNN models with the standard RNN models using two different output functions:
a simple Gaussian distribution (Gauss) and a Gaussian mixture model (GMM).
For each dataset, we conduct an additional set of experiments for a VRNN model without the conditional prior (VRNN-I).

We fix each model to have a single recurrent hidden layer with $2000$ LSTM units (in the case of Blizzard, $4000$ and for IAM-OnDB, $1200$).
All of $\varphi_{\tau}$ shown in Eqs.~\eqref{eq:vrnn_prior}--\eqref{eq:vrnn_trans}, \eqref{eq:vrnn_posterior}
have four hidden layers using rectified linear units~\citep{nair2010rectified} (for IAM-OnDB, we use a single hidden layer).
The standard RNN models only have $\varphi_{\tau}^{\vx}$ and $\varphi_{\tau}^{\text{dec}}$,
while the VRNN models also have $\varphi_{\tau}^{\vz}$, $\varphi_{\tau}^{\text{enc}}$ and $\varphi_{\tau}^{\text{prior}}$.
For the standard RNN models, $\varphi_{\tau}^{\vx}$ is the feature extractor, and $\varphi_{\tau}^{\mathrm{dec}}$
is the generating function.
For the RNN-GMM and VRNN models, we match the total number of parameters of the deep neural networks (DNNs),
$\varphi_{\tau}^{\vx,\vz,\text{enc},\text{dec},\text{prior}}$, as close to the RNN-Gauss model having $600$ hidden units for every layer
that belongs to either $\varphi_{\tau}^{\mathbf{x}}$ or $\varphi_{\tau}^{\mathrm{dec}}$ (we consider $800$ hidden units in the case of Blizzard).
Note that we use $20$ mixture components for models using a GMM as the output function.

For qualitative analysis of speech generation, we train larger models to generate audio sequences.
We stack three recurrent hidden layers, each layer contains $3000$ LSTM units.
Again for the RNN-GMM and VRNN models, we match the total number of parameters of the DNNs to be equal to the RNN-Gauss model having $3200$ hidden units
for each layer that belongs to either $\varphi_{\tau}^{\mathbf{x}}$ or $\varphi_{\tau}^{\mathrm{dec}}$.

\section{Results and Analysis}
\label{sec:results}
We report the average log-likelihood of test examples assigned by each model in Table~\ref{tab:result1}.
For RNN-Gauss and RNN-GMM, we report the exact log-likelihood, while in the case of VRNNs, we report the variational
lower bound (given with $\geq$ sign, see Eq.~\eqref{eq:lowerbound}) and approximated marginal
log-likelihood (given with $\approx$ sign) based on importance sampling using $40$ samples as in \citep{rezende2014stochastic}.
In general, higher numbers are better.
Our results show that the VRNN models have higher log-likelihood, which support our claim that latent random variables are helpful when modelling complex sequences.
The VRNN models perform well even with a unimodal output function (VRNN-Gauss), which is not the case for the standard RNN models.

\paragraph{Latent space analysis}
In Fig.~\ref{fig:latent_anal}, we show an analysis of the latent random variables.
We let a VRNN model read some unseen examples and observe the transitions in the latent space.
We compute $\delta_t=\sum_j(\vmu_{z,t}^j-\vmu_{z,t-1}^j)^2$ at every timestep and plot the results on the top row of Fig.~\ref{fig:latent_anal}.
The middle row shows the $\mathrm{KL}$ divergence computed between the approximate posterior and the conditional prior.
When there is a transition in the waveform, the $\mathrm{KL}$ divergence tends to grow (white is high), and we can clearly observe a peak in $\delta_t$ that can
affect the RNN dynamics to change modality.

\begin{figure}
    \begin{minipage}{1.\textwidth}
        \centering
        \begin{minipage}[b]{0.46\columnwidth}
            \centering
            \includegraphics[width=1.\columnwidth,height=3.0cm,clip=true]{lat_1.pdf}
        \end{minipage}
        \begin{minipage}[b]{0.46\columnwidth}
            \centering
            \includegraphics[width=1.\columnwidth,height=3.0cm,clip=true]{lat_2.pdf}
        \end{minipage}
    \end{minipage}
    \caption{The top row represents the difference $\delta_t$ between $\vmu_{z,t}$ and $\vmu_{z,t-1}$.
             The middle row shows the dominant $\mathrm{KL}$ divergence values in temporal order.
             The bottom row shows the input waveforms.}
    \label{fig:latent_anal}
\end{figure}

\paragraph{Speech generation}
\begin{figure*}[ht]
    \begin{minipage}{1.\textwidth}
        \centering
        \begin{minipage}[b]{0.32\columnwidth}
            \centering
            \includegraphics[width=1.\columnwidth,height=1.6cm,clip=true]{gt_1.pdf}
        \end{minipage}
        \begin{minipage}[b]{0.32\columnwidth}
            \centering
            \includegraphics[width=1.\columnwidth,height=1.6cm,clip=true]{rnngmm_1.pdf}
        \end{minipage}
        \begin{minipage}[b]{0.32\columnwidth}
            \centering
            \includegraphics[width=1.\columnwidth,height=1.6cm,clip=true]{vrnn_1.pdf}
        \end{minipage}
        \begin{minipage}[b]{0.32\columnwidth}
            \centering
            \includegraphics[width=1.\columnwidth,height=1.6cm,clip=true]{gt_2.pdf}
        \end{minipage}
        \begin{minipage}[b]{0.32\columnwidth}
            \centering
            \includegraphics[width=1.\columnwidth,height=1.6cm,clip=true]{rnngmm_2.pdf}
        \end{minipage}
        \begin{minipage}[b]{0.32\columnwidth}
            \centering
            \includegraphics[width=1.\columnwidth,height=1.6cm,clip=true]{vrnn_2.pdf}
        \end{minipage}
        \begin{minipage}[b]{0.32\columnwidth}
            \centering
            \includegraphics[width=1.\columnwidth,height=1.6cm,clip=true]{gt_3.pdf}
        \end{minipage}
        \begin{minipage}[b]{0.32\columnwidth}
            \centering
            \includegraphics[width=1.\columnwidth,height=1.6cm,clip=true]{rnngmm_3.pdf}
        \end{minipage}
        \begin{minipage}[b]{0.32\columnwidth}
            \centering
            \includegraphics[width=1.\columnwidth,height=1.6cm,clip=true]{vrnn_3.pdf}
        \end{minipage}
        \begin{minipage}[b]{0.32\columnwidth}
            \centering
            \includegraphics[width=1.\columnwidth,height=1.6cm,clip=true]{gt_zoom_1.pdf}
        \end{minipage}
        \begin{minipage}[b]{0.32\columnwidth}
            \centering
            \includegraphics[width=1.\columnwidth,height=1.6cm,clip=true]{rnngmm_zoom_1.pdf}
        \end{minipage}
        \begin{minipage}[b]{0.32\columnwidth}
            \centering
            \includegraphics[width=1.\columnwidth,height=1.6cm,clip=true]{vrnn_zoom_1.pdf}
        \end{minipage}
        \begin{minipage}[b]{0.32\columnwidth}
            \centering
            \includegraphics[width=1.\columnwidth,height=1.6cm,clip=true]{gt_zoom_2.pdf}
        \end{minipage}
        \begin{minipage}[b]{0.32\columnwidth}
            \centering
            \includegraphics[width=1.\columnwidth,height=1.6cm,clip=true]{rnngmm_zoom_2.pdf}
        \end{minipage}
        \begin{minipage}[b]{0.32\columnwidth}
            \centering
            \includegraphics[width=1.\columnwidth,height=1.6cm,clip=true]{vrnn_zoom_2.pdf}
        \end{minipage}
        \begin{minipage}[b]{0.32\columnwidth}
            \centering
            \includegraphics[width=1.\columnwidth,height=1.6cm,clip=true]{gt_zoom_3.pdf}
        \end{minipage}
        \begin{minipage}[b]{0.32\columnwidth}
            \centering
            \includegraphics[width=1.\columnwidth,height=1.6cm,clip=true]{rnngmm_zoom_3.pdf}
        \end{minipage}
        \begin{minipage}[b]{0.32\columnwidth}
            \centering
            \includegraphics[width=1.\columnwidth,height=1.6cm,clip=true]{vrnn_zoom_3.pdf}
        \end{minipage}
        \begin{minipage}{0.32\textwidth}
            \centering
            (a) Ground Truth
        \end{minipage}
        \begin{minipage}{0.32\textwidth}
            \centering
            (b) RNN-GMM
        \end{minipage}
        \begin{minipage}{0.32\textwidth}
            \centering
            (c) VRNN-Gauss
        \end{minipage}
    \end{minipage}
    \caption{Examples from the training set and generated samples from
             RNN-GMM and VRNN-Gauss. Top three rows show the global waveforms while
             the bottom three rows show more zoomed-in waveforms.
             Samples from (b) RNN-GMM contain high-frequency noise,
             and samples from (c) VRNN-Gauss have less noise.
             We exclude RNN-Gauss, because the samples are almost close to pure noise.}
    \label{fig:waveform_samples}
\end{figure*}

We generate waveforms with $2.0s$ duration from the models that were trained on Blizzard.
From Fig.~\ref{fig:waveform_samples}, we can clearly see that the waveforms from the
VRNN-Gauss are much less noisy and have less spurious peaks than those from the RNN-GMM.
We suggest that the large amount of noise apparent in the waveforms from the RNN-GMM model is a
consequence of the compromise these models must make between representing a clean signal
consistent with the training data and encoding sufficient input variability to capture the variations across data examples.
The latent random variable models can avoid this compromise by adding variability in the latent space, which can always be mapped to a point close to a relatively clean sample.

\paragraph{Handwriting generation}
Visual inspection of the generated handwriting (as shown in Fig.~\ref{fig:handwriting_samples})
from the trained models reveals that the VRNN model is able to generate
more diverse writing style while maintaining consistency within samples. 

\begin{figure*}[ht]
    \begin{minipage}{1.\textwidth}
        \centering
        \begin{minipage}[b]{0.24\columnwidth}
            \centering
            \includegraphics[width=0.95\columnwidth,height=0.5cm,clip=true]{cherrypicked_data_samples_0.png}
        \end{minipage}
        \hfill
        \begin{minipage}[b]{0.24\columnwidth}
            \centering
            \includegraphics[width=0.95\columnwidth,height=0.5cm,clip=true]{cherrypicked_m0_samples_0.png}
        \end{minipage}
        \hfill
        \begin{minipage}[b]{0.24\columnwidth}
            \centering
            \includegraphics[width=0.95\columnwidth,height=0.5cm,clip=true]{cherrypicked_m1_samples_0.png}
        \end{minipage}
        \hfill
        \begin{minipage}[b]{0.24\columnwidth}
            \centering
            \includegraphics[width=0.95\columnwidth,height=0.5cm,clip=true]{cherrypicked_m3_samples_0.png}
        \end{minipage}
        \begin{minipage}[b]{0.24\columnwidth}
            \centering
            \includegraphics[width=0.95\columnwidth,height=0.5cm,clip=true]{cherrypicked_data_samples_1.png}
        \end{minipage}
        \hfill
        \begin{minipage}[b]{0.24\columnwidth}
            \centering
            \includegraphics[width=0.95\columnwidth,height=0.5cm,clip=true]{cherrypicked_m0_samples_1.png}
        \end{minipage}
        \hfill
        \begin{minipage}[b]{0.24\columnwidth}
            \centering
            \includegraphics[width=0.95\columnwidth,height=0.5cm,clip=true]{cherrypicked_m1_samples_1.png}
        \end{minipage}
        \hfill
        \begin{minipage}[b]{0.24\columnwidth}
            \centering
            \includegraphics[width=0.95\columnwidth,height=0.5cm,clip=true]{cherrypicked_m3_samples_1.png}
        \end{minipage}
        \begin{minipage}[b]{0.24\columnwidth}
            \centering
            \includegraphics[width=0.95\columnwidth,height=0.5cm,clip=true]{cherrypicked_data_samples_2.png}
        \end{minipage}
        \hfill
        \begin{minipage}[b]{0.24\columnwidth}
            \centering
            \includegraphics[width=0.95\columnwidth,height=0.5cm,clip=true]{cherrypicked_m0_samples_2.png}
        \end{minipage}
        \hfill
        \begin{minipage}[b]{0.24\columnwidth}
            \centering
            \includegraphics[width=0.95\columnwidth,height=0.5cm,clip=true]{cherrypicked_m1_samples_2.png}
        \end{minipage}
        \hfill
        \begin{minipage}[b]{0.24\columnwidth}
            \centering
            \includegraphics[width=0.95\columnwidth,height=0.5cm,clip=true]{cherrypicked_m3_samples_2.png}
        \end{minipage}
        \begin{minipage}[b]{0.24\columnwidth}
            \centering
            \includegraphics[width=0.95\columnwidth,height=0.5cm,clip=true]{cherrypicked_data_samples_3.png}
        \end{minipage}
        \hfill
        \begin{minipage}[b]{0.24\columnwidth}
            \centering
            \includegraphics[width=0.95\columnwidth,height=0.5cm,clip=true]{cherrypicked_m0_samples_3.png}
        \end{minipage}
        \hfill
        \begin{minipage}[b]{0.24\columnwidth}
            \centering
            \includegraphics[width=0.95\columnwidth,height=0.5cm,clip=true]{cherrypicked_m1_samples_7.png}
        \end{minipage}
        \hfill
        \begin{minipage}[b]{0.24\columnwidth}
            \centering
            \includegraphics[width=0.95\columnwidth,height=0.5cm,clip=true]{cherrypicked_m3_samples_3.png}
        \end{minipage}
        \begin{minipage}[b]{0.24\columnwidth}
            \centering
            \includegraphics[width=0.95\columnwidth,height=0.5cm,clip=true]{cherrypicked_data_samples_4.png}
        \end{minipage}
        \hfill
        \begin{minipage}[b]{0.24\columnwidth}
            \centering
            \includegraphics[width=0.95\columnwidth,height=0.5cm,clip=true]{cherrypicked_m0_samples_4.png}
        \end{minipage}
        \hfill
        \begin{minipage}[b]{0.24\columnwidth}
            \centering
            \includegraphics[width=0.95\columnwidth,height=0.5cm,clip=true]{cherrypicked_m1_samples_4.png}
        \end{minipage}
        \hfill
        \begin{minipage}[b]{0.24\columnwidth}
            \centering
            \includegraphics[width=0.95\columnwidth,height=0.5cm,clip=true]{cherrypicked_m3_samples_4.png}
        \end{minipage}
        \begin{minipage}[b]{0.24\columnwidth}
            \centering
            \includegraphics[width=0.95\columnwidth,height=0.5cm,clip=true]{cherrypicked_data_samples_5.png}
        \end{minipage}
        \hfill
        \begin{minipage}[b]{0.24\columnwidth}
            \centering
            \includegraphics[width=0.95\columnwidth,height=0.5cm,clip=true]{cherrypicked_m0_samples_5.png}
        \end{minipage}
        \hfill
        \begin{minipage}[b]{0.24\columnwidth}
            \centering
            \includegraphics[width=0.95\columnwidth,height=0.5cm,clip=true]{cherrypicked_m1_samples_6.png}
        \end{minipage}
        \hfill
        \begin{minipage}[b]{0.24\columnwidth}
            \centering
            \includegraphics[width=0.95\columnwidth,height=0.5cm,clip=true]{cherrypicked_m3_samples_5.png}
        \end{minipage}
        \begin{minipage}{0.24\textwidth}
            \centering
            (a) Ground Truth
        \end{minipage}
        \hfill
        \begin{minipage}{0.24\textwidth}
            \centering
            (b) RNN-Gauss
        \end{minipage}
        \hfill
        \begin{minipage}{0.24\textwidth}
            \centering
            (c) RNN-GMM
        \end{minipage}
        \hfill
        \begin{minipage}{0.24\textwidth}
            \centering
            (d) VRNN-GMM
        \end{minipage}
    \end{minipage}
    \caption{Handwriting samples: (a) training examples and
             unconditionally generated handwriting from (b) RNN-Gauss, (c) RNN-GMM and (d) VRNN-GMM.
             The VRNN-GMM retains the writing style from beginning to end while RNN-Gauss and RNN-GMM
             tend to change the writing style during the generation process. This is possibly because
             the sequential latent random variables can guide the model to generate each sample with a consistent writing style.}
    \label{fig:handwriting_samples}
\end{figure*}

\section{Conclusion}
\label{sec:conc}
We propose a novel model that can address sequence modelling problems by incorporating latent random variables into a recurrent neural network (RNN).
Our experiments focus on unconditional natural speech generation as well as handwriting generation.
We show that the introduction of latent random variables can provide significant improvements in modelling highly structured sequences such as natural speech sequences.
We empirically show that the inclusion of randomness into high-level latent space can enable the VRNN to model natural speech sequences with a simple Gaussian distribution as the output function.
However, the standard RNN model using the same output function fails to generate reasonable samples. An RNN-based model using more powerful output function such as a GMM
can generate much better samples, but they contain a large amount of high-frequency noise compared to the samples generated by the VRNN-based models.

We also show the importance of temporal conditioning of the latent random variables by reporting higher log-likelihood numbers on modelling natural speech sequences.
In handwriting generation, the VRNN model is able to model the diversity across examples while maintaining consistent writing style over the course of generation.

%We propose a novel model that can learn to generate complex sequences by incorporating latent random variables into a recurrent neural network (RNN).
%We show that by modelling the dependencies between these latent random variables,
%the proposed VRNN can effectively model the kind of variability observed in sequential data.
%Our experiments focus on unconditional speech generation involving various real-valued datasets as well as unconstrained handwriting generation.
%We find that the introduction of latent random variables can provide significant improvements in modelling natural speech.
%We also show the importance of temporal conditioning of these latent random variables.
%Samples from VRNN models are qualitatively competitive with existing methods and appear to maintain stylistic consistency over the course of generation.
%This is especially apparent in handwriting samples.

\section*{Acknowledgments}
\label{sec:ack}
The authors would like to thank the developers of Theano~\citep{Bastien-Theano-2012}.
Also, the authors thank Kyunghyun Cho, Kelvin Xu and Sungjin Ahn for insightful comments and discussion.
We acknowledge the support of the following agencies for research funding and
computing support: Ubisoft, Nuance Foundation, NSERC, Calcul Qu\'{e}bec, Compute Canada,
the Canada Research Chairs and CIFAR.

\small
\bibliography{strings,strings-shorter,aigaion,ml,junyoung}
\bibliographystyle{abbrvnat}

\end{document}

nips2015.bbl
\begin{thebibliography}{19}
\providecommand{\natexlab}[1]{#1}
\providecommand{\url}[1]{\texttt{#1}}
\expandafter\ifx\csname urlstyle\endcsname\relax
  \providecommand{\doi}[1]{doi: #1}\else
  \providecommand{\doi}{doi: \begingroup \urlstyle{rm}\Url}\fi

\bibitem[Bastien et~al.(2012)Bastien, Lamblin, Pascanu, Bergstra, Goodfellow,
  Bergeron, Bouchard, and Bengio]{Bastien-Theano-2012}
F.~Bastien, P.~Lamblin, R.~Pascanu, J.~Bergstra, I.~J. Goodfellow, A.~Bergeron,
  N.~Bouchard, and Y.~Bengio.
\newblock Theano: new features and speed improvements.
\newblock Deep Learning and Unsupervised Feature Learning NIPS 2012 Workshop,
  2012.

\bibitem[Bayer and Osendorfer(2014)]{Bayer-et-al-arXiv-2014}
J.~Bayer and C.~Osendorfer.
\newblock Learning stochastic recurrent networks.
\newblock \emph{arXiv preprint arXiv:1411.7610}, 2014.

\bibitem[Bertrand et~al.(2008)Bertrand, Demuynck, Stouten, and
  Hamme]{bertrand2008unsupervised_b}
A.~Bertrand, K.~Demuynck, V.~Stouten, and H.~V. Hamme.
\newblock Unsupervised learning of auditory filter banks using non-negative
  matrix factorisation.
\newblock In \emph{IEEE International Conference on Acoustics, Speech and
  Signal Processing (ICASSP)}, pages 4713--4716. IEEE, 2008.

\bibitem[Boulanger-lewandowski et~al.(2012)Boulanger-lewandowski, Bengio, and
  Vincent]{boulanger2012modeling}
N.~Boulanger-lewandowski, Y.~Bengio, and P.~Vincent.
\newblock Modeling temporal dependencies in high-dimensional sequences:
  Application to polyphonic music generation and transcription.
\newblock In \emph{Proceedings of the 29th International Conference on Machine
  Learning (ICML)}, pages 1159--1166, 2012.

\bibitem[Cho et~al.(2014)Cho, van Merrienboer, Gulcehre, Bahdanau, Bougares,
  Schwenk, and Bengio]{cho2014emnlp}
K.~Cho, B.~van Merrienboer, C.~Gulcehre, D.~Bahdanau, F.~Bougares, H.~Schwenk,
  and Y.~Bengio.
\newblock Learning phrase representations using rnn encoder--decoder for
  statistical machine translation.
\newblock In \emph{Proceedings of the 2014 Conference on Empirical Methods in
  Natural Language Processing (EMNLP)}, pages 1724--1734, 2014.

\bibitem[Fabius et~al.(2014)Fabius, van Amersfoort, and
  Kingma]{fabius2014variational}
O.~Fabius, J.~R. van Amersfoort, and D.~P. Kingma.
\newblock Variational recurrent auto-encoders.
\newblock \emph{arXiv preprint arXiv:1412.6581}, 2014.

\bibitem[Graves(2013)]{graves2013generating}
A.~Graves.
\newblock Generating sequences with recurrent neural networks.
\newblock \emph{arXiv preprint arXiv:1308.0850}, 2013.

\bibitem[Gregor et~al.(2015)Gregor, Danihelka, Graves, and
  Wierstra]{gregor2015draw}
K.~Gregor, I.~Danihelka, A.~Graves, and D.~Wierstra.
\newblock Draw: A recurrent neural network for image generation.
\newblock In \emph{Proceedings of The 32nd International Conference on Machine
  Learning (ICML)}, 2015.

\bibitem[Hochreiter and Schmidhuber(1997)]{Hochreiter+Schmidhuber-1997}
S.~Hochreiter and J.~Schmidhuber.
\newblock Long short-term memory.
\newblock \emph{Neural Computation}, 9\penalty0 (8):\penalty0 1735--1780, 1997.

\bibitem[King and Karaiskos(2013)]{King2013}
S.~King and V.~Karaiskos.
\newblock The blizzard challenge 2013.
\newblock In \emph{The Ninth annual Blizzard Challenge}, 2013.

\bibitem[Kingma and Welling(2014)]{Kingma+Welling-ICLR2014}
D.~P. Kingma and M.~Welling.
\newblock Auto-encoding variational bayes.
\newblock In \emph{Proceedings of the International Conference on Learning
  Representations (ICLR)}, 2014.

\bibitem[Kingma and Welling(2015)]{kingma2015adam}
D.~P. Kingma and M.~Welling.
\newblock Adam: A method for stochastic optimization.
\newblock In \emph{Proceedings of the International Conference on Learning
  Representations (ICLR)}, 2015.

\bibitem[Lee et~al.(2009)Lee, Pham, Largman, and Ng]{lee2009unsupervised}
H.~Lee, P.~Pham, Y.~Largman, and A.~Y. Ng.
\newblock Unsupervised feature learning for audio classification using
  convolutional deep belief networks.
\newblock In \emph{Advances in Neural Information Processing Systems (NIPS)},
  pages 1096--1104, 2009.

\bibitem[Liwicki and Bunke(2005)]{liwicki2005iam_b}
M.~Liwicki and H.~Bunke.
\newblock Iam-ondb-an on-line english sentence database acquired from
  handwritten text on a whiteboard.
\newblock In \emph{Proceedings of Eighth International Conference on Document
  Analysis and Recognition}, pages 956--961. IEEE, 2005.

\bibitem[Nair and Hinton(2010)]{nair2010rectified}
V.~Nair and G.~E. Hinton.
\newblock Rectified linear units improve restricted boltzmann machines.
\newblock In \emph{Proceedings of the 27th International Conference on Machine
  Learning (ICML)}, pages 807--814, 2010.

\bibitem[Pachitariu and Sahani(2012)]{Pachitariu2012learning}
M.~Pachitariu and M.~Sahani.
\newblock Learning visual motion in recurrent neural networks.
\newblock In \emph{Advances in Neural Information Processing Systems (NIPS)},
  pages 1322--1330, 2012.

\bibitem[Rezende et~al.(2014)Rezende, Mohamed, and
  Wierstra]{rezende2014stochastic}
D.~J. Rezende, S.~Mohamed, and D.~Wierstra.
\newblock Stochastic backpropagation and approximate inference in deep
  generative models.
\newblock In \emph{Proceedings of The 31st International Conference on Machine
  Learning (ICML)}, pages 1278--1286, 2014.

\bibitem[Tokuda et~al.(2013)Tokuda, Nankaku, Toda, Zen, Yamagishi, and
  Oura]{Tokuda-et-al-2013}
K.~Tokuda, Y.~Nankaku, T.~Toda, H.~Zen, J.~Yamagishi, and K.~Oura.
\newblock Speech synthesis based on hidden markov models.
\newblock \emph{Proceedings of the IEEE}, 101\penalty0 (5):\penalty0
  1234--1252, 2013.

\bibitem[Weinberger(2015)]{speech2015accent}
S.~Weinberger.
\newblock The speech accent archieve.
\newblock \url{http://accent.gmu.edu/}, 2015.

\end{thebibliography}
