Bayesian Neural Network Priors Revisited
Vincent Fortuin\thanks{Equal contribution.
Isotropic Gaussian priors are the de facto standard for modern Bayesian neural network inference. However, it is unclear whether these priors accurately reflect our true beliefs about the weight distributions or give optimal performance. To find better priors, we study summary statistics of neural network weights in networks trained using SGD. We find that convolutional neural network (CNN) weights display strong spatial correlations, while fully connected networks (FCNNs) display heavy-tailed weight distributions. Building these observations into priors leads to improved performance on a variety of image classification datasets. Surprisingly, these priors mitigate the cold posterior effect in FCNNs, but slightly increase the cold posterior effect in ResNets.
ms.tex
\documentclass{article}
\usepackage{arxiv, times}


\usepackage{microtype}
\usepackage{graphicx}
\usepackage{subfigure}
\usepackage{booktabs}
\usepackage{wrapfig}
\usepackage[square,numbers]{natbib}
\usepackage{caption}



%\DeclareUnicodeCharacter{2212}{-}
\usepackage{pgf}
\usepackage{tikz}
\usetikzlibrary{external}
\tikzexternalize[prefix=tikz/]

\newcommand\inputpgf[2]{{
\let\pgfimageWithoutPath\pgfimage
\renewcommand{\pgfimage}[2][]{\pgfimageWithoutPath[##1]{#1/##2}}
\input{#1/#2}
}}


\usepackage{hyperref}

\newcommand{\theHalgorithm}{\arabic{algorithm}}



\makeatletter
\def\set@curr@file#1{\def\@curr@file{#1}} \makeatother

\makeatletter
\newcommand{\printfnsymbol}[1]{\textsuperscript{\@fnsymbol{#1}}}
\makeatother




\usepackage{amsmath,amsfonts,bm}

\newcommand{\Expect}{\mathbb{E}}
\newcommand{\Rhat}{\widehat{R}}


\newcommand{\figleft}{{\em (Left)}}
\newcommand{\figcenter}{{\em (Center)}}
\newcommand{\figright}{{\em (Right)}}
\newcommand{\figtop}{{\em (Top)}}
\newcommand{\figbottom}{{\em (Bottom)}}
\newcommand{\captiona}{{\em (a)}}
\newcommand{\captionb}{{\em (b)}}
\newcommand{\captionc}{{\em (c)}}
\newcommand{\captiond}{{\em (d)}}

\newcommand{\newterm}[1]{{\bf #1}}


\def\figref#1{figure~\ref{#1}}
\def\Figref#1{Figure~\ref{#1}}
\def\twofigref#1#2{figures \ref{#1} and \ref{#2}}
\def\quadfigref#1#2#3#4{figures \ref{#1}, \ref{#2}, \ref{#3} and \ref{#4}}
\def\secref#1{section~\ref{#1}}
\def\Secref#1{Section~\ref{#1}}
\def\twosecrefs#1#2{sections \ref{#1} and \ref{#2}}
\def\secrefs#1#2#3{sections \ref{#1}, \ref{#2} and \ref{#3}}
\def\eqref#1{equation~\ref{#1}}
\def\Eqref#1{Equation~\ref{#1}}
\def\plaineqref#1{\ref{#1}}
\def\chapref#1{chapter~\ref{#1}}
\def\Chapref#1{Chapter~\ref{#1}}
\def\rangechapref#1#2{chapters\ref{#1}--\ref{#2}}
\def\algref#1{algorithm~\ref{#1}}
\def\Algref#1{Algorithm~\ref{#1}}
\def\twoalgref#1#2{algorithms \ref{#1} and \ref{#2}}
\def\Twoalgref#1#2{Algorithms \ref{#1} and \ref{#2}}
\def\partref#1{part~\ref{#1}}
\def\Partref#1{Part~\ref{#1}}
\def\twopartref#1#2{parts \ref{#1} and \ref{#2}}

\def\ceil#1{\lceil #1 \rceil}
\def\floor#1{\lfloor #1 \rfloor}
\def\1{\bm{1}}
\newcommand{\train}{\mathcal{D}}
\newcommand{\valid}{\mathcal{D_{\mathrm{valid}}}}
\newcommand{\test}{\mathcal{D_{\mathrm{test}}}}

\def\eps{{\epsilon}}


\def\reta{{\textnormal{$\eta$}}}
\def\ra{{\textnormal{a}}}
\def\rb{{\textnormal{b}}}
\def\rc{{\textnormal{c}}}
\def\rd{{\textnormal{d}}}
\def\re{{\textnormal{e}}}
\def\rf{{\textnormal{f}}}
\def\rg{{\textnormal{g}}}
\def\rh{{\textnormal{h}}}
\def\ri{{\textnormal{i}}}
\def\rj{{\textnormal{j}}}
\def\rk{{\textnormal{k}}}
\def\rl{{\textnormal{l}}}
\def\rn{{\textnormal{n}}}
\def\ro{{\textnormal{o}}}
\def\rp{{\textnormal{p}}}
\def\rq{{\textnormal{q}}}
\def\rr{{\textnormal{r}}}
\def\rs{{\textnormal{s}}}
\def\rt{{\textnormal{t}}}
\def\ru{{\textnormal{u}}}
\def\rv{{\textnormal{v}}}
\def\rw{{\textnormal{w}}}
\def\rx{{\textnormal{x}}}
\def\ry{{\textnormal{y}}}
\def\rz{{\textnormal{z}}}

\def\rvepsilon{{\mathbf{\epsilon}}}
\def\rvtheta{{\mathbf{\theta}}}
\def\rva{{\mathbf{a}}}
\def\rvb{{\mathbf{b}}}
\def\rvc{{\mathbf{c}}}
\def\rvd{{\mathbf{d}}}
\def\rve{{\mathbf{e}}}
\def\rvf{{\mathbf{f}}}
\def\rvg{{\mathbf{g}}}
\def\rvh{{\mathbf{h}}}
\def\rvu{{\mathbf{i}}}
\def\rvj{{\mathbf{j}}}
\def\rvk{{\mathbf{k}}}
\def\rvl{{\mathbf{l}}}
\def\rvm{{\mathbf{m}}}
\def\rvn{{\mathbf{n}}}
\def\rvo{{\mathbf{o}}}
\def\rvp{{\mathbf{p}}}
\def\rvq{{\mathbf{q}}}
\def\rvr{{\mathbf{r}}}
\def\rvs{{\mathbf{s}}}
\def\rvt{{\mathbf{t}}}
\def\rvu{{\mathbf{u}}}
\def\rvv{{\mathbf{v}}}
\def\rvw{{\mathbf{w}}}
\def\rvx{{\mathbf{x}}}
\def\rvy{{\mathbf{y}}}
\def\rvz{{\mathbf{z}}}

\def\erva{{\textnormal{a}}}
\def\ervb{{\textnormal{b}}}
\def\ervc{{\textnormal{c}}}
\def\ervd{{\textnormal{d}}}
\def\erve{{\textnormal{e}}}
\def\ervf{{\textnormal{f}}}
\def\ervg{{\textnormal{g}}}
\def\ervh{{\textnormal{h}}}
\def\ervi{{\textnormal{i}}}
\def\ervj{{\textnormal{j}}}
\def\ervk{{\textnormal{k}}}
\def\ervl{{\textnormal{l}}}
\def\ervm{{\textnormal{m}}}
\def\ervn{{\textnormal{n}}}
\def\ervo{{\textnormal{o}}}
\def\ervp{{\textnormal{p}}}
\def\ervq{{\textnormal{q}}}
\def\ervr{{\textnormal{r}}}
\def\ervs{{\textnormal{s}}}
\def\ervt{{\textnormal{t}}}
\def\ervu{{\textnormal{u}}}
\def\ervv{{\textnormal{v}}}
\def\ervw{{\textnormal{w}}}
\def\ervx{{\textnormal{x}}}
\def\ervy{{\textnormal{y}}}
\def\ervz{{\textnormal{z}}}

\def\rmA{{\mathbf{A}}}
\def\rmB{{\mathbf{B}}}
\def\rmC{{\mathbf{C}}}
\def\rmD{{\mathbf{D}}}
\def\rmE{{\mathbf{E}}}
\def\rmF{{\mathbf{F}}}
\def\rmG{{\mathbf{G}}}
\def\rmH{{\mathbf{H}}}
\def\rmI{{\mathbf{I}}}
\def\rmJ{{\mathbf{J}}}
\def\rmK{{\mathbf{K}}}
\def\rmL{{\mathbf{L}}}
\def\rmM{{\mathbf{M}}}
\def\rmN{{\mathbf{N}}}
\def\rmO{{\mathbf{O}}}
\def\rmP{{\mathbf{P}}}
\def\rmQ{{\mathbf{Q}}}
\def\rmR{{\mathbf{R}}}
\def\rmS{{\mathbf{S}}}
\def\rmT{{\mathbf{T}}}
\def\rmU{{\mathbf{U}}}
\def\rmV{{\mathbf{V}}}
\def\rmW{{\mathbf{W}}}
\def\rmX{{\mathbf{X}}}
\def\rmY{{\mathbf{Y}}}
\def\rmZ{{\mathbf{Z}}}

\def\ermA{{\textnormal{A}}}
\def\ermB{{\textnormal{B}}}
\def\ermC{{\textnormal{C}}}
\def\ermD{{\textnormal{D}}}
\def\ermE{{\textnormal{E}}}
\def\ermF{{\textnormal{F}}}
\def\ermG{{\textnormal{G}}}
\def\ermH{{\textnormal{H}}}
\def\ermI{{\textnormal{I}}}
\def\ermJ{{\textnormal{J}}}
\def\ermK{{\textnormal{K}}}
\def\ermL{{\textnormal{L}}}
\def\ermM{{\textnormal{M}}}
\def\ermN{{\textnormal{N}}}
\def\ermO{{\textnormal{O}}}
\def\ermP{{\textnormal{P}}}
\def\ermQ{{\textnormal{Q}}}
\def\ermR{{\textnormal{R}}}
\def\ermS{{\textnormal{S}}}
\def\ermT{{\textnormal{T}}}
\def\ermU{{\textnormal{U}}}
\def\ermV{{\textnormal{V}}}
\def\ermW{{\textnormal{W}}}
\def\ermX{{\textnormal{X}}}
\def\ermY{{\textnormal{Y}}}
\def\ermZ{{\textnormal{Z}}}

\def\vzero{{\bm{0}}}
\def\vone{{\bm{1}}}
\def\vmu{{\bm{\mu}}}
\def\vtheta{{\bm{\theta}}}
\def\va{{\bm{a}}}
\def\vb{{\bm{b}}}
\def\vc{{\bm{c}}}
\def\vd{{\bm{d}}}
\def\ve{{\bm{e}}}
\def\vf{{\bm{f}}}
\def\vg{{\bm{g}}}
\def\vh{{\bm{h}}}
\def\vi{{\bm{i}}}
\def\vj{{\bm{j}}}
\def\vk{{\bm{k}}}
\def\vl{{\bm{l}}}
\def\vm{{\bm{m}}}
\def\vn{{\bm{n}}}
\def\vo{{\bm{o}}}
\def\vp{{\bm{p}}}
\def\vq{{\bm{q}}}
\def\vr{{\bm{r}}}
\def\vs{{\bm{s}}}
\def\vt{{\bm{t}}}
\def\vu{{\bm{u}}}
\def\vv{{\bm{v}}}
\def\vw{{\bm{w}}}
\def\vx{{\bm{x}}}
\def\vy{{\bm{y}}}
\def\vz{{\bm{z}}}

\def\evalpha{{\alpha}}
\def\evbeta{{\beta}}
\def\evepsilon{{\epsilon}}
\def\evlambda{{\lambda}}
\def\evomega{{\omega}}
\def\evmu{{\mu}}
\def\evpsi{{\psi}}
\def\evsigma{{\sigma}}
\def\evtheta{{\theta}}
\def\eva{{a}}
\def\evb{{b}}
\def\evc{{c}}
\def\evd{{d}}
\def\eve{{e}}
\def\evf{{f}}
\def\evg{{g}}
\def\evh{{h}}
\def\evi{{i}}
\def\evj{{j}}
\def\evk{{k}}
\def\evl{{l}}
\def\evm{{m}}
\def\evn{{n}}
\def\evo{{o}}
\def\evp{{p}}
\def\evq{{q}}
\def\evr{{r}}
\def\evs{{s}}
\def\evt{{t}}
\def\evu{{u}}
\def\evv{{v}}
\def\evw{{w}}
\def\evx{{x}}
\def\evy{{y}}
\def\evz{{z}}

\def\mA{{\bm{A}}}
\def\mB{{\bm{B}}}
\def\mC{{\bm{C}}}
\def\mD{{\bm{D}}}
\def\mE{{\bm{E}}}
\def\mF{{\bm{F}}}
\def\mG{{\bm{G}}}
\def\mH{{\bm{H}}}
\def\mI{{\bm{I}}}
\def\mJ{{\bm{J}}}
\def\mK{{\bm{K}}}
\def\mL{{\bm{L}}}
\def\mM{{\bm{M}}}
\def\mN{{\bm{N}}}
\def\mO{{\bm{O}}}
\def\mP{{\bm{P}}}
\def\mQ{{\bm{Q}}}
\def\mR{{\bm{R}}}
\def\mS{{\bm{S}}}
\def\mT{{\bm{T}}}
\def\mU{{\bm{U}}}
\def\mV{{\bm{V}}}
\def\mW{{\bm{W}}}
\def\mX{{\bm{X}}}
\def\mY{{\bm{Y}}}
\def\mZ{{\bm{Z}}}
\def\mBeta{{\bm{\beta}}}
\def\mPhi{{\bm{\Phi}}}
\def\mLambda{{\bm{\Lambda}}}
\def\mSigma{{\bm{\Sigma}}}

\DeclareMathAlphabet{\mathsfit}{\encodingdefault}{\sfdefault}{m}{sl}
\SetMathAlphabet{\mathsfit}{bold}{\encodingdefault}{\sfdefault}{bx}{n}
\newcommand{\tens}[1]{\bm{\mathsfit{#1}}}
\def\tA{{\tens{A}}}
\def\tB{{\tens{B}}}
\def\tC{{\tens{C}}}
\def\tD{{\tens{D}}}
\def\tE{{\tens{E}}}
\def\tF{{\tens{F}}}
\def\tG{{\tens{G}}}
\def\tH{{\tens{H}}}
\def\tI{{\tens{I}}}
\def\tJ{{\tens{J}}}
\def\tK{{\tens{K}}}
\def\tL{{\tens{L}}}
\def\tM{{\tens{M}}}
\def\tN{{\tens{N}}}
\def\tO{{\tens{O}}}
\def\tP{{\tens{P}}}
\def\tQ{{\tens{Q}}}
\def\tR{{\tens{R}}}
\def\tS{{\tens{S}}}
\def\tT{{\tens{T}}}
\def\tU{{\tens{U}}}
\def\tV{{\tens{V}}}
\def\tW{{\tens{W}}}
\def\tX{{\tens{X}}}
\def\tY{{\tens{Y}}}
\def\tZ{{\tens{Z}}}


\def\gA{{\mathcal{A}}}
\def\gB{{\mathcal{B}}}
\def\gC{{\mathcal{C}}}
\def\gD{{\mathcal{D}}}
\def\gE{{\mathcal{E}}}
\def\gF{{\mathcal{F}}}
\def\gG{{\mathcal{G}}}
\def\gH{{\mathcal{H}}}
\def\gI{{\mathcal{I}}}
\def\gJ{{\mathcal{J}}}
\def\gK{{\mathcal{K}}}
\def\gL{{\mathcal{L}}}
\def\gM{{\mathcal{M}}}
\def\gN{{\mathcal{N}}}
\def\gO{{\mathcal{O}}}
\def\gP{{\mathcal{P}}}
\def\gQ{{\mathcal{Q}}}
\def\gR{{\mathcal{R}}}
\def\gS{{\mathcal{S}}}
\def\gT{{\mathcal{T}}}
\def\gU{{\mathcal{U}}}
\def\gV{{\mathcal{V}}}
\def\gW{{\mathcal{W}}}
\def\gX{{\mathcal{X}}}
\def\gY{{\mathcal{Y}}}
\def\gZ{{\mathcal{Z}}}

\def\sA{{\mathbb{A}}}
\def\sB{{\mathbb{B}}}
\def\sC{{\mathbb{C}}}
\def\sD{{\mathbb{D}}}
\def\sF{{\mathbb{F}}}
\def\sG{{\mathbb{G}}}
\def\sH{{\mathbb{H}}}
\def\sI{{\mathbb{I}}}
\def\sJ{{\mathbb{J}}}
\def\sK{{\mathbb{K}}}
\def\sL{{\mathbb{L}}}
\def\sM{{\mathbb{M}}}
\def\sN{{\mathbb{N}}}
\def\sO{{\mathbb{O}}}
\def\sP{{\mathbb{P}}}
\def\sQ{{\mathbb{Q}}}
\def\sR{{\mathbb{R}}}
\def\sS{{\mathbb{S}}}
\def\sT{{\mathbb{T}}}
\def\sU{{\mathbb{U}}}
\def\sV{{\mathbb{V}}}
\def\sW{{\mathbb{W}}}
\def\sX{{\mathbb{X}}}
\def\sY{{\mathbb{Y}}}
\def\sZ{{\mathbb{Z}}}

\def\emLambda{{\Lambda}}
\def\emA{{A}}
\def\emB{{B}}
\def\emC{{C}}
\def\emD{{D}}
\def\emE{{E}}
\def\emF{{F}}
\def\emG{{G}}
\def\emH{{H}}
\def\emI{{I}}
\def\emJ{{J}}
\def\emK{{K}}
\def\emL{{L}}
\def\emM{{M}}
\def\emN{{N}}
\def\emO{{O}}
\def\emP{{P}}
\def\emQ{{Q}}
\def\emR{{R}}
\def\emS{{S}}
\def\emT{{T}}
\def\emU{{U}}
\def\emV{{V}}
\def\emW{{W}}
\def\emX{{X}}
\def\emY{{Y}}
\def\emZ{{Z}}
\def\emSigma{{\Sigma}}

\newcommand{\etens}[1]{\mathsfit{#1}}
\def\etLambda{{\etens{\Lambda}}}
\def\etA{{\etens{A}}}
\def\etB{{\etens{B}}}
\def\etC{{\etens{C}}}
\def\etD{{\etens{D}}}
\def\etE{{\etens{E}}}
\def\etF{{\etens{F}}}
\def\etG{{\etens{G}}}
\def\etH{{\etens{H}}}
\def\etI{{\etens{I}}}
\def\etJ{{\etens{J}}}
\def\etK{{\etens{K}}}
\def\etL{{\etens{L}}}
\def\etM{{\etens{M}}}
\def\etN{{\etens{N}}}
\def\etO{{\etens{O}}}
\def\etP{{\etens{P}}}
\def\etQ{{\etens{Q}}}
\def\etR{{\etens{R}}}
\def\etS{{\etens{S}}}
\def\etT{{\etens{T}}}
\def\etU{{\etens{U}}}
\def\etV{{\etens{V}}}
\def\etW{{\etens{W}}}
\def\etX{{\etens{X}}}
\def\etY{{\etens{Y}}}
\def\etZ{{\etens{Z}}}

\newcommand{\pdata}{p_{\rm{data}}}
\newcommand{\ptrain}{\hat{p}_{\rm{data}}}
\newcommand{\Ptrain}{\hat{P}_{\rm{data}}}
\newcommand{\pmodel}{p_{\rm{model}}}
\newcommand{\Pmodel}{P_{\rm{model}}}
\newcommand{\ptildemodel}{\tilde{p}_{\rm{model}}}
\newcommand{\pencode}{p_{\rm{encoder}}}
\newcommand{\pdecode}{p_{\rm{decoder}}}
\newcommand{\precons}{p_{\rm{reconstruct}}}

\newcommand{\laplace}{\mathrm{Laplace}} 

\newcommand{\tp}{\mathsf{T}}  

\newcommand{\E}{\mathbb{E}}
\newcommand{\Ls}{\mathcal{L}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\emp}{\tilde{p}}
\newcommand{\lr}{\alpha}
\newcommand{\reg}{\lambda}
\newcommand{\rect}{\mathrm{rectifier}}
\newcommand{\softmax}{\mathrm{softmax}}
\newcommand{\sigmoid}{\sigma}
\newcommand{\softplus}{\zeta}
\newcommand{\KL}{D_{\mathrm{KL}}}
\newcommand{\Var}{\mathrm{Var}}
\newcommand{\standarderror}{\mathrm{SE}}
\newcommand{\Cov}{\mathrm{Cov}}
\newcommand{\normlzero}{L^0}
\newcommand{\normlone}{L^1}
\newcommand{\normltwo}{L^2}
\newcommand{\normlp}{L^p}
\newcommand{\normmax}{L^\infty}

\newcommand{\parents}{Pa} 

\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}

\DeclareMathOperator{\sign}{sign}
\DeclareMathOperator{\Tr}{Tr}
\let\ab\allowbreak
 
\usepackage{hyperref}
\usepackage{url}
\usepackage{chngcntr}


\usepackage{xspace}

\newcommand{\addcitation}{\textcolor{red}{[ADD CITATION]\xspace}} 
\usepackage{graphicx}
\graphicspath{{figures/}} 

\interfootnotelinepenalty=10000


\title{Bayesian Neural Network Priors Revisited}



\author{
Vincent Fortuin\thanks{Equal contribution.} \\
ETH Z\"urich \\
\texttt{fortuin@inf.ethz.ch} \\
\And
Adri\`a Garriga-Alonso\printfnsymbol{1} \\
University of Cambridge\\
\texttt{ag919@cam.ac.uk} \\
\AND
Florian Wenzel \\
Humboldt University of Berlin\\
\And
Gunnar R\"atsch \\
ETH Z\"urich \\
\And
Richard E. Turner\\
University of Cambridge\\
\AND
Mark {van der Wilk}\thanks{Equal contribution.}\\
Imperial College London\\
\And
Laurence Aitchison\printfnsymbol{2}\\
University of Bristol
}


\begin{document}

\maketitle





\begin{abstract}
Isotropic Gaussian priors are the \emph{de facto} standard for modern Bayesian neural network inference.
However, it is unclear whether these priors accurately reflect our true beliefs about the weight distributions or give optimal performance.
To find better priors, we study summary statistics of neural network weights in networks trained using SGD.
We find that convolutional neural network (CNN) weights display strong spatial correlations, while fully connected networks (FCNNs) display heavy-tailed weight distributions.
Building these observations into priors leads to improved performance on a variety of image classification datasets.
Surprisingly, these priors mitigate the cold posterior effect in FCNNs, but slightly increase the cold posterior effect in ResNets.
\end{abstract}


\section{Introduction}





In a Bayesian neural network (BNN), we specify a prior $p(w)$ over the neural network parameters, and compute the posterior distribution over parameters conditioned on training data, $p( w | x,y ) = p(y | w, x) p(w) / p(y | x)$.
This procedure should give considerable advantages for reasoning about predictive uncertainty, which is especially relevant in the small-data setting.
Crucially, to perform Bayesian inference, we need to choose a prior that accurately reflects our beliefs about the parameters before seeing any data \citep{bayes1763lii, gelman2013bayesian}.
However, the most common choice of prior for BNN weights is the simplest one: the isotropic Gaussian.
Isotropic Gaussians are used across almost all fields of Bayesian deep learning, ranging from variational inference \citep[e.g.,][]{hernandez2015probabilistic, louizos2017multiplicative, dusenberry2020efficient}, sampling-based inference \citep[e.g.,][]{neal1992bayesian, zhang2019cyclical}, and Laplace's method \citep[e.g.,][]{osawa2019practical, immer2021improving}, to even infinite networks \citep[e.g.,][]{lee2017deep, garriga2018deep}.
It is troubling that no alternatives are usually considered, since better choices likely exist. 

Indeed, despite the progress on more accurate and efficient inference procedures, in some settings, the posterior predictive distribution of BNNs using Gaussian priors still leads to worse predictive performance than a baseline obtained by training the network with standard stochastic gradient descent (SGD) \citep[e.g.,][]{zhang2019cyclical,hee2020,wenzel2020good}.
Surprisingly, these issues can largely be fixed by artificially reducing posterior uncertainty using ``cold posteriors'' \citep{wenzel2020good}, and this improvement in performance at low temperatures is known as the ``cold posterior effect''. 
The cold posterior is $p(w | x,y)^{\frac{1}{T}}$ for a temperature $0<T<1$, where the original Bayes posterior would be obtained by setting $T=1$ (see  Eq.~\ref{eq:cold_posterior}). 
Using cold posteriors can be interpreted as overcounting the data and, hence, deviating from the Bayesian paradigm.
This should not happen if the prior and likelihood accurately reflect our beliefs. Assuming inference is working correctly, the Bayesian solution, $T=1$, really should be optimal \citep{gelman2013bayesian}.
Hence, it raises the possibility that either the prior \citep{wenzel2020good} or likelihood \citep{aitchison2020statistical} (or both) are misspecified.









In this work, we study empirically whether isotropic Gaussian priors are indeed suboptimal for BNNs and whether this can explain the cold posterior effect. We analyze the performance of different BNN priors for different network architectures and compare them to the empirical weight distributions of standard SGD-trained neural networks. We conclude that correlated Gaussian priors are better in convolutional neural networks (CNNs), while uncorrelated heavy-tailed priors are better in fully connected neural networks (FCNNs).
Thus, we would recommend these choices instead of the widely-used isotropic Gaussian priors.
While these priors eliminate the cold posterior effect in FCNNs, they slightly increase the cold posterior effect in ResNets.
This provides evidence that the cold posterior effect arises due to a misspecification of the prior \citep{wenzel2020good} in FCNNs.
In ResNets, it is difficult to draw any strong conclusions about the cold posterior effect from our results.
Our observations are compatible with the hypothesis that the cold posterior effect arises in large-scale image models due to a misspecified likelihood \citep{aitchison2020statistical} or due to data augmentation \citep{izmailov2020subspace}, but there could of course be a prior that we did not consider that improves performance and eliminates the cold posterior effect.
We make our library available on Github\footnote{\url{https://github.com/ratschlab/bnn_priors}. MIT licensed.}, inviting other researchers to join us in studying the role of priors in BNNs using state-of-the-art inference.
\begin{wrapfigure}{r}{0.5\linewidth}
    \vspace{-0.2cm}
    \centerline{\includegraphics[width=0.9\linewidth]{figures/210203-qqplots-allgaussian.pdf}}
    \caption{Empirical marginal weight distributions of a layer of FCNNs and CNNs trained with SGD on MNIST, and an early layer of several ResNets trained on CIFAR-10. We show marginal weight histograms (left) and quantile-quantile (Q-Q) plots with different distributions (right). The empirical weights are clearly heavier-tailed than a Gaussian (green line), and seemingly better fit by a Laplace (orange line). }
    \label{fig:dnn_cnn_weights}
    \vspace{-2.5cm}
\end{wrapfigure}

\subsection{Contributions}
Our main contributions are:
\begin{itemize}
    \item An analysis of the empirical weight distributions of SGD-trained neural networks with different architectures, suggesting that FCNNs learn heavy-tailed weight distributions (Sec.~\ref{sec:heavy_tailed}), while CNN weight distributions show significant spatial correlations (Sec.~\ref{sec:correlations}).
    \item Experiments in Bayesian FCNNs showing that heavy-tailed priors give better classification performance than the widely-used Gaussian priors (Sec.~\ref{sec:dnn_experiments}).
    \item Experiments in Bayesian CNNs showing that spatially correlated Gaussian priors give better classification performance than isotropic priors (Sec.~\ref{sec:cnn_experiments}).
    \item Experiments showing that the cold posterior effect can be reduced by choosing better, heavy-tailed priors in FCNNs, while the cold posterior is slightly increased when using better, spatially correlated priors in CNNs (Sec.~\ref{sec:experiments}).
\end{itemize}




\section{Background: the cold posterior effect}
\label{sec:background}







When performing inference in Bayesian models, we can temper the posterior by a positive temperature $T$, giving
\begin{equation}
\label{eq:cold_posterior}
    \log p(w | x,y)^{\frac{1}{T}} = \frac{1}{T} [ \log p(y | w,x) + \log p(w) ] + Z(T)
\end{equation}
for neural network weights $w$, data $(x,y)$, prior $p(w)$, likelihood $p(y|w,x)$, and a normalizing constant $Z(T)$.
Setting $T = 1$ yields the standard Bayesian posterior. The temperature parameter can be easily handled when simulating Langevin dynamics, as used in molecular dynamics and MCMC \citep{lm-aboba}.

In their recent work, \citet{wenzel2020good} have drawn attention to the
fact that cooling the posterior in BNNs (i.e., setting $T < 1$), often improves performance.
Testing different hypotheses for potential problems with the inference, likelihood, and prior, they conclude that the BNN priors (which were Gaussian in their experiments) are misspecified---at least when used in conjuction with standard neural network architectures on standard benchmark tasks---which could be one of the main causes of the cold posterior effect \citep[c.f.,][]{germain2016pac, vdw2018inv}.
Reversing this argument, we can hypothesize that choosing better priors for BNNs may lead to a less pronounced cold posterior effect, which we can use to evaluate different candidate priors.




\section{Empirical analysis of neural network weights}
\label{sec:weight_dists}





As we have discussed, standard Gaussian priors may not be the optimal choice for modern BNN architectures. 
But how can we find more suitable priors? 
Since it is hard to directly formulate reasonable prior beliefs about neural network weights, we turn to an empirical approach.
We trained fully connected neural networks (FCNNs) and convolutional neural networks (CNNs) with SGD on various image classification tasks to obtain an approximation of the empirical distribution of the fitted weights, that is, the distribution of the \emph{maximum a posteriori} (MAP) solutions reached by SGD.
If the distributions over SGD-fitted weights differ strongly from the usual isotropic Gaussian prior, that provides evidence that those features should be incorporated into the prior.
Hence, we can use our insights by inspecting the empirical weight distribution to propose better-suited priors.



Formally, this procedure can be viewed as approximate human-in-the-loop expectation maximization (EM).
In particular, in expectation maximization, we alternate expectation (E) and maximization (M) steps. 
In the expectation (E) step, we infer the posterior $p(w| x, y, \theta_{t-1})$ over the weights, $w$, given the parameters of the prior from the previous step, $\theta_{t-1}$.
In our case, we approximately infer the weights using SGD. Then, in the maximization step, we compute new prior parameters $\theta_t$, by sampling weights $w$ from the posterior computed in the E step, and maximizing the joint probability of sampled weights and data.
As $y$ is independent of the prior parameters if the weights are known, the M-step reduces to fitting a prior distribution to the weights sampled from the posterior, that is,
\begin{align}
  \mathcal{L}_t(\theta) &= \Expect_{p(w| x, y, \theta_{t-1})}[\log p(y| x, w) + \log p(w| \theta)] \nonumber\\
  &= \Expect_{p(w| x, y, \theta_{t-1})}[\log p(w| \theta)] + \text{const}\\
  \theta_t &= \argmax \mathcal{L}_t(\theta) \; .
\end{align}

Intuitively, this procedure allows the prior (and therefore the posterior) to assign more probability mass to the SGD solutions, which are known to work well in practice.
This is also related to ideas from empirical Bayes \citep{robbins1992empirical}, where the (few) hyperparameters of the prior are fit to the data, and to recent ideas in PAC-Bayesian theory, where data-dependent priors have been shown to improve generalization guarantees over data-independent ones \citep{rivasplata2020pac, dziugaite2021role}.
While such approaches introduce a certain risk of overfitting \citep{ober2021promises}, we would argue that standard BNNs are typically thought to be underfitting \citep{neal1996bayesian, wenzel2020good, dusenberry2020efficient} and that we do not directly fit the prior parameters, but merely draw inspiration for the choice of prior family from the qualitative shape of the empirical weight distributions.




We begin by considering whether the weights of FCNNs and CNNs are heavy-tailed, and move on to look at correlational structure in the weights of CNNs.
Note that in the exploratory experiments here, we used SGD to perform MAP inference with a uniform prior (that is, maximum likelihood fitting).
This avoids any prior assumptions obscuring interesting patterns in the inferred weights.
These patterns inspired a choice of novel priors, and we evaluated these priors by showing that they improved classification performance (see Sec.~\ref{sec:experiments}).


\subsection{FCNN weights are heavy-tailed}
\label{sec:heavy_tailed}

\begin{figure}[t]
\begin{subfigure}[Student-t DoF of weights by layer, ResNet20]{\includegraphics[width=0.47\linewidth]{figures/210126-resnet-dof.pdf}}
    \end{subfigure}\hfill \begin{subfigure}[Spatial covariance of the weights, 3-layer CNN]{\includegraphics[width=0.5\linewidth]{figures/210126-mnist-covariances-all.pdf}}
\end{subfigure}
    \caption{\textbf{(a)} Degrees of freedom for Student-t distributions fitted to the weights of a ResNet20 trained on CIFAR-10. The degrees of freedom get larger in deeper layers, implying that the weight distributions become less heavy-tailed and more similar to Gaussians. The layers marked with asterisks (*) are the first layers of their respective ResNet blocks.
\textbf{(b)} Spatial covariance of the weights within CNN filters for a three-hidden layer network trained on MNIST, normalized by the number of channels. The weights correlate
      strongly with neighboring pixels, and anti-correlate (layer 1) or do not
      correlate (layer 2) with distant ones. Each delineated square shows the
      covariances of a filter location (marked with $\times$) with all other
      locations.}
      \label{fig:resnet_dof}
      \label{fig:cnn_filter_corr}
\end{figure}


We trained an FCNN (Fig.~\ref{fig:dnn_cnn_weights}, top) and a CNN (Fig.~\ref{fig:dnn_cnn_weights}, middle) on MNIST \citep{lecun1998gradient}.
The FCNN is a three layer network with 100 hidden units per layer and ReLU nonlinearities.
The CNN is a three layer network, with two convolutional layers and one fully connected layer. The convolutional layers have 64 channels and use $3 \times 3$ convolutions, followed by $2 \times 2$ max-pooling layers.
All layers use ReLU nonlinearities.
Networks were trained with SGD for 450 epochs using a learning rate schedule of $0.05$, $0.005$ and $0.0005$ for 150 epochs each.
We can see in Figure~\ref{fig:dnn_cnn_weights} that the weight values of the FCNNs and CNNs follow a more heavy-tailed distribution than a Gaussian, with the tails being reasonably well approximated by a Laplace distribution.
This suggests that ``true'' BNN priors might be more heavy-tailed than isotropic Gaussians.

Next, we did a similar analysis for a ResNet20 trained on CIFAR-10 \citep{krizhevsky2009learning} (Fig.~\ref{fig:dnn_cnn_weights}, bottom).
Since this network had many layers, we quantified the degree of heavy-tailedness by fitting the degrees of freedom parameter $\nu$ of a Student-t distribution.
For $\nu \rightarrow \infty$, the Student-t becomes Gaussian, so large values of $\nu$ indicate that the weights are approximately Gaussian, whereas smaller values indicate heavy-tailed behavior (see Sec.~\ref{sec:priors}).
We found that at lower layers, $\nu$ was small, so the weights were somewhat heavy-tailed, whereas at higher layers, $\nu$ became much larger, so the weights were approximately Gaussian (Fig.~\ref{fig:resnet_dof}a).





These results are perhaps expected if we assume that the filters have (using neuroscience terminology) ``localized receptive fields'', like those in \citet{olshausen1997sparse}.
Such filters contain a large number of near-zero weights outside the receptive field, with a number of very large weights inside the receptive field \citep{sahani2003evidence,smyth2003receptive}, and thus will follow a heavy-tailed distribution.
As we get into the deeper layers of the networks, receptive fields are expected to become larger, so this effect may be less relevant.











\subsection{CNN weights are spatially correlated}
\label{sec:correlations}



In the second part of our empirical inspection of fitted weight distributions, we looked at spatial correlations in CNN filters.
In particular, we considered 9-dimensional vectors formed by the $3 \times 3$ filters for every input and output channel.
We studied our three-layer network trained on MNIST and found strong correlations between nearby pixels, and lesser (layer 2) or even negative (layer 1) correlations at more distant pixels (Fig.~\ref{fig:cnn_filter_corr}b).
We found similar spatial correlations in a ResNet20 trained on CIFAR-10, across all layers, with correlation strength increasing as we move to later layers (Fig.~\ref{fig:resnet_covariances}). We found by far the strongest evidence of correlations spatially, that is, between weights within the same CNN filter.
This could potentially be due to the smoothness and translation equivariance properties of natural images \citep{simoncelli2009capturing}.
However, we also found some evidence for spatial correlations in the input layer of an FCNN (Fig.~\ref{fig:dnn_covar} in the appendix), but no evidence for correlations between the channels of a CNN (Fig.~\ref{fig:cnn_covar} in the appendix). Note though that this methodology cannot find structured correlations between channels, except at the input and output. This is because NN functions are invariant to permutations of neurons/channels \citep{sussmann1992uniqueness,mackay1992practical,bishop1995neural,aitchison2020bigger,aitchison2020deep}.


These findings suggest that better priors could be designed by explicitly taking this correlation structure into account.
We hypothesize that multivariate distributions with non-diagonal covariance matrices could be good candidates for CNN priors, especially when the covariances are large for neighboring pixels within the CNN filters (see Sec.~\ref{sec:cnn_experiments}).


\begin{figure*}[t]
    \centerline{
    \includegraphics[width=\linewidth]{figures/210204_googleresnet_covariances_all_capped.pdf}}
    \caption{Spatial covariances for the convolutional weights of the layers of a ResNet-20, normalized by the maximum variance for each layer, which is shown on the bottom right. We trained the network with SGD on CIFAR-10 with data augmentation (10 times). Layer 1 is the closest to the input. The first layer of every ResNet block is marked with an asterisk (*). We see that there are significant covariances in all layers, but that their strength increases for later layers.}
    \label{fig:resnet_covariances}
\end{figure*}

Additional evidence for the usefulness of correlated weights comes from the theory of infinitely wide CNNs. \citet{novak2019bayesian} noticed that the effect of weight-sharing disappears when infinite filters are used with isotropic priors. More recently, \citet{garriga2021correlated} showed that this effect can be avoided by using spatially correlated priors, leading to improved performance. Our experiments investigate whether this prior is also useful in the finite-width case.








































\section{Empirical study of Bayesian neural network priors}
\label{sec:experiments}



We performed experiments on MNIST  and on CIFAR-10.
We compare Bayesian FCNNs, CNNs, and ResNets on these tasks.
For the BNN inference, we used Stochastic Gradient Markov Chain Monte Carlo (SG-MCMC), in order to scale to large training datasets. To obtain posterior samples that are close to the true posterior, we used an inference method that builds on the inference approach used in \citet{wenzel2020good}, which has been shown to produce high-quality samples.
In particular, we combined the gradient-guided Monte Carlo (GG-MC) scheme from \citet{guided-mcmc} with the cyclical learning rate schedule from \citet{zhang2019cyclical} and the preconditioning and convergence diagnostics from \citet{wenzel2020good}.
We ran each chain for 60 cycles of 45 epochs each, taking one sample at the end of each of the last five epochs of each cycle, thus yielding 300 samples after 2,700 epochs, out of which we discarded the first 50 samples as a burn-in.
Per temperature setting, dataset, model, and prior, we ran five such chains as replicates.
To the best of our knowledge, this procedure constitutes the best SGLD-based inference approach that is currently available for BNNs.
Additional experimental results can be found in Appendix~\ref{sec:additional_exp}, details about the evaluation metrics in Appendix~\ref{sec:metrics}, about the priors in Appendix~\ref{sec:prior_details}, and about the implementation in Appendix~\ref{sec:details}.
In the figures, we generally include an SGD baseline for the predictive error, where it is often competitive with some of the priors.
For the likelihood, calibration, and OOD detection, the SGD baselines were out of the plotting range and are therefore not shown.
For completeness, we show them in Appendix~\ref{sec:sgd_baselines}.
Moreover, while we focus on image classification tasks in this section, we provide results on UCI regression tasks in Appendix~\ref{sec:uci_regression}, results for higher temperatures ($T > 1$) in Appendix~\ref{sec:higher_temps}, and for different prior variances in Appendix~\ref{sec:prior_variances}.
Finally, we show inference diagnostics highlighting the accuracy of our MCMC sampling in Appendix~\ref{sec:inf_diagnostics}.





\subsection{Priors under consideration}
\label{sec:priors}


We contrast the widely used isotropic Gaussian priors with heavy-tailed distributions, including the Laplace and Student-t distributions, and with correlated Gaussian priors.
We chose these distributions based on our observations of the empirical weight distributions of SGD-trained networks (see Sec.~\ref{sec:weight_dists}) and for their ease of implementation and optimization.
Further details on the distributions and their density functions can be found in Appendix~\ref{sec:prior_details}.


The \textbf{isotropic Gaussian} distribution \citep{gauss1809theoria} is the \emph{de-facto} standard for BNN priors in recent work \citep[e.g.,][]{hernandez2015probabilistic, louizos2017multiplicative, dusenberry2020efficient, wenzel2020good, neal1992bayesian, zhang2019cyclical, osawa2019practical, immer2021improving, lee2017deep, garriga2018deep}.
However, its tails are relatively light compared to some of the other distributions that we will consider and compared to the empirical weight distributions described above.
The \textbf{Laplace} distribution \citep{laplace1774memoire}, for instance, has heavier tails than the Gaussian.
It is often used in the context of (frequentist) \emph{lasso} regression \citep{tibshirani1996regression}.
Similarly, the \textbf{Student-t} distribution is also heavy-tailed.
Moreover, it can be seen as a Gaussian scale-mixture, where the scales are inverse-Gamma distributed \citep{helmert1875berechnung, luroth1876vergelichung}.

For our correlated Bayesian CNN priors, we use \textbf{multivariate Gaussian} priors and define the covariance $\mSigma$ to be block-diagonal, such that the covariance between weights in different filters is 0 and between weights in the same filter is given by a Matérn kernel ($\nu = 1/2$) on the pixel distances.
Formally, for the weights $w_{i,j}$ and $w_{i',j'}$ in filters $i$ and $i'$ and for pixels $j$ and $j'$, the covariance is
\begin{equation}
    \text{cov}(w_{i,j}, w_{i',j'}) =
    \begin{cases}
    \sigma^2 \exp \left( \frac{- d(j, j')}{\lambda} \right) &\text{if} \; i = i' \\
    0 &\text{otherwise}
    \end{cases} \; ,
    \label{eq:rbf_filter_cov}
\end{equation}
where $d(\cdot, \cdot)$ is the Euclidean distance between pixel positions and we set $\sigma = \lambda = 1$.



\subsection{Bayesian FCNN performance with different priors}
\label{sec:dnn_experiments}


\begin{figure*}
    \centerline{
    \includegraphics[width=\linewidth,page=1]{figures/temp_curves/tempering_curves_compressed.pdf}}
    
    \caption{Performances of fully connected BNNs with different priors on MNIST and FashionMNIST (see Sec.~\ref{sec:dnn_experiments}). The heavy-tailed priors generally perform better, especially at higher temperatures, and lead to a less pronounced cold posterior effect.
    Note the reversed y-axis for OOD detection on the right to ensure that lower values are better in all plots.}
    \label{fig:tempering_curves_fcnns}
\end{figure*}

Following our observations from the empirical weight distributions (Sec.~\ref{sec:heavy_tailed}), we hypothesized that heavy-tailed priors should work better than Gaussian priors for Bayesian FCNNs.
We tested this hypothesis by performing BNN inference with the same network architecture as in Sec.~\ref{sec:weight_dists}, using different priors.
We report the predictive error and log likelihood on the MNIST test set.
We follow \citet{ovadia2019can} in reporting the calibration of the uncertainty estimates on rotated MNIST digits and the out-of-distribution (OOD) detection accuracy on FashionMNIST \citep{xiao2017fashion}.
For more details about our evaluation metrics, see Appendix~\ref{sec:metrics}.

We observe that the heavy-tailed priors indeed outperform the Gaussian prior in terms of test error and test NLL in all cases except the Student-t distribution on MNIST at low temperatures (Fig.~\ref{fig:tempering_curves_fcnns}).
That said, calibration and OOD metrics are less clear, with heavy-tailed priors giving worse calibration and roughly similar OOD detection on MNIST and better calibration but worse OOD detection on FashionMNIST.
Despite the unclear results on calibration and OOD detection, the error and NLL performance improvement for heavy-tailed priors at $T=1$ is considerable, and suggests that Gaussian priors over the weights of FCNNs induce poor priors in the function space and inhibit the posterior from assigning probability mass to high-likelihood solutions, such as the SGD solutions analyzed above (Sec.~\ref{sec:weight_dists}).
Finally, the cold posterior effect is removed---or even inverted---when using heavy-tailed priors, which supports the hypothesis that it is caused by prior misspecification in FCNNs.
Note that the cold-posterior effect is typically defined in terms of performance metrics like error and NLL, and not calibration and OOD detection performance \citep{wenzel2020good}.
As such, even with Gaussian priors, we do not necessarily expect calibration and OOD detection to exhibit a cold posterior effect. 
Indeed, only calibration for FashionMNIST exhibits a cold posterior effect, with calibration for MNIST and all OOD detection results exhibiting an inverted cold posterior effect.
Notably, we see in Appendix~\ref{sec:activation_functions} and Appendix~\ref{sec:prior_variances} that these observations generalize to different activation functions and prior variances and in Appendix~\ref{sec:higher_temps} that warm posteriors ($T > 1$) deteriorate the performance for all considered priors, such that for the heavy-tailed priors, $T \approx 1$ is indeed ideal.




\subsection{Bayesian CNN and ResNet performance with different priors}
\label{sec:cnn_experiments}


\begin{figure*}
    \centerline{
    \includegraphics[width=\linewidth, page=7]{figures/temp_curves/tempering_curves_compressed.pdf}}
    \caption{Performances of convolutional BNNs with different priors on MNIST, FashionMNIST, and CIFAR-10 (see Sec.~\ref{sec:cnn_experiments}). The (Fashion)MNIST experiments used CNNs, while the CIFAR-10 experiments used ResNet20. The correlated prior generally performs better than the isotropic ones, but still exhibits a cold posterior effect, while the heavy-tailed priors reduce the cold posterior effect, but yield a worse performance. Note the reversed y-axis for OOD detection on the right to ensure that lower values are better in all plots.}
    \label{fig:tempering_curves_cnns}
\end{figure*}


We repeated the same experiment for Bayesian CNNs on MNIST and FashionMNIST (Fig.~\ref{fig:tempering_curves_cnns}, first two rows).
Given our observations about SGD-trained weights (Sec.~\ref{sec:heavy_tailed}), we might again expect heavy-tailed priors to outperform Gaussian priors.
However, this is not the case: the Gaussian and correlated Gaussian priors perform better in almost all cases, with the exception of calibration for FashionMNIST.
Interestingly, the performance of different methods tends to be very similar at $T=1$, and to diverge for lower temperatures, with performance improving for Gaussian and correlated Gaussian priors (indicating a cold posterior effect), and worsening for heavy-tailed priors, indicating no cold posterior effect.


Our analysis of SGD-trained weights (Sec.~\ref{sec:correlations}) also suggested that introducing spatial correlations in the prior (Sec.~\ref{sec:priors}) might help.
We observe that introducing correlations indeed improves performance compared to the isotropic Gaussian prior (Fig.~\ref{fig:tempering_curves_cnns}).
Notably, the performance improvement is small for CNNs trained on MNIST and FashionMNIST, and for ResNets trained on CIFAR-10 at higher temperatures, but more considerable for ResNets at lower temperatures.
As such, correlated priors actually increase the magnitude of the cold posterior effect in ResNets trained on CIFAR-10.
This might be because ResNets trained at very low temperatures on CIFAR-10 have a tendency to overfit, and imposing the prior helps to mitigate this overfitting.
To support this hypothesis, we indeed see that correlated priors considerably improve over all other methods in terms of calibration and OOD detection at low temperatures for ResNets trained on CIFAR-10.

To reiterate a point raised in Sec.~\ref{sec:dnn_experiments}, the original cold posterior paper \citep{wenzel2020good} considered only predictive performance (error and likelihood), and not other measures of uncertainty such as calibration and OOD detection.
Indeed, we see different effects of temperature on these measures, with calibration improving for FashionMNIST at lower temperatures, but worsening for MNIST and CIFAR-10.
At the same time, we see performance at OOD detection worsen at lower temperatures in the smaller CNN model trained on MNIST and FashionMNIST, but increase at lower temperatures in the ResNet trained on CIFAR-10.
These results are consistent with other observations that measures of uncertainty do not necessarily correlate with predictive performance \citep{ovadia2019can,izmailov2021hmc}, and indicate that the cold-posterior effect is a complex phenomenon that demands careful future investigation.
Again, we see in Appendix~\ref{sec:activation_functions} and Appendix~\ref{sec:prior_variances} that these observations generalize to different activation functions and prior variances and in Appendix~\ref{sec:higher_temps} that warm posteriors ($T > 1$) deteriorate the performance for all considered priors.



In practice, models on this dataset are often trained using data augmentation (as is our model in Fig.~\ref{fig:tempering_curves_cnns}).
While this does indeed improve the performance (Fig.~\ref{fig:tempering_curves_cifar} in the appendix), it also strengthens the cold posterior effect.
When we do not use data augmentation, the cold posterior effect (at least between $T=1$ and lower temperatures) is almost entirely eliminated (see Fig.~\ref{fig:tempering_curves_cifar} in the appendix and \citealp{wenzel2020good,izmailov2021hmc}).
This observation raises the question of why data augmentation drives the cold posterior effect. Given that data augmentation adds terms to the likelihood while leaving the prior unchanged, we could expect that the problem is in the likelihood, as was recently argued by \citet{aitchison2020statistical}. On the other hand, \citet{vdw2018inv} argued that treating synthetic augmented data as extra datapoints for the purposes of the likelihood is incorrect from a Bayesian point of view. Instead, they express data augmentation in the prior, by constraining the classification functions to be invariant to certain transformations. More investigation is hence needed into how data augmentation and the cold posterior effect relate.


\section{Related Work}

\textbf{Empirical analysis of weight distributions.}
There is some history in neuroscience of analysing the statistics of data to inform inductive priors for learning algorithms, especially when it comes to vision \citep{simoncelli2009capturing}.
For instance, it has been noted that correlations help in modeling natural images \citep{srivastava2003advances}, as well as sparsity in the parameters \citep{smyth2003receptive,sahani2003evidence}. In the context of machine learning, the empirical weight distributions of standard neural networks have also been studied before \citep{bellido1993backpropagation, go1999analyzing}, including the insight that SGD can produce heavy-tailed weights \citep{gurbuzbalaban2020heavy}, but these works have not systematically compared different architectures and did not use their insights to inform Bayesian prior choices.

\textbf{BNNs in practice.}
Since the inception of Bayesian neural networks, scholars have thought about choosing good priors for them, including hierarchical \citep{mackay1992practical} and heavy-tailed ones \citep{neal1996bayesian}.
In the context of infinite-width limits of such networks \citep{lee2017deep, matthews2018gaussian, garriga2018deep, yang2019wide, tsuchida2019richer} it has also been shown that networks with very heavy-tailed (i.e., infinite variance) priors have very different properties from finite-variance priors \citep{neal1996bayesian, peluchetti2020stable}.
However, most modern applications of BNNs still relied on simple Gaussian priors. Although a few different priors have been proposed for BNNs, these were mostly designed for specific tasks \citep{atanov2018deep, ghosh2017model, overweg2019interpretable, nalisnick2018priors, cui2020informative, hafner2020noise} or relied heavily on non-standard inference methods \citep{sun2019functional, ma2019variational, karaletsos2020hierarchical, pearce2020expressive}.
Moreover, while many interesting distributions have been proposed as variational posteriors for BNNs \citep{louizos2017multiplicative, swiatkowski2020k, dusenberry2020efficient,ober2020global,aitchison2020deep}, these approaches have still used Gaussian priors. Others use a non-Gaussian prior, but approximate the posterior with a diagonal Gaussian \citep{blundell2015weight,ghosh2017model,nalisnick2015scalemixture}, somewhat limiting the prior's effect.
Another BNN posterior approximation is dropout \citep{gal2016dropout, kingma2015variational}, which is often poorly calibrated \citep{foong2019expressiveness}, but can also be seen to induce a scale-mixture prior, similar to our heavy-tailed priors \citep{molchanov2017variational}. 

\textbf{BNN priors.}
Finally, previous work has investigated the performance of neural network priors chosen without reference to the empirical distributions of SGD-trained networks \citep{blundell2015weight, ghosh2017model, wu2018deterministic, atanov2018deep,nalisnick2018priors,overweg2019interpretable,farquhar2019radial,cui2020informative, rothfuss2020pacoh,hafner2020noise,matsubara2020ridgelet,tran2020all,ober2020global,garriga2021correlated,fortuin2021priors, immer2021scalable}.
While these priors might in certain circumstances offer performance improvements, they did not offer a recipe for finding potentially valuable features to incorporate into the weight priors.
In contrast, we offer such a recipe by examining the distribution of weights trained under a uniform prior with SGD.
Importantly, unlike prior work, we use SG-MCMC with carefully evaluated convergence metrics and systematically address the cold posterior effect.


Contemporaneous work\footnote{released on arXiv two months after ours} \citep{izmailov2021hmc} compared gold-standard HMC inference with the more practical cyclic SGLD used in our work.
They confirmed that cyclic SGLD methods indeed have high-fidelity to the true posterior, and interestingly show that heavy-tailed priors offer slight performance improvements for language modelling tasks (though they do not assess the interaction of the cold posterior effect with these priors).









\section{Conclusion}
\label{sec:conclusion}

We consider empirical weight distributions in non-Bayesian networks trained using SGD, finding that FCNNs displayed heavy-tailed weight distributions, and CNNs and ResNets displayed spatial correlations in the convolutional filters.
We therefore tested the performance of these priors and their interaction with the cold posterior effect.
Indeed, we found that these priors improved performance, but their impact on the cold-posterior effect was more complex, with heavy-tailed priors in FCNNs eliminating the cold-posterior effect, correlated priors in CNNs trained on MNIST and FashionMNIST leaving the cold-posterior largely unchanged, and correlated priors in ResNets trained on CIFAR-10 actually increasing the cold-posterior effect, as it gives much larger performance improvements at lower temperatures.

Thus, we can conclude that isotropic Gaussian priors are often non-optimal, and that it is worth exploring other priors more generally (as always though, the correct prior will heavily depend on the architecture and dataset).
However, it is difficult to come to any strong conclusions as the origin of the cold-posterior effect.
At least in FCNNs, it does indeed appear that a misspecified prior can cause the cold-posterior effect.
However, in perhaps more relevant large-scale image models, we found that better (correlated) priors actually increase the cold posterior effect, which is consistent with other hypotheses, such as a misspecified likelihood \citep{aitchison2020statistical}, though of course we cannot rule out that there is a better prior that eliminates the cold posterior effect that we did not consider.












\bibliography{references}
\bibliographystyle{plainnat}





\newpage

\onecolumn

\appendix
\counterwithin{figure}{section}
\counterwithin{table}{section}






\section{Additional experimental results}
\label{sec:additional_exp}


\subsection{Covariance matrices}

Here we report the full covariance matrices for the layers that were analyzed above (Sec.~\ref{sec:correlations}).
The covariances of the FCNN weights are shown in Figure~\ref{fig:dnn_covar} and of the CNN weights in Figure~\ref{fig:cnn_covar}.


\begin{figure}[h]
    \centering
    \subfigure[Input weight covariance]{
    \includegraphics[width=0.49\linewidth]{201110_mnist_classificationdensenet_layer2_covar_inputs.pdf}}\hfill
    \subfigure[Output weight covariance]{
    \includegraphics[width=0.49\linewidth]{201110_mnist_classificationdensenet_layer2_covar_outputs.pdf}}\caption{Empirical covariances of the weights of FCNNs trained with SGD on MNIST. We see that they contain more systematic correlations than the isotropic Gaussian.}
    \label{fig:dnn_covar_l2}
\end{figure}


\begin{figure}[h]
    \centering
    \subfigure[Input covariance]{
    \centering
    \includegraphics[width=0.49\linewidth]{201110_mnist_classificationconvnet_layer2_covar_inputs.pdf}}\hfill
    \subfigure[Output covariance]{
    \centering
    \includegraphics[width=0.49\linewidth]{201110_mnist_classificationconvnet_layer2_covar_outputs.pdf}}\caption{Empirical covariances of the weights of CNNs trained with SGD on MNIST. We see that they also contain more systematic correlations than the isotropic Gaussian.}
    \label{fig:cnn_covar}
\end{figure}


\subsection{Empirical off-diagonal covariances}


\begin{figure}[p]
\centerline{
    \includegraphics[width=\linewidth]{figures/210125-corr-hist.pdf}}
    \caption{Distributions of off-diagonal elements in the empirical covariances of the weights of FCNNs and CNNs trained with SGD on MNIST. The empirical distributions are plotted as histograms, while the idealized random Gaussian weights are overlaid in orange. We see that the covariances of the empirical weights are more heavy-tailed than for the Gaussian weights.}
    \label{fig:dnn_cnn_offdiag}
\end{figure}



As an example, we report results for the distributions of off-diagonal covariances for the respective second layers of our FCNN and CNN in Figure~\ref{fig:dnn_cnn_offdiag}.
The empirical distribution of off-diagonal elements in the covariance matrices is shown as a histogram, overlayed with a kernel density estimate of the expected distribution if the weights were samples from an isotropic Gaussian.
We see that the empirical covariance distributions are generally more heavy-tailed than the ideal ones, that is, the empirical weights generally have larger covariances than would be expected from isotropic Gaussian weights.
Note that, as observed above, the strongest covariances by far are found spatially in the CNN weights, that is, between weights within the same CNN filter.
We report the same results for the other layers in the following.
The FCNN results are shown in Figures~\ref{fig:add_dnn_offdiag}, \ref{fig:dnn_covar_l2}, \ref{fig:dnn_covar_l3}, and \ref{fig:add_dnn_singvals} and the CNN results in Figures~\ref{fig:add_cnn_offdiag_singvals} and \ref{fig:cnn_covar_l1}.


\begin{figure}[p]
    \centering
    \subfigure[Layer 1, columns]{
    \centering
    \includegraphics[width=0.24\linewidth]{201110_mnist_classificationdensenet_layer1_offdiag_input_reduced.pdf}}\hfill
    \subfigure[Layer 1, rows]{
    \centering
    \includegraphics[width=0.24\linewidth]{201110_mnist_classificationdensenet_layer1_offdiag_output_reduced.pdf}}\hfill
    \subfigure[Layer 3, columns]{
    \centering
    \includegraphics[width=0.24\linewidth]{201110_mnist_classificationdensenet_layer3_offdiag_input_reduced.pdf}}\hfill
    \subfigure[Layer 3, rows]{
    \centering
    \includegraphics[width=0.24\linewidth]{201110_mnist_classificationdensenet_layer3_offdiag_output_reduced.pdf}}\caption{Distributions of off-diagonal elements in the empirical covariances of the weights of the FCNN in the other layers. The empirical distributions are plotted as histograms, while the idealized random Gaussian weights are overlaid in orange. We see that the covariances of the empirical weights are more heavy-tailed than for the Gaussian weights.}
    \label{fig:add_dnn_offdiag}
\end{figure}

\begin{figure}[p]
    \centering
    \subfigure[Input weight covariance]{
    \includegraphics[width=0.49\linewidth]{201110_mnist_classificationdensenet_layer1_covar_inputs.pdf}}\hfill
    \subfigure[Output weight covariance]{
    \includegraphics[width=0.49\linewidth]{201110_mnist_classificationdensenet_layer1_covar_outputs.pdf}}\caption{Empirical covariances of the weights of the FCNN in the first layer. We can see correlations in the spatial direction in the weights of the input layer (left). In the other directions, the covariance matrix is less smooth than we would expect from an isotropic Gaussian draw of the same size (left matrix of every pair), but otherwise has no structure. This suggests that the weights are not isotropic Gaussian.}
    \label{fig:dnn_covar}
\end{figure}

\begin{figure}[p]
    \centering
    \subfigure[Input weight covariance]{
    \includegraphics[width=0.49\linewidth]{201110_mnist_classificationdensenet_layer3_covar_inputs.pdf}}\hfill
    \subfigure[Output weight covariance]{
    \includegraphics[width=0.49\linewidth]{201110_mnist_classificationdensenet_layer3_covar_outputs.pdf}}\caption{Empirical covariances of the weights of the FCNN in the third layer. We can see that the covariance matrix is less smooth than we would expect from an isotropic Gaussian draw of the same size (left matrix of every pair), but otherwise has no structure. This suggests that the weights are not isotropic Gaussian.}
    \label{fig:dnn_covar_l3}
\end{figure}

\begin{figure}[hp]
    \centering
    \subfigure[Second layer]{
        \centering
    \includegraphics[width=0.49\linewidth]{201110_mnist_classificationdensenet_layer2_singvals.pdf}}\hfill
    \subfigure[Third layer]{
    \centering
    \includegraphics[width=0.49\linewidth]{201110_mnist_classificationdensenet_layer3_singvals.pdf}}\caption{Distributions of singular values of the weight matrices of the FCNN in the other layers. We see that the spectra of the empirical weights decay faster than the ones of the Gaussian weights.}
    \label{fig:add_dnn_singvals}
\end{figure}


\begin{figure}[hp]
    \hfill
    \subfigure[Layer 1, columns]{
    \includegraphics[width=0.24\linewidth]{201110_mnist_classificationconvnet_layer1_offdiag_input_reduced.pdf}}\hfill
    \subfigure[Layer 1, rows]{
    \centering
    \includegraphics[width=0.24\linewidth]{201110_mnist_classificationconvnet_layer1_offdiag_output_reduced.pdf}}\hfill
    \subfigure[Layer 1, singular values]{
    \centering
    \includegraphics[width=0.49\linewidth]{201110_mnist_classificationconvnet_layer1_singvals.pdf}}\caption{Distributions of off-diagonal elements in the empirical covariances of the weights and singular values of the CNN in the other layer. The empirical distributions are plotted as histograms, while the idealized random Gaussian weights are overlaid as an orange line. We see that the covariances of the empirical weights are more heavy-tailed than for the Gaussian weights and that the singular value spectrum for the empirical weights decays faster than the Gaussian ones.}
    \label{fig:add_cnn_offdiag_singvals}
\end{figure}


\begin{figure}[hp]
    \centerline{
    \subfigure[Input weight covariance]{
    \includegraphics[width=0.49\linewidth]{201110_mnist_classificationconvnet_layer1_covar_inputs.pdf}}\hfill
    \subfigure[Output weight covariance]{
    \includegraphics[width=0.49\linewidth]{201110_mnist_classificationconvnet_layer1_covar_outputs.pdf}}}\caption{Empirical covariances of the weights of the CNN in the first layer. We see that they contain more systematic correlations than the isotropic Gaussian.}
    \label{fig:cnn_covar_l1}
\end{figure}

\begin{figure}[hp]
    \centerline{
    \includegraphics[width=\linewidth]{figures/210122_resnet_fitted.pdf}}
    \caption{Right: fitted lengthscale of a multivariate Gaussian with a squared exponential kernel (see eq.~\ref{eq:rbf_filter_cov}) to the data of Figure~\ref{fig:resnet_covariances}. All the entries of the SE covariance are positive, so this cannot capture all the features of the data, which has negative empirical covariance.
    Left: fitted degrees of freedom of a multivariate t-distribution, to same data. The empirical covariance was used in this case. The fitting criterion is the log-likelihood of the data.}
    \label{fig:resnet_df_lengthscale}
\end{figure}















\subsection{The influence of data augmentation on the cold posterior effect}
\label{sec:data_augmentation}

When running the CIFAR-10 experiments with Bayesian ResNets with and without data augmentation, we find that data augmentation seems to significantly increase the cold posterior effect (Fig.~\ref{fig:tempering_curves_cifar}).
Moreover, data augmentation seems to increase the performance of the models a lot at colder temperatures, but not at the true Bayes posterior $T=1$.
This suggests that data augmentation can also be one of the reasons for the cold posterior effect, as already hypothesized by \citet{wenzel2020good} and \citet{aitchison2020statistical}.



\subsection{SGD baselines\label{sec:sgd_baselines}}

In terms of likelihood, calibration, and OOD detection, almost all our BNN models consistently outperformed the SGD baselines.
The results including SGD are shown for FCNNs in Figure~\ref{fig:tempering_curves_fcnns_sgd}, for CNNs in Figure~\ref{fig:tempering_curves_cnns_sgd}, and for ResNets in Figure~\ref{fig:tempering_curves_cifar_sgd}.


\begin{figure*}[h]
    \centerline{
    \includegraphics[width=\linewidth,page=3]{figures/temp_curves/tempering_curves_compressed.pdf}}
    \caption{Performances of Bayesian ResNets with different priors on CIFAR-10 with and without data augmentation in terms of different metrics. Data augmentation seems to increase the cold posterior effect.}
    \label{fig:tempering_curves_cifar}
\end{figure*}

\begin{figure}[h]
    \centerline{
    \includegraphics[width=\linewidth,page=4]{figures/temp_curves/tempering_curves_compressed.pdf}}
    \caption{Performances of fully connected BNNs with different priors on MNIST and FashionMNIST in terms of different metrics, compared to SGD solutions. The heavy-tailed priors generally perform better, especially at higher temperatures, and lead to a less pronounced cold posterior effect.}
    \label{fig:tempering_curves_fcnns_sgd}
\end{figure}

\begin{figure}[h]
    \centerline{
    \includegraphics[width=\linewidth,page=5]{figures/temp_curves/tempering_curves_compressed.pdf}}
    \caption{Performances of convolutional BNNs with different priors on MNIST and FashionMNIST in terms of different metrics, compared to SGD solutions. The correlated prior generally performs better than the isotropic ones, but still exhibits a cold posterior effect, while the heavy-tailed priors reduce the cold posterior effect, but yield a worse performance.}
    \label{fig:tempering_curves_cnns_sgd}
\end{figure}

\subsection{Alternative activation functions}
\label{sec:activation_functions}

We repeated the experiments on MNIST with Bayesian FCNNs and CNNs and replaced the ReLU activation functions from Figure~\ref{fig:tempering_curves_fcnns} and Figure~\ref{fig:tempering_curves_cnns} with sigmoid (see Fig.~\ref{fig:temp_curves_sigmoid}) and tanh (see Fig.~\ref{fig:temp_curves_tanh}) activations respectively.
We observe that while the performances are overall worse than with ReLU activations (as is generally expected), the effects of the different priors are qualitatively very similar.

\begin{figure}[h]
    \centerline{
    \includegraphics[width=\linewidth,page=6]{figures/temp_curves/tempering_curves_compressed.pdf}}
    \caption{Performances of Bayesian ResNets with different priors on CIFAR-10 with and without data augmentation in terms of different metrics, compared to SGD solutions. The correlated prior generally outperforms the other ones. Moreover, data augmentation seems to increase the cold posterior effect.}
    \label{fig:tempering_curves_cifar_sgd}
\end{figure}



\begin{figure}[h]
    \centerline{
    \includegraphics[width=\linewidth,page=8]{figures/temp_curves/tempering_curves_compressed.pdf}}
    \caption{Performances of fully-connected and convolutional BNNs with sigmoid activation functions on MNIST. The observed effects are qualitatively similar to the ones with ReLU activations in the main body of the paper.}
    \label{fig:temp_curves_sigmoid}
\end{figure}

\begin{figure}[h]
    \centerline{
    \includegraphics[width=\linewidth,page=9]{figures/temp_curves/tempering_curves_compressed.pdf}}
    \caption{Performances of fully-connected and convolutional BNNs with tanh activation functions on MNIST. The observed effects are qualitatively similar to the ones with ReLU activations in the main body of the paper.}
    \label{fig:temp_curves_tanh}
\end{figure}


\subsection{Higher temperatures}
\label{sec:higher_temps}

In the main body of the paper, we followed \citet{wenzel2020good} in showing only posteriors with temperatures $T \leq 1$, because we were interested in studying cold posteriors.
Here, we also show results for warm posteriors, that is, $T > 1$.
We see in Figure~\ref{fig:tempering_curves_highT_fcnn} and Figure~\ref{fig:tempering_curves_highT_cnn} that these warm posteriors generally do not improve the performance and that hence some of the priors (e.g., heavy-tailed priors in FCNNs) do indeed achieve their optimal performance for $T \approx 1$.

\begin{figure}[h]
    \centerline{
    \includegraphics[width=\linewidth,page=10]{figures/temp_curves/tempering_curves_compressed.pdf}}
    \caption{Performances of Bayesian FCNNs with different priors on (Fashion-)MNIST, including temperatures $T > 1$. The performances generally do not improve for warm posteriors, such that $T \approx 1$ is indeed optimal for some priors.}
    \label{fig:tempering_curves_highT_fcnn}
\end{figure}


\begin{figure}[h]
    \centerline{
    \includegraphics[width=\linewidth,page=11]{figures/temp_curves/tempering_curves_compressed.pdf}}
    \caption{Performances of Bayesian CNNs and Resnets with different priors on (Fashion-)MNIST and CIFAR, including temperatures $T > 1$. The performances generally do not improve for warm posteriors, such that $T \approx 1$ is indeed optimal for some priors. Note that here, we do not use data augmentation for CIFAR.}
    \label{fig:tempering_curves_highT_cnn}
\end{figure}



\subsection{Different prior variances}
\label{sec:prior_variances}

In the main text, we use models where the prior variance is chosen according to the He initialization \citep{he2016deep}, which is motivated by the conservation of the activation norm across the depth of the networks.
Here, we see in Figures~\ref{fig:tempering_curves_vars_fcnn_mnist}, \ref{fig:tempering_curves_vars_cnn_mnist}, \ref{fig:tempering_curves_vars_fcnn_fmnist}, \ref{fig:tempering_curves_vars_cnn_fmnist}, and \ref{fig:tempering_curves_vars_cifar} that our main observations regarding the ordering of the different priors and the cold posterior effect still hold, even for different prior variances (in this case, four times larger and smaller than the He variance).

\begin{figure}[h]
    \centerline{
    \includegraphics[width=\linewidth,page=12]{figures/temp_curves/tempering_curves_compressed.pdf}}
    \caption{Performances of Bayesian FCNNs with different priors and different prior variances on MNIST. The qualitative behavior is similar to the one for the He variance in the main text.}
    \label{fig:tempering_curves_vars_fcnn_mnist}
\end{figure}

\begin{figure}[h]
    \centerline{
    \includegraphics[width=\linewidth,page=13]{figures/temp_curves/tempering_curves_compressed.pdf}}
    \caption{Performances of Bayesian CNNs with different priors and different prior variances on MNIST. The qualitative behavior is similar to the one for the He variance in the main text.}
    \label{fig:tempering_curves_vars_cnn_mnist}
\end{figure}

\begin{figure}[h]
    \centerline{
    \includegraphics[width=\linewidth,page=14]{figures/temp_curves/tempering_curves_compressed.pdf}}
    \caption{Performances of Bayesian FCNNs with different priors and different prior variances on Fashion-MNIST. The qualitative behavior is similar to the one for the He variance in the main text.}
    \label{fig:tempering_curves_vars_fcnn_fmnist}
\end{figure}

\begin{figure}[h]
    \centerline{
    \includegraphics[width=\linewidth,page=15]{figures/temp_curves/tempering_curves_compressed.pdf}}
    \caption{Performances of Bayesian CNNs with different priors and different prior variances on Fashion-MNIST. The qualitative behavior is similar to the one for the He variance in the main text.}
    \label{fig:tempering_curves_vars_cnn_fmnist}
\end{figure}

\begin{figure}[h]
    \centerline{
    \includegraphics[width=\linewidth,page=16]{figures/temp_curves/tempering_curves_compressed.pdf}}
    \caption{Performances of Bayesian Resnets with different priors and different prior variances on CIFAR-10. The qualitative behavior is similar to the one for the He variance in the main text. Note that here, we do not use data augmentation.}
    \label{fig:tempering_curves_vars_cifar}
\end{figure}




\subsection{UCI regression}
\label{sec:uci_regression}

While the experiments in the main paper focus on image classification, we also
performed BNN experiments on UCI regression tasks. The architecture is a 3-layer
FCNN, the hidden layers are 64 units wide. We run GGMC for 30,000 epochs
without minibatching on ``boston'', ``energy'', ``yacht'', and ``wine'',
discarding runs where the potential diverges. For the other datasets, which are larger, we run $3000$
epochs, also without minibatching. The learning rate is a flat $5\cdot 10^{-5}$,
and we do not use a cosine schedule.

Even with full batch MCMC, it is clear that the dynamics for regression networks
are much less stable, especially at lower temperatures (which have a ``sharper''
potential landscape). Figure~\ref{fig:temp-uci} shows that $T=1$ is best for all
datasets, in terms of the median mean squared error (MSE) as well as the
quantiles and outliers.
We also see in Table~\ref{tbl:uci-rhat} that the Gaussian prior is only superior for 2/9 datasets, while on all the others, one of the heavy-tailed priors performs better.
However, the priors are generally reasonably close in performance, such that it is harder in this case to strongly prescribe a certain prior choice.

The split-$\Rhat$ diagnostics  are considerably higher for regression (Table~\ref{tbl:uci-rhat}) than for the classification setting
(Sec.~\ref{sec:rhat_diagnostic}, for which the diagnostics look very good). Thus, the results here should be taken with a grain of salt.
They are representative of how GGMC-trained BNNs behave at each of these
temperatures and priors, but the results may be different for other (more accurate) ways of approximating the posterior.
However, one thing is clear: UCI regression datasets exhibit no cold posterior effect, and for lower temperatures, the GGMC chains are less stable.

\begin{figure}[h]
 \centerline{
  \includegraphics[width=\linewidth]{figures/040621-temp-uci.pdf}}
\caption{Box plots of the mean-squared error of Bayesian FCNNs doing regression on UCI datasets. For each temperature, and prior, each box displays the median $\pm 1.5$ times the inter-quartile range. Outliers are plotted as $\times$.
  We exclude runs where the potential diverges. Temperature 1 is clearly best for all datasets, but otherwise there is no clear trend.}
\label{fig:temp-uci}
\end{figure}

\begin{table}[ht]
  \caption{Diagnostics and median performance at temperature $T=1$ for every prior and UCI dataset. The split-$\Rhat$ is generally high, which shows the chain has not fully explored the posterior. The best prior (in terms of median mean squared error, MSE) for each dataset is bolded, no prior is better overall. Additionally, each prior's performance is very similar for each dataset, which implies that the choice of prior does not matter much here (at least among these three).}
\label{tbl:uci-rhat}
\centerline{
\begin{tabular}{lcccccc}
\toprule
{} & \multicolumn{3}{c}{split-$\Rhat$ diagnostic}  &  \multicolumn{3}{c}{Median MSE} \\
{} &   laplace &  gaussian &  student-t &   laplace &  gaussian &  student-t \\
\midrule
boston   &  1.984856 &  1.664200 &   2.164047 &  \textbf{0.051899} &  0.061199 &   0.056180 \\
concrete &  1.923906 &  1.960722 &   1.654736 &  0.962948 &  0.802203 &   \textbf{0.728850} \\
energy   &  2.026471 &  1.939554 &   2.309070 &  0.000759 &  \textbf{0.000705} &   0.001500 \\
kin8nm   &  1.523657 &  1.795404 &   1.609185 &  0.976947 &  1.520133 &   \textbf{0.958203} \\
naval    &  1.506857 &  1.700794 &   1.583324 &  \textbf{1.241157} &  1.146679 &   1.679176 \\
power    &  1.666368 &  1.738743 &   2.471113 &  0.156302 &  0.286625 &   \textbf{0.148974} \\
protein  &  1.574833 &  2.037037 &   1.510434 &  \textbf{1.151049} &  1.296588 &   1.259395 \\
wine     &  2.133220 &  1.936430 &   1.844044 &  0.674713 &  0.617795 &   \textbf{0.604185} \\
yacht    &  2.034536 &  1.880282 &   2.414279 &  0.000612 &  \textbf{0.000559} &   0.000599 \\
  \bottomrule
\end{tabular}}
\end{table}



\subsection{Inference diagnostics}
\label{sec:inf_diagnostics}


\begin{figure}[t]
\begin{minipage}{0.42\linewidth}
\centerline{
\includegraphics[width=\linewidth, page=1]{figures/diagnostics/diagnostics_merged_compressed.pdf}}
\captionof{figure}{Kinetic temperature diagnostics of the ResNet CIFAR-10 experiments with data augmentation. We see that the kinetic temperatures agree almost perfectly with the target temperature of the sampler.}
    \label{fig:temp_diag_main}
\end{minipage}\hfill
\begin{minipage}{0.53\linewidth}
\captionof{table}{Worst (highest) $\Rhat$ values for different models and neuron-permutation-invariant functions.}
\label{tab:rhat_worst}
\centerline{
\resizebox{\linewidth}{!}{
\begin{tabular}{lccc}
\toprule
{} &  Loss &  Potential &  Log-prior \\
\midrule
FCNN MNIST         & 1.006 & 1.023 & 1.101 \\
FCNN Fashion  & 1.007 & 1.013 & 1.104 \\
CNN MNIST          & 1.002 & 1.001 & 1.013 \\
CNN Fashion   & 1.009 & 1.007 & 1.013 \\
ResNet CIFAR-10   & 1.125 & 1.171 & 1.404 \\
ResNet C.-10 (aug) & 1.066 & 1.090 & 1.346 \\
\bottomrule
\end{tabular}}}
\end{minipage}
\end{figure}

One of the main goals of our work is to make statements about the true BNN posteriors that are as accurate as possible. To this end, we closely monitored the accuracy of our inference algorithm.
In order to check the correctness of our SG-MCMC inference, we estimated the temperature of the sampler using the two diagnostics from \citet{wenzel2020good}, namely the \emph{kinetic temperature} and the \emph{configurational temperature}.

The kinetic temperature is derived from the sampler's momentum $\vm \in \mathbb{R}^d$. The inner product $\frac{1}{d} \vm^\tp \mM^{-1} \vm$, for the (in this case diagonal) mass matrix $\mM$, is an estimate of the scaled variance of the momenta. If the sampler is correct it should, in expectation, be equal to the desired temperature.
The configurational temperature is slightly more involved and is discussed in Appendix~\ref{sec:add_temp_diagnostics}.

As an example, we show the estimated kinetic temperatures for our ResNet experiment on CIFAR-10 in Figure~\ref{fig:temp_diag_main}. The desired temperature is shown as a dotted horizontal line. The kinetic temperatures for the other experiments look qualitatively similar and are shown in Appendix~\ref{sec:add_temp_diagnostics}.
We see that the kinetic temperatures generally agree well with the true temperatures, so the sampler works as expected there.
In contrast, the configurational temperature estimates can be somewhat larger than $T$, especially when $T$ is small (see Appendix~\ref{sec:add_temp_diagnostics}). This suggests that there could be small inference inaccuracies at low temperatures.
However, these inaccuracies are small, and the configurational temperature certainly decreases as $T$ decreases, so there should be no impact on the overall trends.


We also computed the rank-normalized split-$\Rhat$ diagnostic \cite{vehtari2021rank}, which measures how well a collection of independent Markov chains have mixed. The split-$\Rhat$ is related to the ratio of between-chain and within-chain variances, and should be as close to 1 as possible.
Given the complexity of neural network weight posteriors, we report the $\Rhat$ for the quantities we are interested in estimating (the y-values in Figs.~\ref{fig:tempering_curves_fcnns} and~\ref{fig:tempering_curves_cnns}). For every considered model and function, Table~\ref{tab:rhat_worst} contains the worst (highest) $\Rhat$ estimate we obtained across all priors.
Appendix~\ref{sec:rhat_diagnostic} contains a more detailed explanation and empirical $\Rhat$ estimates for different priors.

We can see that, for most experiments, the chains have mixed sufficiently.
Only for the larger models (CIFAR10 ResNets)---and, to a lesser extent, Student-t FCNNs---the chains have mixed less well. Interestingly,  for all convolutional networks, the correlated prior mixes best. This further supports its suitability as a prior for image data and CNNs.



\subsubsection{Kinetic and configurational temperature estimates}
\label{sec:add_temp_diagnostics}

As described above, we use two temperature diagnostics (inspired by \citet{wenzel2020good}): the \emph{kinetic temperature} and the \emph{configurational temperature}.
The kinetic temperature is derived from the sampler's momentum $\vm \in \mathbb{R}^d$. The inner product $\frac{1}{d} \vm^\tp \mM^{-1} \vm$, for the (in this case diagonal) mass matrix $\mM$, is an estimate of the scaled variance of the momenta. It is always positive and should, in expectation, be equal to the desired temperature. In contrast, the configurational temperature is $\frac{1}{d} \vtheta^\tp \nabla H(\vtheta, \vm)$, where $H(\vtheta, \vm) = -\log p(\vtheta \,|\, \mathcal{D}) + \frac{1}{2}\vm^\tp \mM^{-1}\vm + \text{const}$ is the Hamiltonian. In expectation, this should also equal $T$. Unlike the kinetic temperature estimator, the configurational temperature estimator is not guaranteed to be always positive, even though the temperature \emph{is} always positive. Using subsets of a parameter or momentum also yields estimators of the temperature.

In both cases, we estimate the mean and its standard error from a weighted average of parameters or momenta. That is, for each separate NN weight matrix or bias vector, we estimate its kinetic and configurational temperature using the expressions above. Then, we take their average and standard-deviation, weighted by the number of elements in that parameter matrix or vector. 

We show the estimated temperatures of all our BNN experiments in Figures~\ref{fig:temp_diag_mnist}, \ref{fig:temp_diag_mnist_cnn}, \ref{fig:temp_diag_fashionmnist}, \ref{fig:temp_diag_fashionmnist_cnn}, \ref{fig:temp_diag_cifar}, and \ref{fig:temp_diag_cifar_augmented}, as a mean $\pm$ one standard error. The desired temperature is shown as a dotted horizontal line.
The kinetic temperatures generally agree well with the true temperatures, so our sampler works as expected there.

The configurational temperature estimates have a higher variance than the kinetic ones.
Especially in the regime of small true temperatures, they often tend to slightly over- or underestimate the temperature.
This is not surprising, since at low temperatures the noise in the gradients is dominated by the minibatching as opposed to the temperature noise.
Correctly estimating the temperature from the gradients thus becomes harder.

Note that while the relative deviations can seem large in this regime, the absolute deviations are still quite small.
Note also that while the conditioned momenta are strictly positive, the inner products between gradients and parameters can become negative in principle, which is why at low temperatures (close to 0) the configurational temperature estimates might sometimes be a bit below 0.
Overall, the sampler is still within the tolerance levels of working correctly here, but there could be some small inaccuracies at low temperatures.
However, judging from the shape of the actual tempering curves (see Sec.~\ref{sec:experiments}), the measures usually change more in the higher temperature regimes than in the lower ones, so there is no strong reason to believe that the inference at low temperatures was too inaccurate to support the results.

\begin{figure}[h]
    \centerline{
    \includegraphics[width=\linewidth,page=7]{figures/diagnostics/diagnostics_merged_compressed.pdf}}
    \caption{Temperature diagnostics of the MNIST experiment with FCNNs.}
    \label{fig:temp_diag_mnist}
\end{figure}

\begin{figure}[h]
    \centerline{
    \includegraphics[width=\linewidth,page=6]{figures/diagnostics/diagnostics_merged_compressed.pdf}}
    \caption{Temperature diagnostics of the MNIST experiment with CNNs.}
    \label{fig:temp_diag_mnist_cnn}
\end{figure}

\begin{figure}[h]
    \centerline{
    \includegraphics[width=\linewidth,page=5]{figures/diagnostics/diagnostics_merged_compressed.pdf}}
    \caption{Temperature diagnostics of the FashionMNIST experiment with FCNNs.}
    \label{fig:temp_diag_fashionmnist}
\end{figure}

\begin{figure}[h]
    \centerline{
    \includegraphics[width=\linewidth,page=4]{figures/diagnostics/diagnostics_merged_compressed.pdf}}
    \caption{Temperature diagnostics of the FashionMNIST experiment with CNNs.}
    \label{fig:temp_diag_fashionmnist_cnn}
\end{figure}

\begin{figure}[h]
    \centerline{
    \includegraphics[width=\linewidth,page=3]{figures/diagnostics/diagnostics_merged_compressed.pdf}}
    \caption{Temperature diagnostics of the CIFAR-10 experiment with ResNets without data augmentation.}
    \label{fig:temp_diag_cifar}
\end{figure}

\begin{figure}[h]
    \centerline{
    \includegraphics[width=\linewidth,page=2]{figures/diagnostics/diagnostics_merged_compressed.pdf}}
    \caption{Temperature diagnostics of the CIFAR-10 experiment with ResNets with data augmentation.}
    \label{fig:temp_diag_cifar_augmented}
\end{figure}

\subsubsection{Between-chain and within-chain variances}
\label{sec:rhat_diagnostic}

The split-$\Rhat$ estimator measures the difference between posterior variance estimate in each chain, and between chains. It is roughly the square root of the between-chain variance divided by the within-chain variance \citep[eq.~1--3]{vehtari2021rank}. Its value is usually not smaller than 1, and a chain that has mixed well should have a value no larger than $\Rhat \le 1.01$ \citep{vehtari2021rank}. (Previously, a threshold of $1.1$ was considered enough \citep[Section~11.5]{gelman2013bayesian}.)

Neural network functional forms have a large number of parameter symmetries (for example, permutation invariance). Accordingly, the true BNN posterior should sample from all these modified parameters with probability proportional to their prior. However, for prediction purposes, it does not matter if the parameters are stuck in a single ``permutation'' and do not mix.

Therefore, for the purposes of this paper, we calculate the $\Rhat$ diagnostic not directly on the parameters, but on symmetry-invariant functions of the parameters. In practice, this amounts to evaluating the NN on a test set, and calculating the $\Rhat$ diagnostic for functions of the logits and the prior probability. Tables~\ref{tab:rhat_loss}, \ref{tab:rhat_potential} and~\ref{tab:rhat_log_prior} display the value of the diagnostic $\hat{R}$ for different such functions: the log-likelihood, the unnormalized log-posterior (potential), and  the log-prior, respectively.

We employ the rank-normalized $\Rhat$ estimator \citep[eq.~14]{vehtari2021rank} as implemented in the \texttt{ArviZ} library \citep{arviz_2019}.

The diagnostics are generally favorable ($\Rhat \le 1.01$, mostly) for smaller NNs (FCNNs and 2-layer CNNs) and for MNIST. Within the ResNets applied to CIFAR10, the prior distribution with the $\Rhat$ closer to 1 is the correlated Gaussian. This provides evidence that inference is easier in the case of the correlated Gaussian, and therefore that the correlated Gaussian is a better prior \citep{gelman2013bayesian, yang2015improving}. This is because if the prior is good, the data are plausible simulations from it; so the posterior is close to the prior and will be easy to approximate.

\begin{table}
\centering
\caption{Estimated $\Rhat$ values for the different models and priors with respect to the loss.}
\label{tab:rhat_loss}
\begin{tabular}{lrrrr}
\toprule
{} &  Gaussian &  Laplace &  Student-t &  Correlated \\
\midrule
MNIST FCNN                 &     1.000 &    1.001 &      1.006 &           - \\
FashionMNIST FCNN          &     1.000 &    1.000 &      1.007 &           - \\
MNIST CNN                  &     1.000 &    1.000 &      1.002 &       1.000 \\
FashionMNIST CNN           &     1.001 &    1.003 &      1.009 &       1.001 \\
CIFAR10 ResNet             &     1.115 &    1.115 &      1.125 &       1.109 \\
CIFAR10 ResNet (augmented) &     1.057 &    1.054 &      1.047 &       1.066 \\
\bottomrule
\end{tabular}
\end{table}


\begin{table}
\centering
\caption{Estimated $\Rhat$ values for the different models and priors with respect to the potential.}
\label{tab:rhat_potential}
\begin{tabular}{lrrrr}
\toprule
{} &  Gaussian &  Laplace &  Student-t &  Correlated \\
\midrule
MNIST FCNN                 &     1.000 &    1.002 &      1.023 &           - \\
FashionMNIST FCNN          &     1.000 &    1.000 &      1.013 &           - \\
MNIST CNN                  &     1.000 &    1.000 &      1.001 &       1.000 \\
FashionMNIST CNN           &     1.000 &    1.002 &      1.007 &       1.000 \\
CIFAR10 ResNet             &     1.166 &    1.147 &      1.171 &       1.139 \\
CIFAR10 ResNet (augmented) &     1.085 &    1.083 &      1.073 &       1.090 \\
\bottomrule
\end{tabular}
\end{table}


\begin{table}
\centering
\caption{Estimated $\Rhat$ values for the different models and priors with respect to the log prior.}
\label{tab:rhat_log_prior}
\begin{tabular}{lrrrr}
\toprule
{} &  Gaussian &  Laplace &  Student-t &  Correlated \\
\midrule
MNIST FCNN                 &     1.000 &    1.005 &      1.101 &           - \\
FashionMNIST FCNN          &     1.000 &    1.003 &      1.104 &           - \\
MNIST CNN                  &     1.001 &    1.002 &      1.013 &       1.001 \\
FashionMNIST CNN           &     1.002 &    1.006 &      1.013 &       1.001 \\
CIFAR10 ResNet             &     1.404 &    1.232 &      1.366 &       1.195 \\
CIFAR10 ResNet (augmented) &     1.274 &    1.346 &      1.264 &       1.198 \\
\bottomrule
\end{tabular}
\end{table}


\section{Evaluation Metrics}
\label{sec:metrics}

When using BNNs, practitioners might care about different outcomes.
In some applications, the predictive accuracy might be the only metric of interest, while in other applications calibrated uncertainty estimates could be crucial.
We therefore use a range of different metrics in our experiments in order to highlight the respective strengths and weaknesses of different priors.
Moreover, we compare the priors to the empirical weight distributions of conventionally trained networks.





\subsection{Empirical test performance}


\paragraph{Test error}

The test error is probably the most widely used metric in supervised learning.
It intuitively measures the performance of the model on a held-out test set and is often seen as an empirical approximation to the true generalization error.
While it is often used for model selection, it comes with the risk of overfitting to the used test set \citep{bishop2006pattern} and in the case of BNNs also fails to account for the predictive variance of the posterior.

\paragraph{Test log-likelihood}

The predictive log-likelihood also requires a test set for its evaluation, but it takes the predictive posterior variance into account.
It can thus offer a built-in tradeoff between the mean fit and the quality of the uncertainty estimates.
Moreover, it is a proper scoring rule \citep{gneiting2007strictly}.

\subsection{Uncertainty estimates}

\paragraph{Uncertainty calibration}

Bayesian methods are often chosen for their superior uncertainty estimates, so many users of BNNs will not be satisfied with only fitting the posterior mean well.
The calibration measures how well the uncertainty estimates of the model correlate with predictive performance.
Intuitively, when the model is for instance 70~\% certain about a prediction, this prediction should be correct with 70~\% probability.
Many deep learning models are not well calibrated, because they are often overconfident and assign too low uncertainties to their predictions \citep{ovadia2019can, wenzel2020hyper}.
When the models are supposed to be used in safety-critical scenarios, it is often crucial to be able to tell when they encounter an input that they are not certain about \citep{kendall2017uncertainties}.
For these applications, metrics such as the expected calibration error \citep{naeini2015obtaining} might be the most important criteria.

\paragraph{Out-of-distribution detection}

The out-of-distribution (OOD) detection measures how well one can tell in-distribution and out-of-distribution examples apart based on the uncertainties.
This is important when we believe that the model might be deployed under some degree of dataset shift.
In this case, the model should be able to detect these OOD examples and be able to reject them, that is, refuse to make a prediction on them.




\section{Details about the considered priors}
\label{sec:prior_details}

We contrast the widely used isotropic Gaussian priors with heavy-tailed distributions, including the Laplace and Student-t distributions, and with correlated Gaussian priors.
We chose these distributions based on our observations of the empirical weight distributions of SGD-trained networks (see Sec.~\ref{sec:weight_dists}) and for their ease of implementation and optimization.
We now give a quick overview over these different distributions and their most salient properties.


\paragraph{Gaussian.}

The isotropic Gaussian distribution \citep{gauss1809theoria} is the \emph{de-facto} standard for BNN priors in recent work \citep[e.g.,][]{hernandez2015probabilistic, louizos2017multiplicative, dusenberry2020efficient, wenzel2020good, neal1992bayesian, zhang2019cyclical, osawa2019practical, immer2021improving, lee2017deep, garriga2018deep}.
Its probability density function (PDF) is
\begin{equation*}
    p(x; \mu, \sigma^2) = \frac{1}{\sqrt{2 \pi \sigma^2}} \exp \left( - \frac{(x - \mu)^2}{2 \sigma^2} \right)
\end{equation*}
with mean $\mu$ and standard deviation $\sigma$.
It is attractive, because it is the central limit of all finite-variance distributions \citep{billingsley1961lindeberg} and the maximum entropy distribution for a given mean and scale \citep{bishop2006pattern}.
However, its tails are relatively light compared to some of the other distributions that we will consider.

\paragraph{Laplace.}




The Laplace distribution \citep{laplace1774memoire} has heavier tails than the Gaussian and is discontinuous at $x = \mu$.
Its PDF is
\begin{equation*}
    p(x ; \mu, b) = \frac{1}{2b} \exp \left( - \frac{|x-\mu|}{b} \right)
\end{equation*}
with mean $\mu$ and scale $b$.
It is often used in the context of (frequentist) \emph{lasso} regression \citep{tibshirani1996regression}.


\paragraph{Student-t.}


The Student-t distribution characterizes the mean of a finite number of samples from a Gaussian distribution \citep{student1908probable}.
Its PDF is
\begin{equation*}
    p(x ; \mu, \nu) = \frac{\Gamma(\frac{\nu+1}{2})}{ \Gamma(\frac{\nu}{2}) \sqrt{\nu \pi}} \left( 1 + \frac{(x - \mu)^2}{\nu} \right)^{- \frac{\nu+1}{2}} \;,
\end{equation*}
where $\mu$ is the mean, $\Gamma$ is the gamma function, and $\nu$ are the degrees of freedom.
The Student-t also arises as the marginal distribution over Gaussians with an inverse-Gamma prior over the variances \citep{helmert1875berechnung, luroth1876vergelichung}.
For $\nu \rightarrow \infty$, the Student-t distribution approaches the Gaussian.
For any finite $\nu$ it has heavier tails than the Gaussian.
Its $k$-th moment is only finite for $\nu > k$.
The $\nu$ parameter thus offers a convenient way to adjust the heaviness of the tails.
Note that it also controls the variance of the distribution, which is $\nu /(\nu - 2)$ (or else undefined).
Unless otherwise stated, we set $\nu = 3$ in our experiments, such that the distribution has rather heavy tails, while still having a finite mean and variance.

\paragraph{Multivariate Gaussian with Matérn covariance.}

For our correlated Bayesian CNN priors, we use multivariate Gaussian priors
\begin{align*}
    p(\vx; \vmu, \mSigma) &= \frac{1}{\sqrt{(2 \pi)^d \det\mSigma}} \exp \left( - \frac{1}{2} \| \vx - \vmu \|_{\mSigma^{-1}}^2  \right) \\
    \text{with} \quad &\| \vx - \vmu \|_{\mSigma^{-1}}^2 = (\vx - \vmu)^\top \mSigma^{-1} (\vx - \vmu) \; ,
\end{align*}
where $d$ is the dimensionality.

In our experiments, we set $\vmu = \mathbf{0}$ and define the covariance $\mSigma$ to be block-diagonal, such that the covariance between weights in different filters is 0 and between weights in the same filter is given by a Matérn kernel ($\nu = 1/2$) on the pixel distances, as applied by \citet{garriga2021correlated} in the infinite-width case.
Formally, for the weights $w_{i,j}$ and $w_{i',j'}$ in filters $i$ and $i'$ and for pixels $j$ and $j'$, the covariance is

\begin{equation}
    \text{cov}(w_{i,j}, w_{i',j'}) =
    \begin{cases}
    \sigma^2 \exp \left( \frac{- d(j, j')}{\lambda} \right) &\text{if} \; i = i' \\
    0 &\text{else}
    \end{cases} \; ,
\end{equation}
where $d(\cdot, \cdot)$ is the Euclidean distance in pixel space and we set $\sigma = \lambda = 1$.




\section{Implementation details}
\label{sec:details}



\paragraph{Training setup.}
For all the MNIST BNN experiments, we perform 60 cycles of SG-MCMC \citep{zhang2019cyclical} with 45 epochs each.
We draw one sample each at the end of the respective last five epochs of each cycle.
From these 300 samples, we discard the first 50 as a burn-in of the chain.
Moreover, in each cycle, we only add Langevin noise in the last 15 epochs (similar to \citet{zhang2019cyclical}).
We start each cycle with a learning rate of $0.01$ and decay to 0 using a cosine schedule.
We use a mini-batch size of 128.

For the SGD experiments yielding the empirical weight distributions, we use the same settings, but do not add any Langevin noise.
We also do not use any cycles and just train the networks once to convergence, which in our case took 600 epochs.

We ran the experiments on GPUs of the type \emph{NVIDIA GeForce GTX 1080 Ti} and \emph{NVIDIA GeForce RTX 2080 Ti} on our local cluster.
The main experiments (see Fig.~\ref{fig:tempering_curves_fcnns} and Fig.~\ref{fig:tempering_curves_cnns}) took around 10,000 GPU hours to run.


\paragraph{FCNN architecture.}

For the FCNN experiments, we used a feedforward neural network with three layers, a hidden layer width of 100, and ReLU activations.

\paragraph{CNN architecture.}

For the CNN experiments, we use a convolutional network with two convolutional layers and one fully-connected layer.
The hidden convolutional layers have 64 channels each and use $3 \times 3$ convolutions and ReLU activations.
Each convolutional layer is followed by a $2 \times 2$ max-pooling layer.

\paragraph{ResNet architecture.}
For the ResNet experiments on CIFAR-10, we use a ResNet20 architecture \citep{he2016deep}, equal to the one used in \citet{wenzel2020good}.

\paragraph{Software packages.} We implemented the inference and models with the PyTorch library \citep{pytorch}. To manage our experiments and schedule runs with several settings, we used Sacred \citep{sacred} and Jug \citep{jug} respectively. For the diagnostics, we also use Arviz \citep{arviz_2019}.

 

\end{document}
ms.bbl
\begin{thebibliography}{92}
\providecommand{\natexlab}[1]{#1}
\providecommand{\url}[1]{\texttt{#1}}
\expandafter\ifx\csname urlstyle\endcsname\relax
  \providecommand{\doi}[1]{doi: #1}\else
  \providecommand{\doi}{doi: \begingroup \urlstyle{rm}\Url}\fi

\bibitem[Aitchison(2020{\natexlab{a}})]{aitchison2020bigger}
Laurence Aitchison.
\newblock Why bigger is not always better: on finite and infinite neural
  networks.
\newblock In \emph{International Conference on Machine Learning}, pages
  156--164. PMLR, 2020{\natexlab{a}}.

\bibitem[Aitchison(2020{\natexlab{b}})]{aitchison2020statistical}
Laurence Aitchison.
\newblock A statistical theory of cold posteriors in deep neural networks.
\newblock \emph{arXiv preprint arXiv:2008.05912}, 2020{\natexlab{b}}.

\bibitem[Aitchison et~al.(2020)Aitchison, Yang, and Ober]{aitchison2020deep}
Laurence Aitchison, Adam~X Yang, and Sebastian~W Ober.
\newblock Deep kernel processes.
\newblock \emph{arXiv preprint arXiv:2010.01590}, 2020.

\bibitem[Atanov et~al.(2018)Atanov, Ashukha, Struminsky, Vetrov, and
  Welling]{atanov2018deep}
Andrei Atanov, Arsenii Ashukha, Kirill Struminsky, Dmitry Vetrov, and Max
  Welling.
\newblock The deep weight prior.
\newblock \emph{arXiv preprint arXiv:1810.06943}, 2018.

\bibitem[Bayes(1763)]{bayes1763lii}
Thomas Bayes.
\newblock An essay towards solving a problem in the doctrine of chances.
\newblock \emph{Philosophical transactions of the Royal Society of London},
  53:\penalty0 370--418, 1763.
\newblock By the late Rev. Mr. Bayes, {FRS} communicated by Mr. Price, in a
  letter to John Canton, {AMFRS}.

\bibitem[Bellido and Fiesler(1993)]{bellido1993backpropagation}
I~Bellido and Emile Fiesler.
\newblock Do backpropagation trained neural networks have normal weight
  distributions?
\newblock In \emph{International Conference on Artificial Neural Networks},
  pages 772--775. Springer, 1993.

\bibitem[Billingsley(1961)]{billingsley1961lindeberg}
Patrick Billingsley.
\newblock The {L}indeberg-{Lé}vy theorem for martingales.
\newblock \emph{Proceedings of the American Mathematical Society}, 12\penalty0
  (5):\penalty0 788--792, 1961.

\bibitem[Bishop(2006)]{bishop2006pattern}
Christopher~M Bishop.
\newblock \emph{Pattern recognition and machine learning}.
\newblock Springer, 2006.

\bibitem[Bishop et~al.(1995)]{bishop1995neural}
Christopher~M Bishop et~al.
\newblock \emph{Neural networks for pattern recognition}.
\newblock Oxford university press, 1995.

\bibitem[Blundell et~al.(2015)Blundell, Cornebise, Kavukcuoglu, and
  Wierstra]{blundell2015weight}
Charles Blundell, Julien Cornebise, Koray Kavukcuoglu, and Daan Wierstra.
\newblock Weight uncertainty in neural networks.
\newblock \emph{arXiv preprint arXiv:1505.05424}, 2015.

\bibitem[Coelho(2017)]{jug}
Luis~Pedro Coelho.
\newblock {J}ug: Software for parallel reproducible computation in {P}ython.
\newblock \emph{Journal of Open Research Software.}, 5:\penalty0 30, 2017.
\newblock \doi{10.5334/jors.161}.

\bibitem[Cui et~al.(2020)Cui, Havulinna, Marttinen, and
  Kaski]{cui2020informative}
Tianyu Cui, A.~Havulinna, P.~Marttinen, and S.~Kaski.
\newblock Informative {G}aussian scale mixture priors for {B}ayesian neural
  networks.
\newblock \emph{arXiv preprint arXiv:2002.10243}, 2020.

\bibitem[Dusenberry et~al.(2020)Dusenberry, Jerfel, Wen, Ma, Snoek, Heller,
  Lakshminarayanan, and Tran]{dusenberry2020efficient}
Michael~W Dusenberry, Ghassen Jerfel, Yeming Wen, Yi-an Ma, Jasper Snoek,
  Katherine Heller, Balaji Lakshminarayanan, and Dustin Tran.
\newblock Efficient and scalable {B}ayesian neural nets with rank-1 factors.
\newblock \emph{arXiv preprint arXiv:2005.07186}, 2020.

\bibitem[Dziugaite et~al.(2021)Dziugaite, Hsu, Gharbieh, Arpino, and
  Roy]{dziugaite2021role}
Gintare~Karolina Dziugaite, Kyle Hsu, Waseem Gharbieh, Gabriel Arpino, and
  Daniel Roy.
\newblock On the role of data in pac-bayes.
\newblock In \emph{International Conference on Artificial Intelligence and
  Statistics}, pages 604--612. PMLR, 2021.

\bibitem[Farquhar et~al.(2019)Farquhar, Osborne, and Gal]{farquhar2019radial}
Sebastian Farquhar, Michael Osborne, and Yarin Gal.
\newblock Radial {B}ayesian neural networks: Robust variational inference in
  big models.
\newblock \emph{arXiv preprint arXiv:1907.00865}, 2019.

\bibitem[Foong et~al.(2019)Foong, Burt, Li, and
  Turner]{foong2019expressiveness}
Andrew~YK Foong, David~R Burt, Yingzhen Li, and Richard~E Turner.
\newblock On the expressiveness of approximate inference in {B}ayesian neural
  networks.
\newblock \emph{arXiv preprint arXiv:1909.00719}, 2019.

\bibitem[Fortuin(2021)]{fortuin2021priors}
Vincent Fortuin.
\newblock Priors in bayesian deep learning: A review.
\newblock \emph{arXiv preprint arXiv:2105.06868}, 2021.

\bibitem[Gal and Ghahramani(2016)]{gal2016dropout}
Yarin Gal and Zoubin Ghahramani.
\newblock Dropout as a {B}ayesian approximation: Representing model uncertainty
  in deep learning.
\newblock In \emph{International Conference on Machine Learning}, pages
  1050--1059. PMLR, 2016.

\bibitem[Garriga-Alonso and Fortuin(2021)]{guided-mcmc}
Adri{\`a} Garriga-Alonso and Vincent Fortuin.
\newblock Exact {L}angevin dynamics with stochastic gradients.
\newblock \emph{arXiv preprint arXiv:2102.01691}, 2021.

\bibitem[Garriga-Alonso and van~der Wilk(2021)]{garriga2021correlated}
Adri{\`a} Garriga-Alonso and Mark van~der Wilk.
\newblock Correlated weights in infinite limits of deep convolutional neural
  networks.
\newblock \emph{arXiv preprint arXiv:2101.04097}, 2021.

\bibitem[Garriga{-}Alonso et~al.(2019)Garriga{-}Alonso, Rasmussen, and
  Aitchison]{garriga2018deep}
Adri{\`{a}} Garriga{-}Alonso, Carl~Edward Rasmussen, and Laurence Aitchison.
\newblock Deep convolutional networks as shallow {G}aussian processes.
\newblock In \emph{7th International Conference on Learning Representations},
  2019.
\newblock URL \url{https://openreview.net/forum?id=Bklfsi0cKm}.

\bibitem[Gauss(1809)]{gauss1809theoria}
Carl~Friedrich Gauss.
\newblock \emph{Theoria motvs corporvm coelestivm in sectionibvs conicis solem
  ambientivm}.
\newblock Sumtibus F. Perthes et IH Besser, 1809.

\bibitem[Gelman et~al.(2013)Gelman, Carlin, Stern, Dunson, Vehtari, and
  Rubin]{gelman2013bayesian}
Andrew Gelman, John~B Carlin, Hal~S Stern, David~B Dunson, Aki Vehtari, and
  Donald~B Rubin.
\newblock \emph{{B}ayesian data analysis}.
\newblock CRC press, 2013.

\bibitem[Germain et~al.(2016)Germain, Bach, Lacoste, and
  Lacoste-Julien]{germain2016pac}
Pascal Germain, Francis Bach, Alexandre Lacoste, and Simon Lacoste-Julien.
\newblock {PAC}-{B}ayesian theory meets {B}ayesian inference.
\newblock \emph{arXiv preprint arXiv:1605.08636}, 2016.

\bibitem[Ghosh and Doshi-Velez(2017)]{ghosh2017model}
Soumya Ghosh and Finale Doshi-Velez.
\newblock Model selection in {B}ayesian neural networks via horseshoe priors.
\newblock \emph{arXiv preprint arXiv:1705.10388}, 2017.

\bibitem[Gneiting and Raftery(2007)]{gneiting2007strictly}
Tilmann Gneiting and Adrian~E Raftery.
\newblock Strictly proper scoring rules, prediction, and estimation.
\newblock \emph{Journal of the American Statistical Association}, 102\penalty0
  (477):\penalty0 359--378, 2007.

\bibitem[Go and Lee(1999)]{go1999analyzing}
Jinwook Go and Chulhee Lee.
\newblock Analyzing weight distribution of neural networks.
\newblock In \emph{IJCNN'99. International Joint Conference on Neural Networks.
  Proceedings (Cat. No. 99CH36339)}, volume~2, pages 1154--1157. IEEE, 1999.

\bibitem[Greff et~al.(2017)Greff, Klein, Chovanec, Hutter, and
  Schmidhuber]{sacred}
Klaus Greff, Aaron Klein, Martin Chovanec, Frank Hutter, and J\"urgen
  Schmidhuber.
\newblock {T}he {S}acred {I}nfrastructure for {C}omputational {R}esearch.
\newblock In {K}aty {H}uff, {D}avid {L}ippa, {D}illon {N}iederhut, and {M}
  {P}acer, editors, \emph{{P}roceedings of the 16th {P}ython in {S}cience
  {C}onference}, pages 49 -- 56, 2017.
\newblock \doi{10.25080/shinma-7f4c6e7-008}.

\bibitem[Gurbuzbalaban and Simsekli(2020)]{gurbuzbalaban2020heavy}
Mert Gurbuzbalaban and Lingjiong Simsekli, Umut an~Zhu.
\newblock The heavy-tail phenomenon in {SGD}.
\newblock \emph{arXiv preprint arXiv:2006.04740}, 2020.

\bibitem[Hafner et~al.(2020)Hafner, Tran, Lillicrap, Irpan, and
  Davidson]{hafner2020noise}
Danijar Hafner, Dustin Tran, Timothy Lillicrap, Alex Irpan, and James Davidson.
\newblock Noise contrastive priors for functional uncertainty.
\newblock In \emph{Uncertainty in Artificial Intelligence}, pages 905--914.
  PMLR, 2020.

\bibitem[He et~al.(2016)He, Zhang, Ren, and Sun]{he2016deep}
Kaiming He, Xiangyu Zhang, Shaoqing Ren, and Jian Sun.
\newblock Deep residual learning for image recognition.
\newblock In \emph{Proceedings of the IEEE conference on computer vision and
  pattern recognition}, pages 770--778, 2016.

\bibitem[Heek and Kalchbrenner(2019)]{hee2020}
Jonathan Heek and Nal Kalchbrenner.
\newblock {B}ayesian inference for large scale image classification.
\newblock \emph{arXiv preprint arXiv:1908.03491}, 2019.

\bibitem[Helmert(1875)]{helmert1875berechnung}
Friedrich~Robert Helmert.
\newblock {\"U}ber die {B}erechnung des wahrscheinlichen {F}ehlers aus einer
  endlichen {A}nzahl wahrer {B}eobachtungsfehler.
\newblock \emph{Z. Math. U. Physik}, 20\penalty0 (1875):\penalty0 300--303,
  1875.

\bibitem[Hern{\'a}ndez-Lobato and Adams(2015)]{hernandez2015probabilistic}
Jos{\'e}~Miguel Hern{\'a}ndez-Lobato and Ryan Adams.
\newblock Probabilistic backpropagation for scalable learning of bayesian
  neural networks.
\newblock In \emph{International Conference on Machine Learning}, pages
  1861--1869. PMLR, 2015.

\bibitem[Immer et~al.(2021{\natexlab{a}})Immer, Bauer, Fortuin, R{\"a}tsch, and
  Khan]{immer2021scalable}
Alexander Immer, Matthias Bauer, Vincent Fortuin, Gunnar R{\"a}tsch, and
  Mohammad~Emtiyaz Khan.
\newblock Scalable marginal likelihood estimation for model selection in deep
  learning.
\newblock \emph{arXiv preprint arXiv:2104.04975}, 2021{\natexlab{a}}.

\bibitem[Immer et~al.(2021{\natexlab{b}})Immer, Korzepa, and
  Bauer]{immer2021improving}
Alexander Immer, Maciej Korzepa, and Matthias Bauer.
\newblock Improving predictions of bayesian neural nets via local
  linearization.
\newblock In \emph{International Conference on Artificial Intelligence and
  Statistics}, pages 703--711. PMLR, 2021{\natexlab{b}}.

\bibitem[Izmailov et~al.(2020)Izmailov, Maddox, Kirichenko, Garipov, Vetrov,
  and Wilson]{izmailov2020subspace}
Pavel Izmailov, Wesley~J Maddox, Polina Kirichenko, Timur Garipov, Dmitry
  Vetrov, and Andrew~Gordon Wilson.
\newblock Subspace inference for {B}ayesian deep learning.
\newblock In \emph{Uncertainty in Artificial Intelligence}, pages 1169--1179.
  PMLR, 2020.

\bibitem[Izmailov et~al.(2021)Izmailov, Vikram, Hoffman, and
  Wilson]{izmailov2021hmc}
Pavel Izmailov, Sharad Vikram, Matthew~D Hoffman, and Andrew~Gordon Wilson.
\newblock What are {B}ayesian neural network posteriors really like?
\newblock \emph{arXiv preprint arXiv:2104.14421}, 2021.

\bibitem[Karaletsos and Bui(2020)]{karaletsos2020hierarchical}
Theofanis Karaletsos and Thang~D Bui.
\newblock Hierarchical {G}aussian process priors for {B}ayesian neural network
  weights.
\newblock \emph{arXiv preprint arXiv:2002.04033}, 2020.

\bibitem[Kendall and Gal(2017)]{kendall2017uncertainties}
Alex Kendall and Yarin Gal.
\newblock What uncertainties do we need in {B}ayesian deep learning for
  computer vision?
\newblock In \emph{Advances in neural information processing systems}, pages
  5574--5584, 2017.

\bibitem[Kingma et~al.(2015)Kingma, Salimans, and
  Welling]{kingma2015variational}
Diederik~P Kingma, Tim Salimans, and Max Welling.
\newblock Variational dropout and the local reparameterization trick.
\newblock In \emph{Proceedings of the 28th International Conference on Neural
  Information Processing Systems-Volume 2}, pages 2575--2583, 2015.

\bibitem[Krizhevsky(2009)]{krizhevsky2009learning}
Alex Krizhevsky.
\newblock Learning multiple layers of features from tiny images.
\newblock Technical report, University of Toronto, 2009.

\bibitem[Kumar et~al.(2019)Kumar, Carroll, Hartikainen, and Martin]{arviz_2019}
Ravin Kumar, Colin Carroll, Ari Hartikainen, and Osvaldo Martin.
\newblock Arviz a unified library for exploratory analysis of bayesian models
  in python.
\newblock \emph{Journal of Open Source Software}, 4\penalty0 (33):\penalty0
  1143, 2019.
\newblock \doi{10.21105/joss.01143}.
\newblock URL \url{https://doi.org/10.21105/joss.01143}.

\bibitem[Laplace(1774)]{laplace1774memoire}
Pierre~Simon Laplace.
\newblock {M}{\'e}moire sur la probabilit{\'e} de causes par les
  {\'e}venements.
\newblock \emph{Memoire de l'Academie Royale des Sciences}, 1774.

\bibitem[LeCun et~al.(1998)LeCun, Bottou, Bengio, and
  Haffner]{lecun1998gradient}
Yann LeCun, L{\'e}on Bottou, Yoshua Bengio, and Patrick Haffner.
\newblock Gradient-based learning applied to document recognition.
\newblock \emph{Proceedings of the IEEE}, 86\penalty0 (11):\penalty0
  2278--2324, 1998.

\bibitem[Lee et~al.(2017)Lee, Bahri, Novak, Schoenholz, Pennington, and
  Sohl-Dickstein]{lee2017deep}
Jaehoon Lee, Yasaman Bahri, Roman Novak, Samuel~S Schoenholz, Jeffrey
  Pennington, and Jascha Sohl-Dickstein.
\newblock Deep neural networks as {G}aussian processes.
\newblock \emph{arXiv preprint arXiv:1711.00165}, 2017.

\bibitem[Leimkuhler and Matthews(2012)]{lm-aboba}
Benedict Leimkuhler and Charles Matthews.
\newblock {Rational Construction of Stochastic Numerical Methods for Molecular
  Sampling}.
\newblock \emph{Applied Mathematics Research eXpress}, 2013\penalty0
  (1):\penalty0 34--56, 06 2012.
\newblock ISSN 1687-1200.
\newblock \doi{10.1093/amrx/abs010}.

\bibitem[Louizos and Welling(2017)]{louizos2017multiplicative}
Christos Louizos and Max Welling.
\newblock Multiplicative normalizing flows for variational {B}ayesian neural
  networks.
\newblock \emph{arXiv preprint arXiv:1703.01961}, 2017.

\bibitem[L{\"u}roth(1876)]{luroth1876vergelichung}
J~L{\"u}roth.
\newblock {V}ergleichung von zwei {W}erthen des wahrscheinlichen {F}ehlers.
\newblock \emph{Astronomische Nachrichten}, 87:\penalty0 209, 1876.

\bibitem[Ma et~al.(2019)Ma, Li, and Hern{\'a}ndez-Lobato]{ma2019variational}
Chao Ma, Yingzhen Li, and Jos{\'e}~Miguel Hern{\'a}ndez-Lobato.
\newblock Variational implicit processes.
\newblock In \emph{International Conference on Machine Learning}, pages
  4222--4233, 2019.

\bibitem[MacKay(1992)]{mackay1992practical}
David~J.C. MacKay.
\newblock A practical {B}ayesian framework for backpropagation networks.
\newblock \emph{Neural computation}, 4\penalty0 (3):\penalty0 448--472, 1992.

\bibitem[Matsubara et~al.(2020)Matsubara, Oates, and
  Briol]{matsubara2020ridgelet}
Takuo Matsubara, Chris~J Oates, and Fran{\c{c}}ois-Xavier Briol.
\newblock The ridgelet prior: A covariance function approach to prior
  specification for {B}ayesian neural networks.
\newblock \emph{arXiv preprint arXiv:2010.08488}, 2020.

\bibitem[Matthews et~al.(2018)Matthews, Rowland, Hron, Turner, and
  Ghahramani]{matthews2018gaussian}
Alexander G de~G Matthews, Mark Rowland, Jiri Hron, Richard~E Turner, and
  Zoubin Ghahramani.
\newblock {G}aussian process behaviour in wide deep neural networks.
\newblock \emph{arXiv preprint arXiv:1804.11271}, 2018.

\bibitem[Molchanov et~al.(2017)Molchanov, Ashukha, and
  Vetrov]{molchanov2017variational}
Dmitry Molchanov, Arsenii Ashukha, and Dmitry Vetrov.
\newblock Variational dropout sparsifies deep neural networks.
\newblock In \emph{International Conference on Machine Learning}, pages
  2498--2507. PMLR, 2017.

\bibitem[Naeini et~al.(2015)Naeini, Cooper, and
  Hauskrecht]{naeini2015obtaining}
Mahdi~Pakdaman Naeini, Gregory~F Cooper, and Milos Hauskrecht.
\newblock Obtaining well calibrated probabilities using {B}ayesian binning.
\newblock In \emph{Proceedings of the... AAAI Conference on Artificial
  Intelligence. AAAI Conference on Artificial Intelligence}, volume 2015, page
  2901. NIH Public Access, 2015.

\bibitem[Nalisnick(2018)]{nalisnick2018priors}
Eric~T Nalisnick.
\newblock \emph{On priors for {B}ayesian neural networks}.
\newblock PhD thesis, UC Irvine, 2018.

\bibitem[Nalisnick et~al.(2015)Nalisnick, Anandkumar, and
  Smyth]{nalisnick2015scalemixture}
Eric~T Nalisnick, Anima Anandkumar, and Padhraic Smyth.
\newblock A scale mixture perspective of multiplicative noise in neural
  networks.
\newblock \emph{arXiv preprint arXiv:1506.03208}, 2015.

\bibitem[Neal(1992)]{neal1992bayesian}
Radford~M. Neal.
\newblock {B}ayesian training of backpropagation networks by the {H}ybrid
  {M}onte {C}arlo method.
\newblock Technical report, University of Toronto, 1992.

\bibitem[Neal(1996)]{neal1996bayesian}
Radford~M. Neal.
\newblock \emph{{B}ayesian learning for neural networks}, volume 118.
\newblock Springer, 1996.

\bibitem[Novak et~al.(2019)Novak, Xiao, Bahri, Lee, Yang, Abolafia, Pennington,
  and Sohl-dickstein]{novak2019bayesian}
Roman Novak, Lechao Xiao, Yasaman Bahri, Jaehoon Lee, Greg Yang, Daniel~A.
  Abolafia, Jeffrey Pennington, and Jascha Sohl-dickstein.
\newblock {B}ayesian deep convolutional networks with many channels are
  {G}aussian processes.
\newblock In \emph{International Conference on Learning Representations}, 2019.
\newblock URL \url{https://openreview.net/forum?id=B1g30j0qF7}.

\bibitem[Ober and Aitchison(2020)]{ober2020global}
Sebastian~W Ober and Laurence Aitchison.
\newblock Global inducing point variational posteriors for {B}ayesian neural
  networks and deep {G}aussian processes.
\newblock \emph{arXiv preprint arXiv:2005.08140}, 2020.

\bibitem[Ober et~al.(2021)Ober, Rasmussen, and van~der Wilk]{ober2021promises}
Sebastian~W Ober, Carl~E Rasmussen, and Mark van~der Wilk.
\newblock The promises and pitfalls of deep kernel learning.
\newblock \emph{arXiv preprint arXiv:2102.12108}, 2021.

\bibitem[Olshausen and Field(1997)]{olshausen1997sparse}
Bruno~A Olshausen and David~J Field.
\newblock Sparse coding with an overcomplete basis set: A strategy employed by
  v1?
\newblock \emph{Vision research}, 37\penalty0 (23):\penalty0 3311--3325, 1997.

\bibitem[Osawa et~al.(2019)Osawa, Swaroop, Khan, Jain, Eschenhagen, Turner, and
  Yokota]{osawa2019practical}
Kazuki Osawa, Siddharth Swaroop, Mohammad Emtiyaz~E Khan, Anirudh Jain, Runa
  Eschenhagen, Richard~E Turner, and Rio Yokota.
\newblock Practical deep learning with {B}ayesian principles.
\newblock In \emph{Advances in neural information processing systems}, pages
  4287--4299, 2019.

\bibitem[Ovadia et~al.(2019)Ovadia, Fertig, Ren, Nado, Sculley, Nowozin,
  Dillon, Lakshminarayanan, and Snoek]{ovadia2019can}
Yaniv Ovadia, Emily Fertig, Jie Ren, Zachary Nado, David Sculley, Sebastian
  Nowozin, Joshua Dillon, Balaji Lakshminarayanan, and Jasper Snoek.
\newblock Can you trust your model's uncertainty? {E}valuating predictive
  uncertainty under dataset shift.
\newblock In \emph{Advances in Neural Information Processing Systems}, pages
  13991--14002, 2019.

\bibitem[Overweg et~al.(2019)Overweg, Popkes, Ercole, Li, Hern{\'a}ndez-Lobato,
  Zaykov, and Zhang]{overweg2019interpretable}
Hiske Overweg, Anna-Lena Popkes, Ari Ercole, Yingzhen Li, Jos{\'e}~Miguel
  Hern{\'a}ndez-Lobato, Yordan Zaykov, and Cheng Zhang.
\newblock Interpretable outcome prediction with sparse {B}ayesian neural
  networks in intensive care.
\newblock \emph{arXiv preprint arXiv:1905.02599}, 2019.

\bibitem[Paszke et~al.(2019)Paszke, Gross, Massa, Lerer, Bradbury, Chanan,
  Killeen, Lin, Gimelshein, Antiga, Desmaison, Kopf, Yang, DeVito, Raison,
  Tejani, Chilamkurthy, Steiner, Fang, Bai, and Chintala]{pytorch}
Adam Paszke, Sam Gross, Francisco Massa, Adam Lerer, James Bradbury, Gregory
  Chanan, Trevor Killeen, Zeming Lin, Natalia Gimelshein, Luca Antiga, Alban
  Desmaison, Andreas Kopf, Edward Yang, Zachary DeVito, Martin Raison, Alykhan
  Tejani, Sasank Chilamkurthy, Benoit Steiner, Lu~Fang, Junjie Bai, and Soumith
  Chintala.
\newblock {P}y{T}orch: An imperative style, high-performance deep learning
  library.
\newblock In \emph{Advances in Neural Information Processing Systems 32}, pages
  8024--8035. 2019.

\bibitem[Pearce et~al.(2020)Pearce, Tsuchida, Zaki, Brintrup, and
  Neely]{pearce2020expressive}
Tim Pearce, Russell Tsuchida, Mohamed Zaki, Alexandra Brintrup, and Andy Neely.
\newblock Expressive priors in {B}ayesian neural networks: Kernel combinations
  and periodic functions.
\newblock In \emph{Uncertainty in Artificial Intelligence}, pages 134--144.
  PMLR, 2020.

\bibitem[Peluchetti et~al.(2020)Peluchetti, Favaro, and
  Fortini]{peluchetti2020stable}
Stefano Peluchetti, Stefano Favaro, and Sandra Fortini.
\newblock Stable behaviour of infinitely wide deep neural networks.
\newblock In \emph{International Conference on Artificial Intelligence and
  Statistics}, pages 1137--1146. PMLR, 2020.

\bibitem[Rivasplata et~al.(2020)Rivasplata, Kuzborskij, Szepesv{\'a}ri, and
  Shawe-Taylor]{rivasplata2020pac}
Omar Rivasplata, Ilja Kuzborskij, Csaba Szepesv{\'a}ri, and John Shawe-Taylor.
\newblock Pac-bayes analysis beyond the usual bounds.
\newblock \emph{arXiv preprint arXiv:2006.13057}, 2020.

\bibitem[Robbins(1992)]{robbins1992empirical}
Herbert~E Robbins.
\newblock An empirical {B}ayes approach to statistics.
\newblock In \emph{Breakthroughs in statistics}, pages 388--394. Springer,
  1992.

\bibitem[Rothfuss et~al.(2020)Rothfuss, Fortuin, and Krause]{rothfuss2020pacoh}
Jonas Rothfuss, Vincent Fortuin, and Andreas Krause.
\newblock {PACOH}: Bayes-optimal meta-learning with {PAC}-guarantees.
\newblock \emph{arXiv preprint arXiv:2002.05551}, 2020.

\bibitem[Sahani and Linden(2003)]{sahani2003evidence}
Maneesh Sahani and Jennifer~F Linden.
\newblock Evidence optimization techniques for estimating stimulus-response
  functions.
\newblock \emph{Advances in neural information processing systems}, pages
  317--324, 2003.

\bibitem[Simoncelli(2009)]{simoncelli2009capturing}
Eero~P Simoncelli.
\newblock Capturing visual image properties with probabilistic models.
\newblock In \emph{The Essential Guide to Image Processing}, pages 205--223.
  Elsevier, 2009.

\bibitem[Smyth et~al.(2003)Smyth, Willmore, Baker, Thompson, and
  Tolhurst]{smyth2003receptive}
Darragh Smyth, Ben Willmore, Gary~E Baker, Ian~D Thompson, and David~J
  Tolhurst.
\newblock The receptive-field organization of simple cells in primary visual
  cortex of ferrets under natural scene stimulation.
\newblock \emph{Journal of Neuroscience}, 23\penalty0 (11):\penalty0
  4746--4759, 2003.

\bibitem[Srivastava et~al.(2003)Srivastava, Lee, Simoncelli, and
  Zhu]{srivastava2003advances}
Anuj Srivastava, Ann~B Lee, Eero~P Simoncelli, and S-C Zhu.
\newblock On advances in statistical modeling of natural images.
\newblock \emph{Journal of mathematical imaging and vision}, 18\penalty0
  (1):\penalty0 17--33, 2003.

\bibitem[Student(1908)]{student1908probable}
Student.
\newblock The probable error of a mean.
\newblock \emph{Biometrika}, pages 1--25, 1908.

\bibitem[Sun et~al.(2019)Sun, Zhang, Shi, and Grosse]{sun2019functional}
Shengyang Sun, Guodong Zhang, Jiaxin Shi, and Roger Grosse.
\newblock Functional variational {B}ayesian neural networks.
\newblock \emph{arXiv preprint arXiv:1903.05779}, 2019.

\bibitem[Sussmann(1992)]{sussmann1992uniqueness}
H{\'e}ctor~J Sussmann.
\newblock Uniqueness of the weights for minimal feedforward nets with a given
  input-output map.
\newblock \emph{Neural networks}, 5\penalty0 (4):\penalty0 589--593, 1992.

\bibitem[Swiatkowski et~al.(2020)Swiatkowski, Roth, Veeling, Tran, Dillon,
  Mandt, Snoek, Salimans, Jenatton, and Nowozin]{swiatkowski2020k}
Jakub Swiatkowski, Kevin Roth, Bastiaan~S Veeling, Linh Tran, Joshua~V Dillon,
  Stephan Mandt, Jasper Snoek, Tim Salimans, Rodolphe Jenatton, and Sebastian
  Nowozin.
\newblock The k-tied normal distribution: A compact parameterization of
  {G}aussian mean field posteriors in {B}ayesian neural networks.
\newblock \emph{arXiv preprint arXiv:2002.02655}, 2020.

\bibitem[Tibshirani(1996)]{tibshirani1996regression}
Robert Tibshirani.
\newblock Regression shrinkage and selection via the lasso.
\newblock \emph{Journal of the Royal Statistical Society: Series B
  (Methodological)}, 58\penalty0 (1):\penalty0 267--288, 1996.

\bibitem[Tran et~al.(2020)Tran, Rossi, Milios, and Filippone]{tran2020all}
Ba-Hien Tran, Simone Rossi, Dimitrios Milios, and Maurizio Filippone.
\newblock All you need is a good functional prior for {B}ayesian deep learning.
\newblock \emph{arXiv preprint arXiv:2011.12829}, 2020.

\bibitem[Tsuchida et~al.(2019)Tsuchida, Roosta, and
  Gallagher]{tsuchida2019richer}
Russell Tsuchida, Fred Roosta, and Marcus Gallagher.
\newblock Richer priors for infinitely wide multi-layer perceptrons.
\newblock \emph{arXiv preprint arXiv:1911.12927}, 2019.

\bibitem[van~der Wilk et~al.(2018)van~der Wilk, Bauer, John, and
  Hensman]{vdw2018inv}
Mark van~der Wilk, Matthias Bauer, ST~John, and James Hensman.
\newblock Learning invariances using the marginal likelihood.
\newblock In \emph{Advances in Neural Information Processing Systems},
  volume~31, pages 9938--9948, 2018.
\newblock URL
  \url{https://proceedings.neurips.cc/paper/2018/file/d465f14a648b3d0a1faa6f447e526c60-Paper.pdf}.

\bibitem[Vehtari et~al.(2021)Vehtari, Gelman, Simpson, Carpenter, and
  B{\"u}rkner]{vehtari2021rank}
Aki Vehtari, Andrew Gelman, Daniel Simpson, Bob Carpenter, and Paul-Christian
  B{\"u}rkner.
\newblock Rank-normalization, folding, and localization: an improved
  $\widehat{R}$ for assessing convergence of {MCMC}.
\newblock \emph{Bayesian analysis}, 1\penalty0 (1):\penalty0 1--28, 2021.

\bibitem[Wenzel et~al.(2020{\natexlab{a}})Wenzel, Roth, Veeling,
  {\'S}wi{\,{a}}tkowski, Tran, Mandt, Snoek, Salimans, Jenatton, and
  Nowozin]{wenzel2020good}
Florian Wenzel, Kevin Roth, Bastiaan~S Veeling, Jakub {\'S}wi{\,{a}}tkowski,
  Linh Tran, Stephan Mandt, Jasper Snoek, Tim Salimans, Rodolphe Jenatton, and
  Sebastian Nowozin.
\newblock How good is the {B}ayes posterior in deep neural networks really?
\newblock In \emph{International Conference on Machine Learning},
  2020{\natexlab{a}}.

\bibitem[Wenzel et~al.(2020{\natexlab{b}})Wenzel, Snoek, Tran, and
  Jenatton]{wenzel2020hyper}
Florian Wenzel, Jasper Snoek, Dustin Tran, and Rodolphe Jenatton.
\newblock Hyperparameter ensembles for robustness and uncertainty
  quantification.
\newblock In \emph{Advances in Neural Information Processing Systems},
  2020{\natexlab{b}}.

\bibitem[Wu et~al.(2018)Wu, Nowozin, Meeds, Turner, Hern{\'a}ndez-Lobato, and
  Gaunt]{wu2018deterministic}
Anqi Wu, Sebastian Nowozin, Edward Meeds, Richard~E Turner, Jos{\'e}~Miguel
  Hern{\'a}ndez-Lobato, and Alexander~L Gaunt.
\newblock Deterministic variational inference for robust {B}ayesian neural
  networks.
\newblock \emph{arXiv preprint arXiv:1810.03958}, 2018.

\bibitem[Xiao et~al.(2017)Xiao, Rasul, and Vollgraf]{xiao2017fashion}
Han Xiao, Kashif Rasul, and Roland Vollgraf.
\newblock Fashion-{MNIST}: a novel image dataset for benchmarking machine
  learning algorithms.
\newblock \emph{arXiv preprint arXiv:1708.07747}, 2017.

\bibitem[Yang(2019)]{yang2019wide}
Greg Yang.
\newblock Wide feedforward or recurrent neural networks of any architecture are
  {G}aussian processes.
\newblock In \emph{Advances in Neural Information Processing Systems}, pages
  9951--9960, 2019.

\bibitem[Yang et~al.(2015)Yang, Chen, and Tempelman]{yang2015improving}
Wenzhao Yang, Chunyu Chen, and Robert~J Tempelman.
\newblock Improving the computational efficiency of fully bayes inference and
  assessing the effect of misspecification of hyperparameters in whole-genome
  prediction models.
\newblock \emph{Genetics Selection Evolution}, 47\penalty0 (1):\penalty0 1--14,
  2015.

\bibitem[Zhang et~al.(2019)Zhang, Li, Zhang, Chen, and
  Wilson]{zhang2019cyclical}
Ruqi Zhang, Chunyuan Li, Jianyi Zhang, Changyou Chen, and Andrew~Gordon Wilson.
\newblock Cyclical stochastic gradient {MCMC} for {B}ayesian deep learning.
\newblock \emph{arXiv preprint arXiv:1902.03932}, 2019.

\end{thebibliography}
