How close are we to understanding image-based saliency?
Matthias Kümmerer
Thomas Wallis
Matthias Bethge
data.tex
\newcommand{\MutualInformationImageSpatial}{2.1}
\newcommand{\BaselineToBestPercentOfGold}{19}
\newcommand{\BestModelPerformance}{1.29}
\newcommand{\BestModelPercentOfGoldRelBaseline}{34}
\newcommand{\BestModelCenterbiasToBlurPercent}{0}
\newcommand{\BaselineSelfExcitationPerformance}{1.30}
\newcommand{\BestModel}{eDN}
\newcommand{\PossibleInformationGain}{1.20}
\newcommand{\BestModelNonlinearityToCenterbiasPercent}{4}
\newcommand{\BaselinePerformance}{0.89}
\newcommand{\BestModelPercentOfGold}{62}
\newcommand{\BaselinePercentOfGold}{42}
\newcommand{\BestModelNonlinearityPercent}{96}

data_kienzle.tex
\newcommand{\KienzleBestModel}{GBVS}
\newcommand{\KienzleBestModelCenterbiasToBlurPercent}{-0}
\newcommand{\KienzleBaselineToBestPercentOfGold}{15}
\newcommand{\KienzleBestModelPercentOfGoldRelBaseline}{22}
\newcommand{\KienzleBestModelNonlinearityToCenterbiasPercent}{8}
\newcommand{\KienzleBaselinePerformance}{0.48}
\newcommand{\KienzleBestModelPerformance}{0.68}
\newcommand{\KienzleBaselinePercentOfGold}{34}
\newcommand{\KienzleBestModelPercentOfGold}{49}
\newcommand{\KienzlePossibleInformationGain}{0.92}
\newcommand{\KienzleMutualInformationImageSpatial}{1.4}
\newcommand{\KienzleBestModelNonlinearityPercent}{92}
\newcommand{\KienzleBaselineSelfExcitationPerformance}{1.16}

intro_figure.pgf.tex
\ifdefined\noktikz
%  \input{figures/auc_examples}
\else
  \usetikzlibrary{calc}
  \usetikzlibrary{shapes}
  \graphicspath{{/kyb/agmb/mkuemmerer/Documents/Uni/Bethge/Saliency/TPAMI/figures/}}

%  \input{/kyb/agmb/mkuemmerer/Documents/Uni/Bethge/Saliency/TPAMI/figures/auc_examples}
\fi

\ifdefined\abstikzunit
\else
    \newlength\abstikzunit
    \newlength\tikzunit
\fi

\setlength\abstikzunit{1cm}
\newcommand\scalingfactor{0.6}
\newcommand\imgwidth{5.0}
\newcommand\fullimgwidth{11}
\newcommand\labelalpha{0.4}
\newcommand\labelheight{1.5}
\setlength\tikzunit{\scalingfactor\abstikzunit}


\newcommand\loglik[1]{\textsf{$\mathsf{#1}$\,bit/fix}}
\newcommand\auc[1]{\textsf{$\mathsf{#1}$\,\%}}
%\newcommand\imgwidth{5cm}
%\newcommand\imgwidth{4cm}

\begin{tikzpicture}[scale=\scalingfactor]

  \coordinate (col2) at (5.9, 0);
  \coordinate (row2) at (0, -4.2);
  \coordinate (label_rel_subfig) at (-0.6, 0);

%  \node (labela) at (label_rel_fig) {\textsf{a)}};

  \node[anchor=north west] (fix_high)  at (0,0)
    {\includegraphics[width=\imgwidth\tikzunit]{orig_image_with_fixations_and_fixation_train.pdf}};

  \node[anchor=north west] at (label_rel_subfig) {\textsf{a)}};


  \node[anchor=north west] (fix_high)  at (col2)
    {\includegraphics[width=\imgwidth\tikzunit]{saliencymap.png}};

  \node[anchor=north west] at ($ (col2) + (label_rel_subfig) $) {\textsf{b)}};


  \node[anchor=north west] (fix_high)  at (row2)
    {\includegraphics[width=\fullimgwidth\tikzunit]{saliency_as_pointprocess.pdf}};

  \node[anchor=north west] at ($ (row2) + (label_rel_subfig) $) {\textsf{c)}};



\end{tikzpicture}

% Clean up
\let\abstikzunit\undef
\let\scalingfactor\undef
\let\imgwidth\undef
\let\tikzunit\undef

modeling_stationary.pgf.tex
\ifdefined\noktikz
  \input{figures/model_colors}
\else
  \usetikzlibrary{calc}
  \usetikzlibrary{shapes}
  \graphicspath{{/kyb/agmb/mkuemmerer/Documents/Uni/Bethge/Saliency/TPAMI/figures/}}

  \input{/kyb/agmb/mkuemmerer/Documents/Uni/Bethge/Saliency/TPAMI/figures/model_colors}
\fi


\newcommand\loglik[1]{\textsf{$\mathsf{#1}$\,bit/fix}}
\newcommand\auc[1]{\textsf{$\mathsf{#1}$\,\%}}
%\newcommand\imgwidth{5cm}
%\newlength\imgwidth
%\setlength\imgwidth{5cm}
%\newlength\abstikzunit
%\setlength\abstikzunit{1cm}

%\newcommand\scalingfactor{0.8}
%\newlength\tikzunit
%\setlength\tikzunit{\scalingfactor\abstikzunit}

\ifdefined\abstikzunit
	\setlength\abstikzunit{1cm}
	\newcommand\scalingfactor{0.8}
	\newcommand\imgwidth{5}
	\setlength\tikzunit{\scalingfactor\abstikzunit}
\else
	\newlength\abstikzunit
	\setlength\abstikzunit{1cm}
	\newlength\tikzunit
	\newcommand\scalingfactor{0.8}
	\newcommand\imgwidth{5}
	\setlength\tikzunit{\scalingfactor\abstikzunit}
\fi


\begin{tikzpicture}[scale=\scalingfactor,font=\sffamily]
  \definecolor{fillcolor}{rgb}{0.9, 0.9, 0.9};
  \tikzset{boxstyle/.style={fill=fillcolor, rounded corners=2mm}};

  % Data left column

  \coordinate (leftcolwidth) at (6.7,0);
  \coordinate (leftcolheight) at (0,-2);
  \newcommand{\leftcolimgheight}{1.2}

  \coordinate (leftcolsize) at ($ (leftcolwidth) + (leftcolheight) $);

  \coordinate (leftcolsep) at (0, -1);
  \coordinate (leftcolop) at ($ (leftcolheight) + 0.5*(leftcolsep) + 0.5*(leftcolwidth) $);

  \coordinate (lefttotalcolheight) at ($ (leftcolheight) + (leftcolsep) $);

  \coordinate (leftcolimage) at (0.3,-0.4);
  \coordinate (leftcoltext) at ($ 0.5*(leftcolheight) + (2,0) $);
  \coordinate (leftcolimagestack) at (+0.15,-0.15);


  \coordinate (leftcoltemporalsep) at (0,-0.2);

  \coordinate (col2) at (5.9, 0);
  \coordinate (row2) at (0, -4.2);
  \coordinate (label_rel_subfig) at (-0.6, 0);

  % Data right column

  \coordinate (rightcol) at ($ (leftcolwidth) + (1,0) $);
  \newcommand\rightcolwidth{9}

  \coordinate (rightcolheight) at (0,-6);

  \node[anchor=north west] at ($ (label_rel_subfig) + (-0.2, 0) $) {a)};

  % Saliency map

  \path[draw, boxstyle] (0,0) rectangle  ($ (leftcolwidth) + (leftcolheight) $);
  \node[anchor=north west, inner sep=0] at (leftcolimage)
    {\includegraphics[height=\leftcolimgheight\tikzunit]{saliencymap.png}};
  \node[anchor=west] at (leftcoltext) {Saliency map};

  \draw[->,thick] ($ (leftcolheight) + 0.5*(leftcolwidth) + (0, -0.15) $) -- +($ (leftcolsep) + (0, 0.3) $);

  % Nonlinearity

  \path[draw, boxstyle,fill=colornonlinearity] (lefttotalcolheight) rectangle  ($  (lefttotalcolheight) + (leftcolsize) $);
  \node[anchor=north west, inner sep=0] at ($ (lefttotalcolheight) + (leftcolimage) $)
    {\includegraphics[height=\leftcolimgheight\tikzunit]{nonlinearity_symbol.pdf}};
  \node[anchor=west] at ($ (lefttotalcolheight) + (leftcoltext) $) {Nonlinearity};

  \draw[->,thick] ($ 1*(lefttotalcolheight) + (leftcolheight) + 0.5*(leftcolwidth) + (0, -0.15) $) -- +($ (leftcolsep) + (0, 0.3) $);
  %\node at ($ 1*(lefttotalcolheight) + (leftcolop) $) {+};

  % Centerbias

  \path[draw, boxstyle, fill=colorcenterbias]
    ($ 2*(lefttotalcolheight) $) rectangle ($  2*(lefttotalcolheight) + (leftcolsize) $);
  \coordinate (this_symbol_pos) at ($ 2*(lefttotalcolheight) + (leftcolimage) $);
  \node[anchor=north west, inner sep=0] at ($ (this_symbol_pos) -0.5*(leftcolimagestack) $)
    {\includegraphics[height=\leftcolimgheight\tikzunit]{nonlinearity_symbol.pdf}};
  \node[anchor=north west, inner sep=0] at ($ (this_symbol_pos) +0.5*(leftcolimagestack) $)
    {\includegraphics[height=\leftcolimgheight\tikzunit]{saliencymap_nonparametric.png}};
  \node[anchor=west,align=left] at ($ 2*(lefttotalcolheight) + (leftcoltext) $) {Nonlinearity\\Centre bias};

  \draw[->,thick] ($ 2*(lefttotalcolheight) + (leftcolheight) + 0.5*(leftcolwidth) + (0, -0.15) $) -- +($ (leftcolsep) + (0, 0.3) $);
  %\node at ($ 2*(lefttotalcolheight) + (leftcolop) $) {+};

  % Blur

  \path[draw, boxstyle, fill=colorblur]
    ($ 3*(lefttotalcolheight) $) rectangle ($  3*(lefttotalcolheight) + (leftcolsize) $);
  \coordinate (this_symbol_pos) at ($ 3*(lefttotalcolheight) + (leftcolimage) $);
  \node[anchor=north west, inner sep=0] at ($ (this_symbol_pos) -1*(leftcolimagestack) $)
    {\includegraphics[height=\leftcolimgheight\tikzunit]{nonlinearity_symbol.pdf}};
  \node[anchor=north west, inner sep=0] at ($ (this_symbol_pos) +0.0*(leftcolimagestack) $)
    {\includegraphics[height=\leftcolimgheight\tikzunit]{saliencymap_nonparametric.png}};
  \node[anchor=north west, inner sep=0] at ($ (this_symbol_pos) +1.0*(leftcolimagestack) $)
    {\includegraphics[height=\leftcolimgheight\tikzunit]{blur_symbol.pdf}};
  \node[anchor=west,align=left] at ($ 3*(lefttotalcolheight) + (leftcoltext) $) {Nonlinearity\\Centre bias\\Blurring};


  %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  %%%% Right column

  \node[anchor=north west] at ($ (rightcol) + (label_rel_subfig) $) {b)};

  \node[anchor=north west] at (rightcol) {\includegraphics[width=\rightcolwidth\tikzunit]
    {log_likelihoods_JuddEtAl2009Cor.pdf}};

  \node[anchor=north west] at ($ (rightcol) + (rightcolheight) + (label_rel_subfig) $) {c)};

  \node[anchor=north west] at ($ (rightcol) + (rightcolheight) $)
   {\includegraphics[width=\rightcolwidth\tikzunit]%
     {log_likelihoods_rel_centerbias_percent_JuddEtAl2009Cor.pdf}};



%  \path[draw, boxstyle] ($ (leftcolheight) + (leftcolsep) $)
%       rectangle  ($ (leftcolheight) + (leftcolsep) + (leftcolwidth) + (leftcolheight) $);



\end{tikzpicture}

% Clean up
\let\abstikzunit\undef
\let\scalingfactor\undef
\let\imgwidth\undef


model_colors.tex
\definecolor{colornonlinearity}{rgb}{0.98046875, 0.6015625, 0.59765625}
\definecolor{colorcenterbias}{rgb}{0.19921875, 0.625, 0.171875}
\definecolor{colorblur}{rgb}{0.6953125, 0.87109375, 0.5390625}
\definecolor{colorexcitation}{rgb}{0.12109375, 0.46875, 0.703125}
\definecolor{colorexcentric}{rgb}{0.6484375, 0.8046875, 0.88671875}

information_gains_schema.pgf.tex
\ifdefined\noktikz
  \input{figures/informationgainsdata}
  \input{figures/ig_colors}
\else
  \usetikzlibrary{calc}
  \usetikzlibrary{shapes}
  \graphicspath{{/kyb/agmb/mkuemmerer/Documents/Uni/Bethge/Saliency/TPAMI/figures/}}

  \input{/kyb/agmb/mkuemmerer/Documents/Uni/Bethge/Saliency/TPAMI/figures/informationgainsdata}
  \input{/kyb/agmb/mkuemmerer/Documents/Uni/Bethge/Saliency/TPAMI/figures/ig_colors}
\fi


\newcommand\loglik[1]{\textsf{$\mathsf{#1}$\,bits/fix}}
\newcommand\auc[1]{\textsf{$\mathsf{#1}$\,\%}}

%\newlength\imgwidth
%\setlength\imgwidth{4cm}

\ifdefined\abstikzunit
\else
	\newlength\abstikzunit
	\newlength\tikzunit
\fi

\setlength\abstikzunit{1cm}
\newcommand\scalingfactor{0.5}
\newcommand\imgwidth{3.5}
\newcommand\labelalpha{0.4}
\newcommand\labelheight{1.5}
\newcommand\symbolwidth{1}
\setlength\tikzunit{\scalingfactor\abstikzunit}


\setlength\abstikzunit{1cm}
\setlength\tikzunit{\scalingfactor\abstikzunit}

%\newcommand\imgwidth{5cm}
%\newcommand\imgwidth{4cm}

% TODO:
% - colorbar, probably per image and with values

\begin{tikzpicture}[scale=\scalingfactor,font=\tiny\sffamily,every node/.style={inner sep=0,outer sep=0}]
  \tikzset{labelnode/.style={anchor=south, inner ysep=1,align=center,font=\tiny\sffamily,
    minimum height=\labelheight\tikzunit,text width=\imgwidth\tikzunit,
    fill opacity=\labelalpha,text opacity=1.0}}


  \coordinate (colwidth) at (4.0, 0);
  \coordinate (row2) at (0, -4);

  \coordinate (col2) at (5.5, 0);
  \coordinate (col3) at ($ (col2) + 1.3*(colwidth) $);
  \coordinate (col4) at ($ (col3) + 1.4*(colwidth) $);
  \coordinate (col5) at ($ (col4) + 1.25*(colwidth) $);

%  \coordinate (row_center) at (0, -1.3);
  \coordinate (row_center) at (0, 0);

  \coordinate (centerbias) at ($ (col2) + 0.5*(colwidth) + (0,4.5) $);
  \coordinate (centerbias_div) at ($ (col2) + 0.7*(colwidth)  + (row_center) $);

  \coordinate (nonlinearity) at ($ (col3) + 0.65*(colwidth)  + (row_center) $);
  \coordinate (gold_mul) at ($ (col3) + 0.85*(colwidth)  + (row_center) $);

  \coordinate (gold_mul_central) at ($ (gold_mul) + 0.5*(row2) $);

  \coordinate (ig_diff) at ($ (col4) + 0.6*(colwidth) $);


  \coordinate (colwidth) at (4.0, 0);
  \coordinate (colheight) at (0, -3);


  \coordinate (label_rel_fig) at (-4.5,3.5);
  \coordinate (ll_rel) at (2.5,-2);

  \coordinate (imagesep) at (0.0, -0.5);

  \coordinate (input_image) at ($ 0.5*(row2) $);

%%%% Input image

  \node (input_image_node) at (input_image)
      {\includegraphics[width=\imgwidth\tikzunit]{information_gains_schema_image.pdf}};

%%%% Densities

  \node (model_density_node) at ($ (col2) $)
      {\includegraphics[width=\imgwidth\tikzunit]{information_gains_schema_model_density.pdf}};

  \node (gold_density_node) at ($ (col2) + (row2) $)
      {\includegraphics[width=\imgwidth\tikzunit]{information_gains_schema_gold_density.pdf}};

  \draw[->] (input_image_node) -- (model_density_node) node[pos=0.5,label={[label distance=5pt]above:model}] {};
  \draw[->] (input_image_node) -- (gold_density_node) node[pos=0.5,label={[label distance=5pt]below:true}] {};

%%%% Saliency maps: Centerbias

  \node (centerbias_node) at ($ (centerbias) $)
      {\includegraphics[width=\imgwidth\tikzunit]{information_gains_schema_centerbias.pdf}};

  \node[circle,draw] (centerbias_model_div_node) at ($ (centerbias_div) $) {$\div$};
  \node[circle,draw] (centerbias_gold_div_node) at ($ (centerbias_div) + (row2) $) {$\div$};

  \draw[->] (centerbias_node) -- (centerbias_model_div_node);
  \draw[->] (centerbias_node) -- (centerbias_gold_div_node);

  \draw (model_density_node) -- (centerbias_model_div_node);
  \draw (gold_density_node) -- (centerbias_gold_div_node);


%%%% Saliencymaps


  \node (model_saliency_node) at ($ (col3) $)
      {\includegraphics[width=\imgwidth\tikzunit]{information_gains_schema_model_saliency.pdf}};

  \node (gold_saliency_node) at ($ (col3) + (row2) $)
      {\includegraphics[width=\imgwidth\tikzunit]{information_gains_schema_gold_saliency.pdf}};

  \draw[->] (centerbias_model_div_node) -- (model_saliency_node);
  \draw[->] (centerbias_gold_div_node) -- (gold_saliency_node);

%%% nonlinearity

  \node[rectangle,draw,inner sep=2pt] (model_nonlinearity_node) at (nonlinearity)
    {$\log$};
  \node[rectangle,draw,inner sep=2pt] (gold_nonlinearity_node) at ($ (nonlinearity) + (row2) $)
    {$\log$};

  \draw[->] (model_saliency_node) -- (model_nonlinearity_node);
  \draw[->] (gold_saliency_node) -- (gold_nonlinearity_node);

%%%% Multiply gold density

  \node[circle,draw] (gold_mul_model_node) at ($ (gold_mul) $) {$\ \cdot\ $};
  \node[circle,draw] (gold_mul_gold_node) at ($ (gold_mul) + (row2) $) {$\ \cdot\ $};

  \draw (gold_density_node) |- (gold_mul_central);
  \draw[->] (gold_mul_central) -- (gold_mul_model_node);
  \draw[->] (gold_mul_central) -- (gold_mul_gold_node);

  \draw (model_nonlinearity_node) -- (gold_mul_model_node);
  \draw (gold_nonlinearity_node) -- (gold_mul_gold_node);

%% %information gain

  \node (model_ig_node) at ($ (col4) $)
      {\includegraphics[width=\imgwidth\tikzunit]{information_gains_schema_model_ig.pdf}};
  \node (gold_ig_node) at ($ (col4) + (row2) $)
      {\includegraphics[width=\imgwidth\tikzunit]{information_gains_schema_gold_ig.pdf}};

  \draw[->] (gold_mul_model_node) -- (model_ig_node);
  \draw[->] (gold_mul_gold_node) -- (gold_ig_node);

%%% diff

  \node[circle,draw] (model_diff_node) at ($ (ig_diff) $) {$-$};

  \draw (model_ig_node) -- (model_diff_node);
  \draw (gold_ig_node) -| (model_diff_node);


  \node (model_igdiff_node) at ($ (col5) $)
      {\includegraphics[width=\imgwidth\tikzunit]{information_gains_schema_model_igdiff.pdf}};

  \draw[->] (model_diff_node) -- (model_igdiff_node);


%%%%%%%%%% Labels

  \node[anchor=south] at (input_image_node.north) {image};
  \node[labelnode,fill=colordensity] at (model_density_node.north) {model density};
  \node[anchor=south] at (centerbias_node.north) {prior};
  \node[labelnode,fill=colorsaliency] at (model_saliency_node.north) {model\\ image-based\\ saliency};
  \node[labelnode,fill=colorig] at (model_ig_node.north) {model\\information gain};
  \node[labelnode,fill=colorigdiff] at (model_igdiff_node.north) {difference to real\\ information gain};
%  \node[anchor=south] at (model_ig_node.north) {information gain};

\end{tikzpicture}

% Clean up
\let\abstikzunit\undef
\let\scalingfactor\undef
\let\imgwidth\undef
\let\pwidth\undef
\let\labelheight\undef
\let\labelalpha\undef
\let\tikzunit\undef

informationgainsdata.tex
\expandafter\xdef\csname informationgain260BMSig\endcsname{-0.40}
\expandafter\xdef\csname informationgain210BMSig\endcsname{2.59}
\expandafter\xdef\csname informationgain163goldig\endcsname{2.80}
\expandafter\xdef\csname informationgain210AIMig\endcsname{2.28}
\expandafter\xdef\csname informationgain163eDNig\endcsname{0.16}
\expandafter\xdef\csname informationgain163AIMig\endcsname{-1.06}
\expandafter\xdef\csname informationgain260eDNig\endcsname{-0.69}
\expandafter\xdef\csname informationgain210eDNig\endcsname{3.12}
\expandafter\xdef\csname informationgain163BMSig\endcsname{-0.92}
\expandafter\xdef\csname informationgain260AIMig\endcsname{-0.59}
\expandafter\xdef\csname informationgain210goldig\endcsname{3.92}
\expandafter\xdef\csname informationgain260goldig\endcsname{2.10}
\expandafter\xdef\csname imgindexa\endcsname{210}
\expandafter\xdef\csname imgindexb\endcsname{260}
\expandafter\xdef\csname imgindexc\endcsname{163}

ig_colors.tex
\definecolor{colordensity}{rgb}{0.19921875, 0.625, 0.171875}
\definecolor{colorsaliency}{rgb}{0.6953125, 0.87109375, 0.5390625}
\definecolor{colorig}{rgb}{0.12109375, 0.46875, 0.703125}
\definecolor{colorigdiff}{rgb}{0.6484375, 0.8046875, 0.88671875}

information_gains.pgf.tex
\ifdefined\noktikz
  \input{figures/informationgainsdata}
  \input{figures/ig_colors}
\else
  \usetikzlibrary{calc}
  \usetikzlibrary{shapes}
  \graphicspath{{/kyb/agmb/mkuemmerer/Documents/Uni/Bethge/Saliency/TPAMI/figures/}}
  \input{/kyb/agmb/mkuemmerer/Documents/Uni/Bethge/Saliency/TPAMI/figures/ig_colors}
  \input{/kyb/agmb/mkuemmerer/Documents/Uni/Bethge/Saliency/TPAMI/figures/informationgainsdata}
\fi


\newcommand\loglik[1]{\textsf{$\mathsf{#1}$\,bits/fix}}
\newcommand\auc[1]{\textsf{$\mathsf{#1}$\,\%}}

%\newlength\imgwidth
%\setlength\imgwidth{4cm}

\ifdefined\abstikzunit
\else
	\newlength\abstikzunit
	\newlength\tikzunit
\fi

\setlength\abstikzunit{1cm}
\newcommand\scalingfactor{0.5}
\newcommand\imgwidth{5.0}
\newcommand\pwidth{3.5}
\newcommand\labelalpha{0.4}
\newcommand\labelheight{1.5}
\setlength\tikzunit{\scalingfactor\abstikzunit}


\setlength\abstikzunit{1cm}
\setlength\tikzunit{\scalingfactor\abstikzunit}

%\newcommand\imgwidth{5cm}
%\newcommand\imgwidth{4cm}

% TODO:
% - colorbar, probably per image and with values

\begin{tikzpicture}[scale=\scalingfactor,font=\small\sffamily,every node/.style={inner sep=0,outer sep=0}]
  \tikzset{labelnode/.style={anchor=south, inner ysep=1,align=center,font=\tiny\sffamily,
    minimum height=\labelheight\tikzunit,text width=\pwidth\tikzunit,
    fill opacity=\labelalpha,text opacity=1.0}}
  \coordinate (col2) at (5.5, 0);
  \coordinate (colwidth) at (4.0, 0);
  \coordinate (colheight) at (0, -3);

  \coordinate (row2) at (0, -4);
  \coordinate (label_rel_fig) at (-4.5,3.5);
  \coordinate (ll_rel) at (2.5,-2);

  \coordinate (imagesep) at (0.0, -0.5);

  \node[labelnode,fill=colordensity]
    at ($ (col2) + 0*(colwidth) + 0.5*(\pwidth, 0) $)
    {\tiny \textsf{model density}};
  \node[labelnode,fill=colorsaliency]
    at ($ (col2) + 1*(colwidth) + 0.5*(\pwidth, 0) $)
    {model\\ image-based\\ saliency};
  \node[labelnode,fill=colorig]
     at ($ (col2) + 2*(colwidth) + 0.5*(\pwidth, 0) $)
    {model\\information gain};
  \node[labelnode,fill=colorigdiff]
     at ($ (col2) + 3*(colwidth) + 0.5*(\pwidth, 0) $)
    {difference to real\\ information gain};



 
  \foreach[count=\imgpos] \imgindex/\labelname in {\imgindexa/(a),\imgindexb/(b),\imgindexc/(c)}
  {
    \coordinate (imgcoord) at ($ \imgpos*4*(colheight) - 4*(colheight) + \imgpos*(imagesep) - (imagesep) $);

    \node[anchor=north east] at ($ (imgcoord) - (0.3, 0) $) {\textbf{\labelname}};

    \node[anchor=north west] at (imgcoord)
      {\includegraphics[width=\imgwidth\tikzunit]{informationgains-\imgindex-image.png}};

    \foreach[count=\rowpos] \modelname in {gold,eDN,BMS,AIM}
    {
	   \node[anchor=south east, rotate=90, color=gray] at ($ (imgcoord) + (col2) + \rowpos*(colheight) - (colheight)  $)
            {\tiny \modelname };

      \foreach \pos/\name in {0/p,1/ratio, 2/ig, 3/igdiff} {
        %\ifnum\pos<3
          %        \ifnum\pdfstrcmp{\modelname}{gold}=0
          \node[draw,anchor=north west] at ($ (imgcoord) + (col2) + \pos*(colwidth) + \rowpos*(colheight) - (colheight) $)
            {\includegraphics[width=\pwidth\tikzunit]{informationgains-\imgindex-\modelname-\name.pdf}};
      %\fi\fi
      }

        %\node[anchor=north west] at ($ (imgcoord) + (col2) + 2*(colwidth) + \rowpos*(colheight) - (colheight) + (-0.1, -0.1) $)
         % % as the tex commands containing the information gains are not compatible
          %% with the usual tex syntax, we have to use \csname to call them.
          %{\tiny \csname informationgain\imgindex\modelname ig\endcsname\ bits/fix};

        \node[anchor=north east, rotate=90] at ($ (imgcoord) + (col2) + 4*(colwidth) + \rowpos*(colheight) - (colheight) + (-0.3, -0.0) $)
          % as the tex commands containing the information gains are not compatible
          % with the usual tex syntax, we have to use \csname to call them.
          {\tiny \csname informationgain\imgindex\modelname ig\endcsname\ bit/fix};

    }
  }

%   \node[anchor=north west] at (col2)
%     {\includegraphics[width=\pwidth\tikzunit]{informationgains-163-gold-p.pdf}};
% 
%   \node[anchor=north west] at ($ (col2) + (colwidth) $)
%     {\includegraphics[width=\pwidth\tikzunit]{informationgains-163-gold-ratio.pdf}};



\end{tikzpicture}

% Clean up
\let\abstikzunit\undef
\let\scalingfactor\undef
\let\imgwidth\undef
\let\pwidth\undef
\let\labelheight\undef
\let\labelalpha\undef
\let\tikzunit\undef

information_gains_scatter_combined.pgf.tex
\ifdefined\noktikz
  \input{figures/informationgainsdata}
  \newcommand\figurepath{figures}
\else
  \usetikzlibrary{calc}
  \usetikzlibrary{shapes}
  \graphicspath{{/kyb/agmb/mkuemmerer/Documents/Uni/Bethge/Saliency/TPAMI/figures/}}
  \newcommand\figurepath{/kyb/agmb/mkuemmerer/Documents/Uni/Bethge/Saliency/TPAMI/figures}

%  \input{/kyb/agmb/mkuemmerer/Documents/Uni/Bethge/Saliency/TPAMI/figures/informationgainsscatterdata}
\fi

%\ifdefined\abstikzunit
%\else
%    \newlength\abstikzunit
%    \newlength\tikzunit
%\fi

%\setlength\abstikzunit{1cm}
%\newcommand\scalingfactor{2}
%\newcommand\imgwidth{5.0}
%\newcommand\pwidth{0.5}
%\setlength\tikzunit{\scalingfactor\abstikzunit}


%\setlength\abstikzunit{1cm}
%\setlength\tikzunit{\scalingfactor\abstikzunit}

\begin{tikzpicture}[font=\small\sffamily,every node/.style={inner sep=0,outer sep=0}]
  \coordinate (col2) at (7.5,0);
  \coordinate (rowsep) at (0,-0.7);
  \coordinate (labelsep) at (-0.5, 0);

  \node[anchor=north west] (eDN1) at (0,0) {
      
     \begin{minipage}[t]{8.5cm}
        {
             \newcommand\imageprefix{general}
             \newcommand\imagesuffix{image}
             \newcommand\imageext{png}
			 \newcommand\modelname{eDN}
             \input{\figurepath/information_gains_scatter_all.pgf}
       }
     \end{minipage}
     };

  \node[anchor=north west] (eDN2) at ($ (eDN1.north west) + (col2) $) {
       \begin{minipage}[t]{8.5cm}
        {
             \newcommand\imageprefix{eDN}
             \newcommand\imagesuffix{igdiff}
             \newcommand\imageext{pdf}
			 \newcommand\modelname{eDN}
             \input{\figurepath/information_gains_scatter_all.pgf}
       }
     \end{minipage}
     };

  \node[anchor=north east] at ($ (eDN1.north west) + (labelsep) $) {\textbf{eDN}};

  \node[anchor=north west] (Judd1) at ($ (eDN1.south west) + (rowsep) $) {
      
     \begin{minipage}[t]{8.5cm}
        {
             \newcommand\imageprefix{general}
             \newcommand\imagesuffix{image}
             \newcommand\imageext{png}
			 \newcommand\modelname{Judd}
             \input{\figurepath/information_gains_scatter_all.pgf}
       }
     \end{minipage}
     };

  \node[anchor=north west] (Judd2) at ($ (Judd1.north west) + (col2) $) {
       \begin{minipage}[t]{8.5cm}
        {
             \newcommand\imageprefix{Judd}
             \newcommand\imagesuffix{igdiff}
             \newcommand\imageext{pdf}
			 \newcommand\modelname{Judd}
             \input{\figurepath/information_gains_scatter_all.pgf}
       }
     \end{minipage}
     };

  \node[anchor=north east] at ($ (Judd1.north west) + (labelsep) $) {\textbf{Judd}};

\end{tikzpicture}

information_gains_scatter_all.pgf.tex
\ifdefined\noktikz
  \input{figures/informationgainsdata}
\else
  \usetikzlibrary{calc}
  \usetikzlibrary{shapes}
  \graphicspath{{/kyb/agmb/mkuemmerer/Documents/Uni/Bethge/Saliency/presentations/poster_Bernstein_2014/figures/}}

  \input{/kyb/agmb/mkuemmerer/Documents/Uni/Bethge/Saliency/presentations/poster_Bernstein_2014/figures/informationgainsscatterdata}
\fi


\newcommand\loglik[1]{\textsf{$\mathsf{#1}$\,bits/fix}}
\newcommand\auc[1]{\textsf{$\mathsf{#1}$\,\%}}

%\newlength\imgwidth
%\setlength\imgwidth{4cm}

\ifdefined\abstikzunit
\else
	\newlength\abstikzunit
	\newlength\tikzunit
\fi

\setlength\abstikzunit{1cm}
\newcommand\scalingfactor{1.15}
\newcommand\imgwidth{5.0}
\newcommand\pwidth{0.5}
\setlength\tikzunit{\scalingfactor\abstikzunit}


\setlength\abstikzunit{1cm}
\setlength\tikzunit{\scalingfactor\abstikzunit}

%\newcommand\imgwidth{5cm}
%\newcommand\imgwidth{4cm}

% TODO:
% - colorbar, probably per image and with values

\ifdefined\noktikz
\else
  \newcommand\imageprefix{general}
  \newcommand\imagesuffix{image}
  \newcommand\imageext{png}
  \newcommand\modelname{eDN}
\fi

\begin{tikzpicture}[x=1.3*\tikzunit,y=0.03*\tikzunit,font=\small\sffamily,every node/.style={inner sep=0,outer sep=0}]
  \coordinate (xstart)  at (1.5,0);
  \coordinate (rightend) at (5.5,0);
  \coordinate (width) at ($ (rightend) - (xstart) $);
  \coordinate (height) at (0,100);
  \coordinate (height2) at (0,-50);
  \coordinate (O) at (0, 0);
  \coordinate (xticksize) at (0, -3);

  \draw[line width=0.5,dotted] ($ (xstart) + (height) $) -- ($ (rightend) + (height) $);
  \draw[line width=0.5,dotted] ($ (xstart) + (O) $) -- ($ (rightend) + (O) $);


  \ifdefined\noktikz
    \input{figures/information_gains_\modelname}
  \else
    \input{/kyb/agmb/mkuemmerer/Documents/Uni/Bethge/Saliency/presentations/poster_Bernstein_2014/figures/information_gains_\modelname}
  \fi

  \draw[line width=0.5] ($ (xstart) + (height) $)-- ($ (xstart) + (height2) $);
  \draw[line width=0.5] ($ (xstart) + (height2) $) -- ($ (xstart) + (height2) + (width) $);

  \node[anchor=north, outer sep = 5] at ($ (height2) + 0.5*(width) + (xstart) $) {\tiny Gold [bit/fix]};
  \node[anchor=south, outer sep = 5,rotate=90] at ($ (xstart) + 0.5*(height) $) {\tiny explained information};

  \foreach \xtick in {2.0,5.0}
  {
    \draw[line width=0.5] ($ (\xtick, 0) + (height2) $) -- ($ (\xtick, 0) + (height2) - (xticksize) $);
    \node[anchor=north, outer sep=5] at ($ (\xtick, 0) + (height2) $) {\tiny\xtick};
  }

  \foreach \ytick in {0,100}
  {
%    \draw[line width=1] ($ (0, \ytick) + (ticksize) $) -- ($ (\xtick, 0) + (height) - (ticksize) $);
    \node[anchor=east, outer sep=3] at ($ (0, \ytick) + (xstart) $) {\tiny\ytick \%};
  }


\end{tikzpicture}

% Clean up
\let\abstikzunit\undef
\let\scalingfactor\undef
\let\imgwidth\undef
\let\pwidth\undef
\let\tikzunit\undef

information_gains_eDN.tex
\filldraw (2.75628735815, 48.8573788505) circle (1pt);
\filldraw (2.24298312186, 52.9469598324) circle (1pt);
\filldraw (2.97801211785, 37.175038457) circle (1pt);
\filldraw (2.34572990786, 50.5047853311) circle (1pt);
\filldraw (2.61244411197, 44.5805927707) circle (1pt);
\filldraw (2.58626052495, 51.4878393492) circle (1pt);
\filldraw (2.46729941717, 67.3352933532) circle (1pt);
\filldraw (2.30747631695, 47.9593213983) circle (1pt);
\filldraw (3.33078763402, 58.2023000591) circle (1pt);
\filldraw (2.91484561608, 45.4464700758) circle (1pt);
\filldraw (2.60360118347, 16.8947030003) circle (1pt);
\filldraw (2.66507773942, 55.9738809543) circle (1pt);
\filldraw (2.31997095442, 53.5400337017) circle (1pt);
\filldraw (3.65515491734, 54.3835830405) circle (1pt);
\filldraw (3.0329520141, 38.6740075297) circle (1pt);
\filldraw (2.95968348844, 29.5897393323) circle (1pt);
\filldraw (2.95155992666, 54.0412318945) circle (1pt);
\filldraw (3.22235926758, 49.2485915042) circle (1pt);
\filldraw (2.46373672668, 49.5710960275) circle (1pt);
\filldraw (2.66197679004, 57.0698843678) circle (1pt);
\filldraw (2.78988300487, 41.3340617684) circle (1pt);
\filldraw (3.07605779985, 16.6695395908) circle (1pt);
\filldraw (2.41016676761, 43.1092790185) circle (1pt);
\filldraw (2.51526082509, 49.5033653162) circle (1pt);
\filldraw (3.81813052482, 73.8400623186) circle (1pt);
\filldraw (3.30915494738, 65.9628149612) circle (1pt);
\filldraw (2.33948031515, 30.8027930084) circle (1pt);
\filldraw (2.8048642717, 50.0401635488) circle (1pt);
\filldraw (2.64395860481, 33.5408313212) circle (1pt);
\filldraw (3.63572265949, 68.1484347454) circle (1pt);
\filldraw (2.457339633, 37.5109655607) circle (1pt);
\filldraw (2.92782906242, 68.7670471319) circle (1pt);
\filldraw (3.15958971312, 23.2521432804) circle (1pt);
\filldraw (2.53905930651, 31.8115632199) circle (1pt);
\filldraw (2.67685357278, 11.1673040742) circle (1pt);
\filldraw (3.17318514919, 55.1137657538) circle (1pt);
\filldraw (1.90385760345, 12.6203487555) circle (1pt);
\filldraw (3.46555938588, 64.0813472643) circle (1pt);
\filldraw (2.57801224683, 42.1761439186) circle (1pt);
\filldraw (2.2767703124, 37.3762581486) circle (1pt);
\filldraw (3.12969729221, 35.6041765995) circle (1pt);
\filldraw (2.95895927482, 25.5666712183) circle (1pt);
\filldraw (2.70467463589, 57.3822704145) circle (1pt);
\filldraw (2.80848882487, 36.7770506004) circle (1pt);
\filldraw (3.04135161212, 45.9101343055) circle (1pt);
\filldraw (2.97251555438, 50.6400067887) circle (1pt);
\filldraw (2.88106941369, 34.5785823152) circle (1pt);
\filldraw (2.24093688711, 33.2915345917) circle (1pt);
\filldraw (2.65369274586, 43.1894963244) circle (1pt);
\filldraw (2.80917533892, 46.6871808539) circle (1pt);
\filldraw (2.85036608977, 47.7440356457) circle (1pt);
\filldraw (2.45270722533, 53.4546921046) circle (1pt);
\filldraw (3.09639755938, 53.1336889235) circle (1pt);
\filldraw (2.88591605328, 51.7182014815) circle (1pt);
\filldraw (2.60982746578, 21.5828494793) circle (1pt);
\filldraw (2.21117352229, 20.3600888869) circle (1pt);
\filldraw (2.09361467566, 26.3947980485) circle (1pt);
\filldraw (2.53027408135, 64.3921641401) circle (1pt);
\filldraw (2.55499748005, 65.8728310177) circle (1pt);
\filldraw (2.95135185707, 56.8207968321) circle (1pt);
\filldraw (2.7834354129, 61.5314625838) circle (1pt);
\filldraw (2.4971459385, 64.1507450256) circle (1pt);
\filldraw (2.56999572106, 63.7692814775) circle (1pt);
\filldraw (2.87800355161, 54.0823947671) circle (1pt);
\filldraw (3.47940927804, 54.9260039959) circle (1pt);
\filldraw (2.96895017484, 27.5357076523) circle (1pt);
\filldraw (2.41327908521, 34.9585163125) circle (1pt);
\filldraw (2.56496598565, 52.6868757114) circle (1pt);
\filldraw (2.64653803204, 66.1890816463) circle (1pt);
\filldraw (3.11256125497, 62.4842720801) circle (1pt);
\filldraw (2.4912851486, 46.7777417766) circle (1pt);
\filldraw (2.85631184337, 56.5956988396) circle (1pt);
\filldraw (2.92914629726, 46.1118236681) circle (1pt);
\filldraw (2.79746594029, 66.2190412036) circle (1pt);
\filldraw (2.48143731709, 16.7129025342) circle (1pt);
\filldraw (3.9494449262, 53.5545704385) circle (1pt);
\filldraw (3.0749608549, 63.5684419139) circle (1pt);
\filldraw (3.03872454688, 44.3945245713) circle (1pt);
\filldraw (3.35075873339, 33.601534526) circle (1pt);
\filldraw (3.21927412194, 36.4894134716) circle (1pt);
\filldraw (2.70169877705, 57.1823312) circle (1pt);
\filldraw (2.62709222423, 21.8171137042) circle (1pt);
\filldraw (2.1453917887, 55.7010268641) circle (1pt);
\filldraw (2.23419042313, 48.3033391646) circle (1pt);
\filldraw (3.34322780157, 39.4091411739) circle (1pt);
\filldraw (3.72488817418, 53.8769889555) circle (1pt);
\filldraw (3.04448850223, 34.5555049333) circle (1pt);
\filldraw (3.39107048598, 40.9455250063) circle (1pt);
\filldraw (2.60316719083, 66.4854741513) circle (1pt);
\filldraw (3.1521985385, 39.5265967986) circle (1pt);
\filldraw (2.40548775918, 50.0135629198) circle (1pt);
\filldraw (2.34886881669, 42.8613029833) circle (1pt);
\filldraw (2.69636621937, 54.0854485304) circle (1pt);
\filldraw (2.47722364939, 56.8360491693) circle (1pt);
\filldraw (2.26843358119, 33.2312504221) circle (1pt);
\filldraw (2.48210412197, 62.3097253326) circle (1pt);
\filldraw (2.81407573955, 48.279530931) circle (1pt);
\filldraw (3.33659759686, 61.2432698242) circle (1pt);
\filldraw (3.49161934471, 51.3412338117) circle (1pt);
\filldraw (2.84128625467, 30.418743819) circle (1pt);
\filldraw (2.18652098541, 45.1812805894) circle (1pt);
\filldraw (2.28160163129, 28.4407310139) circle (1pt);
\filldraw (2.69800434151, 41.3910670604) circle (1pt);
\filldraw (3.19288271646, 54.8401143966) circle (1pt);
\filldraw (2.76297904171, 33.8260454727) circle (1pt);
\filldraw (2.68062227435, 53.8898046255) circle (1pt);
\filldraw (2.82153332929, 53.5491589077) circle (1pt);
\filldraw (3.11659150638, 37.086268769) circle (1pt);
\filldraw (3.13468662555, 42.794699809) circle (1pt);
\filldraw (3.54607667145, 33.5860600971) circle (1pt);
\filldraw (2.29647128477, 27.826476526) circle (1pt);
\filldraw (2.38228508563, 34.4861472525) circle (1pt);
\filldraw (3.15435965733, 21.9064081611) circle (1pt);
\filldraw (2.83774912265, 55.9315771161) circle (1pt);
\filldraw (2.81871652392, 57.8748628329) circle (1pt);
\filldraw (2.82441855463, 63.4781104546) circle (1pt);
\filldraw (2.78150939745, 23.3765846614) circle (1pt);
\filldraw (3.90084227816, 37.4546949745) circle (1pt);
\filldraw (2.76323718248, 45.6490320616) circle (1pt);
\filldraw (3.06804589243, 55.4841372588) circle (1pt);
\filldraw (3.30631134059, 48.0026095484) circle (1pt);
\filldraw (2.98492162199, 45.0567125837) circle (1pt);
\filldraw (2.75285773065, 65.7032496846) circle (1pt);
\filldraw (2.68918909611, 49.5912652492) circle (1pt);
\filldraw (4.09781856656, 63.3763258312) circle (1pt);
\filldraw (2.6180121029, 33.1356588189) circle (1pt);
\filldraw (3.77904424789, 28.9186674382) circle (1pt);
\filldraw (2.95777341877, 49.4506966237) circle (1pt);
\filldraw (2.38934020862, 47.7707647883) circle (1pt);
\filldraw (3.17122607663, 35.1518466404) circle (1pt);
\filldraw (4.17575852741, 47.1961554483) circle (1pt);
\filldraw (2.49199406207, 32.6064882551) circle (1pt);
\filldraw (2.92435861971, 35.8975148225) circle (1pt);
\filldraw (3.10809272865, 41.317096806) circle (1pt);
\filldraw (2.86020579265, 72.7318789894) circle (1pt);
\filldraw (3.26637437632, 51.8977634823) circle (1pt);
\filldraw (3.2133085336, 35.0483512971) circle (1pt);
\filldraw (2.36901958624, 44.1256332111) circle (1pt);
\filldraw (2.92052275717, 56.9178956614) circle (1pt);
\filldraw (2.77900464806, 36.3773597437) circle (1pt);
\filldraw (2.29729335467, 45.0757688665) circle (1pt);
\filldraw (2.76635762228, 9.35525069186) circle (1pt);
\filldraw (3.76452829744, 57.1269810839) circle (1pt);
\filldraw (3.05335508826, 42.3463690439) circle (1pt);
\filldraw (3.22181726739, 52.193449816) circle (1pt);
\filldraw (2.95637804486, 42.634955105) circle (1pt);
\filldraw (2.61274754154, 57.0047399013) circle (1pt);
\filldraw (2.33694479808, 31.3957403939) circle (1pt);
\filldraw (3.50821376546, 36.4406477043) circle (1pt);
\filldraw (2.324765685, 36.7162882015) circle (1pt);
\filldraw (2.5021883186, 63.6291330801) circle (1pt);
\filldraw (3.08781745451, 62.5120201708) circle (1pt);
\filldraw (2.43583394796, 65.1472685127) circle (1pt);
\filldraw (3.08916204695, 30.3939819873) circle (1pt);
\filldraw (2.55755129544, 59.0797031729) circle (1pt);
\filldraw (2.74417990566, 42.7767875684) circle (1pt);
\filldraw (2.44202478994, 9.51727704496) circle (1pt);
\filldraw (3.1795299594, 48.7304206009) circle (1pt);
\filldraw (2.32936807493, 52.6366662057) circle (1pt);
\filldraw (3.204625918, 45.4373210459) circle (1pt);
\filldraw (3.24176822975, 36.2785626135) circle (1pt);
\filldraw (2.77567353362, 14.286758818) circle (1pt);
\filldraw (2.46424967258, 11.8620815237) circle (1pt);
\filldraw (2.57849895356, 62.7802125045) circle (1pt);
\filldraw (3.53481742706, 60.0348342627) circle (1pt);
\filldraw (2.42944163452, 28.4128847821) circle (1pt);
\filldraw (2.35113463883, 56.4501294452) circle (1pt);
\filldraw (3.86309982107, 50.6787459532) circle (1pt);
\filldraw (3.71369348777, 50.8716531898) circle (1pt);
\filldraw (2.63934227042, 62.9671076383) circle (1pt);
\filldraw (2.702170052, 0.928567988542) circle (1pt);
\filldraw (2.76545639724, 39.1889565225) circle (1pt);
\filldraw (3.33962827315, 31.6934443151) circle (1pt);
\filldraw (2.92544900927, 64.4313544388) circle (1pt);
\filldraw (2.39583582869, 23.5093909752) circle (1pt);
\filldraw (2.74174749, 68.9132219083) circle (1pt);
\filldraw (2.69296637046, 42.2885449285) circle (1pt);
\filldraw (3.0855296822, 69.4713908829) circle (1pt);
\filldraw (2.11591504049, 24.7112592734) circle (1pt);
\filldraw (3.14852808913, 45.2861693027) circle (1pt);
\filldraw (2.79445584341, 46.4701140225) circle (1pt);
\filldraw (2.79599905623, 55.3404503822) circle (1pt);
\filldraw (1.85726770026, 39.06885335) circle (1pt);
\filldraw (2.69981616984, 48.0963443477) circle (1pt);
\filldraw (2.25890958952, 26.4942138534) circle (1pt);
\filldraw (2.2669950975, 26.7491649527) circle (1pt);
\filldraw (3.15540188569, 39.7904415412) circle (1pt);
\filldraw (2.50188485039, 45.3751091022) circle (1pt);
\filldraw (3.44037931922, 54.6190238211) circle (1pt);
\filldraw (2.92390830976, 52.6275796343) circle (1pt);
\filldraw (3.11674458299, 35.9169966647) circle (1pt);
\filldraw (2.36098134693, 50.546753732) circle (1pt);
\filldraw (3.39673105057, 50.8968960264) circle (1pt);
\filldraw (2.79249615531, 35.7293667176) circle (1pt);
\filldraw (2.63144322244, 38.4820359031) circle (1pt);
\filldraw (2.79443179581, 38.5720596416) circle (1pt);
\filldraw (2.56705273233, 54.7342600266) circle (1pt);
\filldraw (2.6916933733, 48.565913711) circle (1pt);
\filldraw (2.86363739968, 54.3930035397) circle (1pt);
\filldraw (2.80006180504, 50.181877857) circle (1pt);
\filldraw (3.04818473348, 44.2664001318) circle (1pt);
\filldraw (2.77699908715, 55.7577144394) circle (1pt);
\filldraw (3.8015607397, 28.0883341897) circle (1pt);
\filldraw (2.48281194638, 30.1572624827) circle (1pt);
\filldraw (2.65659881831, 30.5133514813) circle (1pt);
\filldraw (2.30168222691, 40.7550570686) circle (1pt);
\filldraw (2.43436823546, 52.125852087) circle (1pt);
\filldraw (2.25662724973, 20.413497732) circle (1pt);
\filldraw (2.57887876166, 19.0176211013) circle (1pt);
\filldraw (2.71654738539, 51.5740226595) circle (1pt);
\filldraw (3.06319590806, 54.6152104869) circle (1pt);
\filldraw (2.19869388505, 38.0620874425) circle (1pt);
\filldraw (3.55555396812, 55.2836987044) circle (1pt);
\filldraw (3.11704792023, 65.2041613871) circle (1pt);
\filldraw (3.2352143723, 61.1835125526) circle (1pt);
\filldraw (2.15334701869, 38.1639094048) circle (1pt);
\filldraw (2.79650190679, 46.46814488) circle (1pt);
\filldraw (2.76732034248, 55.0931919393) circle (1pt);
\filldraw (2.86662875205, 56.675501172) circle (1pt);
\filldraw (3.09855643238, 16.6056348664) circle (1pt);
\filldraw (3.35362812086, 68.7013734616) circle (1pt);
\filldraw (3.30158312847, 58.610774508) circle (1pt);
\filldraw (2.41234898302, 62.2401782203) circle (1pt);
\filldraw (3.04933414907, 43.8625912886) circle (1pt);
\filldraw (3.1514403271, 40.8553912587) circle (1pt);
\filldraw (2.49876679641, 40.4708381995) circle (1pt);
\filldraw (3.04317580738, 22.2733879057) circle (1pt);
\filldraw (3.98942212466, 56.316392059) circle (1pt);
\filldraw (2.67003144792, 13.0067401777) circle (1pt);
\filldraw (2.57219326496, 32.3667035069) circle (1pt);
\filldraw (3.10064490939, 65.1442794148) circle (1pt);
\filldraw (2.89130020701, 30.7863957277) circle (1pt);
\filldraw (4.21482792059, 59.9680746642) circle (1pt);
\filldraw (2.98808396391, 55.3229909682) circle (1pt);
\filldraw (2.8667541359, 36.4067571812) circle (1pt);
\filldraw (2.38897425327, 53.3741691284) circle (1pt);
\filldraw (3.55701301457, 62.6264682767) circle (1pt);
\filldraw (2.90372417752, 45.358772083) circle (1pt);
\filldraw (3.24352323956, 56.120553217) circle (1pt);
\filldraw (3.0794856707, 40.977776501) circle (1pt);
\filldraw (2.90682756203, 44.7024100442) circle (1pt);
\filldraw (2.6584328992, 36.191310293) circle (1pt);
\filldraw (3.02438462868, 71.6555815846) circle (1pt);
\filldraw (2.29298655025, 54.0682931661) circle (1pt);
\filldraw (2.68213757045, 27.8225291769) circle (1pt);
\filldraw (3.45742828884, 64.2612923667) circle (1pt);
\filldraw (3.4943148838, 17.1825545862) circle (1pt);
\filldraw (3.41681721143, 40.5858098132) circle (1pt);
\filldraw (2.51380355887, 41.604542149) circle (1pt);
\filldraw (2.84805957288, 51.7786338377) circle (1pt);
\filldraw (1.9923127202, 54.9749014646) circle (1pt);
\filldraw (2.81408542949, 27.0889442999) circle (1pt);
\filldraw (2.89517648226, 5.78530667074) circle (1pt);
\filldraw (3.07466586169, 57.9530213688) circle (1pt);
\filldraw (2.53002213323, 37.5432784249) circle (1pt);
\filldraw (2.7127547372, 60.1127177459) circle (1pt);
\filldraw (2.53320257013, 19.5937327719) circle (1pt);
\filldraw (2.24148953834, 13.5013823493) circle (1pt);
\filldraw (2.31539909856, 37.1070461859) circle (1pt);
\filldraw (2.2689990907, 49.707195317) circle (1pt);
\filldraw (2.63574670666, 56.036020826) circle (1pt);
\filldraw (3.46587805049, 64.4869277712) circle (1pt);
\filldraw (3.00742348204, 48.0818154673) circle (1pt);
\filldraw (3.69600466214, 61.4816993602) circle (1pt);
\filldraw (2.8779700098, 15.5836593207) circle (1pt);
\filldraw (2.48161775132, 54.8790571685) circle (1pt);
\filldraw (2.13691668521, 48.0385636076) circle (1pt);
\filldraw (1.85360670891, 34.6487358127) circle (1pt);
\filldraw (3.02445201106, 67.3045505882) circle (1pt);
\filldraw (2.84908920095, 57.789712406) circle (1pt);
\filldraw (3.54775233476, 33.7576845441) circle (1pt);
\filldraw (3.71148900205, 63.7764493637) circle (1pt);
\filldraw (2.92452587279, 65.8384319093) circle (1pt);
\filldraw (2.67757696734, 10.322973171) circle (1pt);
\filldraw (2.04051854342, 35.3826247424) circle (1pt);
\filldraw (3.231479136, 71.5984285069) circle (1pt);
\filldraw (2.68650596684, 56.6178136871) circle (1pt);
\filldraw (3.29891944332, 65.4723702257) circle (1pt);
\filldraw (2.79971815649, 30.150713693) circle (1pt);
\filldraw (2.96533058946, 49.4551357488) circle (1pt);
\filldraw (2.54957640485, 57.4549530113) circle (1pt);
\filldraw (2.70499912965, 51.2044127325) circle (1pt);
\filldraw (2.50093315963, 49.6843454766) circle (1pt);
\filldraw (3.06235061038, 67.8032500193) circle (1pt);
\filldraw (1.96131531408, 18.1392581523) circle (1pt);
\filldraw (2.79866322861, 5.54499483489) circle (1pt);
\filldraw (2.63714362481, 32.549284221) circle (1pt);
\filldraw (3.28693598115, 58.6957496621) circle (1pt);
\filldraw (3.90878239968, 58.1141138237) circle (1pt);
\filldraw (2.76491782628, 61.458697655) circle (1pt);
\filldraw (2.50879054867, 43.8273193472) circle (1pt);
\filldraw (2.31029140575, 44.6192787123) circle (1pt);
\filldraw (3.20079429964, 52.2946248533) circle (1pt);
\filldraw (3.57633033988, 48.1022029158) circle (1pt);
\filldraw (2.29268959165, 43.897535707) circle (1pt);
\filldraw (2.94967417629, 48.8083740947) circle (1pt);
\filldraw (3.00894124182, 30.5296597943) circle (1pt);
\filldraw (2.73384957096, 34.4504294674) circle (1pt);
\filldraw (1.97096492885, 27.8951892918) circle (1pt);
\filldraw (3.01912063727, 61.1545036537) circle (1pt);
\filldraw (2.33943210156, 50.76194355) circle (1pt);
\filldraw (2.83421691085, 25.7336463565) circle (1pt);
\filldraw (2.37405008767, 7.04231993975) circle (1pt);
\filldraw (3.84481353269, 52.4699604094) circle (1pt);
\filldraw (3.24120424267, 51.2007017125) circle (1pt);
\filldraw (2.61527779345, 7.63327356803) circle (1pt);
\filldraw (2.61885074379, 53.6583978163) circle (1pt);
\filldraw (2.83347546389, 40.6124665509) circle (1pt);
\filldraw (2.59875621755, 45.9981399448) circle (1pt);
\filldraw (2.41199812649, 56.0554840861) circle (1pt);
\filldraw (2.27786707343, 48.2256191051) circle (1pt);
\filldraw (2.75118687756, 59.7807007169) circle (1pt);
\filldraw (2.35543509058, 44.568997807) circle (1pt);
\filldraw (3.23648097861, 55.1539409715) circle (1pt);
\filldraw (2.40787106924, 50.3772077765) circle (1pt);
\filldraw (2.06017594745, 25.5951352569) circle (1pt);
\filldraw (3.62416199961, 73.7926622396) circle (1pt);
\filldraw (2.9873891576, 49.8131341962) circle (1pt);
\filldraw (2.29672095846, 41.8402090044) circle (1pt);
\filldraw (2.14627504496, 51.5430406361) circle (1pt);
\filldraw (3.39428630244, 44.2907860148) circle (1pt);
\filldraw (3.28408801105, 25.353735311) circle (1pt);
\filldraw (4.02850252866, 58.0746913828) circle (1pt);
\filldraw (2.66156506891, 22.3573505878) circle (1pt);
\filldraw (3.69840764902, 54.596692253) circle (1pt);
\filldraw (2.94640229489, 58.9536960266) circle (1pt);
\filldraw (3.52114417998, 44.9782705605) circle (1pt);
\filldraw (2.14410382816, 19.4819402846) circle (1pt);
\filldraw (2.55189276993, 60.7996819205) circle (1pt);
\filldraw (2.80588338969, 51.9078264711) circle (1pt);
\filldraw (2.90245948241, 64.4060031489) circle (1pt);
\filldraw (2.91971582302, 31.6672345727) circle (1pt);
\filldraw (3.54045553507, 61.5795157234) circle (1pt);
\filldraw (2.65424295239, 39.4216879992) circle (1pt);
\filldraw (3.55316057098, 73.3383252255) circle (1pt);
\filldraw (2.91511499028, 23.8543419355) circle (1pt);
\filldraw (3.32732512581, 59.2729466466) circle (1pt);
\filldraw (3.13031258211, 42.9343739698) circle (1pt);
\filldraw (3.20871215834, 45.8726590699) circle (1pt);
\filldraw (2.23984975581, 33.7755615831) circle (1pt);
\filldraw (3.25168069769, 49.8228101563) circle (1pt);
\filldraw (2.89974684568, 60.7981882913) circle (1pt);
\filldraw (3.8256890776, 62.144562928) circle (1pt);
\filldraw (3.23682278301, 64.5360525734) circle (1pt);
\filldraw (3.92853399209, 67.4769726147) circle (1pt);
\filldraw (2.89262444449, 60.8686072611) circle (1pt);
\filldraw (2.51864538702, 39.9146542059) circle (1pt);
\filldraw (3.31607704486, 69.6779378501) circle (1pt);
\filldraw (3.50537624892, 71.1937591687) circle (1pt);
\filldraw (2.74735410631, 49.7967777577) circle (1pt);
\filldraw (2.76922081749, 15.6553041721) circle (1pt);
\filldraw (2.157402006, 37.7972938469) circle (1pt);
\filldraw (2.92782237247, 51.7867555601) circle (1pt);
\filldraw (2.02181383895, 18.0082725119) circle (1pt);
\filldraw (2.82431838692, 50.3317883043) circle (1pt);
\filldraw (1.84653816245, 26.8050463449) circle (1pt);
\filldraw (3.19106479257, 47.3370753018) circle (1pt);
\filldraw (2.17453489689, 30.5678438356) circle (1pt);
\filldraw (2.53573459029, 28.783317716) circle (1pt);
\filldraw (3.35851749561, 52.039130444) circle (1pt);
\filldraw (2.97496102155, 40.8797978896) circle (1pt);
\filldraw (2.71942153331, 55.1881391756) circle (1pt);
\filldraw (2.31381755548, 50.9464958627) circle (1pt);
\filldraw (2.26876792855, 53.1967161658) circle (1pt);
\filldraw (3.05539507234, 60.5046117072) circle (1pt);
\filldraw (2.99308552961, 13.2937969947) circle (1pt);
\filldraw (3.1746101815, 56.0195940693) circle (1pt);
\filldraw (3.21119479014, 46.4718538573) circle (1pt);
\filldraw (4.0132934313, 60.1472888146) circle (1pt);
\filldraw (3.21488227676, 57.3935729727) circle (1pt);
\filldraw (2.97330203063, 21.3642607376) circle (1pt);
\filldraw (3.06202662121, 46.5793147899) circle (1pt);
\filldraw (2.73538672009, 36.8371068279) circle (1pt);
\filldraw (2.35335472434, 47.5263255052) circle (1pt);
\filldraw (3.04164239987, 43.0395294697) circle (1pt);
\filldraw (2.29477976596, 48.9246802272) circle (1pt);
\filldraw (2.12387012284, 32.9940790556) circle (1pt);
\filldraw (2.16160112439, 53.3485160249) circle (1pt);
\filldraw (2.30100862588, 35.8496848157) circle (1pt);
\filldraw (2.89637599973, 22.7251874636) circle (1pt);
\filldraw (2.61887914155, 59.4038162286) circle (1pt);
\filldraw (2.55414320729, 46.9085908556) circle (1pt);
\filldraw (3.3623696642, 45.0609408532) circle (1pt);
\filldraw (2.48912437378, 48.5259913601) circle (1pt);
\filldraw (3.27781042045, 47.4022706723) circle (1pt);
\filldraw (2.73181144779, 53.3035459604) circle (1pt);
\filldraw (3.52252679333, 42.9657132272) circle (1pt);
\filldraw (2.68183313451, 46.8086206403) circle (1pt);
\filldraw (2.56079590474, 51.9842527089) circle (1pt);
\filldraw (2.43338356017, 53.8440665661) circle (1pt);
\filldraw (2.99329834299, 32.4715916008) circle (1pt);
\filldraw (3.55583077077, 70.3238134689) circle (1pt);
\filldraw (2.54143998755, 55.4722457077) circle (1pt);
\filldraw (3.03080050658, 45.6288115656) circle (1pt);
\filldraw (2.84764374411, 44.7014922513) circle (1pt);
\filldraw (3.08252574367, 71.1033401684) circle (1pt);
\filldraw (2.99312743759, 20.6628804861) circle (1pt);
\filldraw (3.02461917829, 36.0270888832) circle (1pt);
\filldraw (3.29902319005, 42.9821076876) circle (1pt);
\filldraw (2.28112205287, 31.4769093106) circle (1pt);
\filldraw (3.50358160432, 75.3963880881) circle (1pt);
\filldraw (2.46664046462, 58.5765433868) circle (1pt);
\filldraw (3.2792660596, 49.4766802618) circle (1pt);
\filldraw (3.5734069816, 59.2718593497) circle (1pt);
\filldraw (3.147558292, 61.3372818468) circle (1pt);
\filldraw (2.48626458371, 51.1514158686) circle (1pt);
\filldraw (2.38600923153, 47.4813384758) circle (1pt);
\filldraw (2.93470092347, 43.0669118331) circle (1pt);
\filldraw (2.32460514053, 31.4702628394) circle (1pt);
\filldraw (2.42436988427, 41.4917344084) circle (1pt);
\filldraw (1.99848079644, 41.8290479883) circle (1pt);
\filldraw (3.8181809271, 50.6216168559) circle (1pt);
\filldraw (3.21164546973, 27.721389452) circle (1pt);
\filldraw (3.51453311653, 68.1587194581) circle (1pt);
\filldraw (2.89090932861, 66.0973930627) circle (1pt);
\filldraw (2.77839963241, 43.732002064) circle (1pt);
\filldraw (3.31642587774, 38.3835599712) circle (1pt);
\filldraw (2.56327550259, 38.3801208995) circle (1pt);
\filldraw (2.15490305821, 47.8709463452) circle (1pt);
\filldraw (2.3265349552, 54.6228072829) circle (1pt);
\filldraw (2.68975099478, 46.0798930306) circle (1pt);
\filldraw (3.63496855861, 75.6925684136) circle (1pt);
\filldraw (2.42081116075, 45.6734283072) circle (1pt);
\filldraw (3.29978479372, 43.8101682011) circle (1pt);
\filldraw (2.90244527698, 55.7495336412) circle (1pt);
\filldraw (2.8827943808, 70.3114203958) circle (1pt);
\filldraw (3.66329162502, 46.1306416375) circle (1pt);
\filldraw (3.16545862029, 37.3471024029) circle (1pt);
\filldraw (2.30542466009, 60.902429135) circle (1pt);
\filldraw (2.76640025672, 57.1799207505) circle (1pt);
\filldraw (3.19037453367, 60.209702653) circle (1pt);
\filldraw (3.03900312698, 47.2435952623) circle (1pt);
\filldraw (2.74259996029, 69.4541735354) circle (1pt);
\filldraw (2.22802320985, 52.2807114336) circle (1pt);
\node[anchor=center] at (1.8353666626, 27.782516397) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-424-\imagesuffix.\imageext}};
\node[anchor=center] at (2.59706232542, 73.632615538) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-361-\imagesuffix.\imageext}};
\node[anchor=center] at (2.25339483575, -13.4319659131) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-360-\imagesuffix.\imageext}};
\node[anchor=center] at (1.80003102238, 47.2267642987) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-353-\imagesuffix.\imageext}};
\node[anchor=center] at (5.26196144276, 66.1390326526) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-327-\imagesuffix.\imageext}};
\node[anchor=center] at (3.19897276416, -19.2050523905) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-318-\imagesuffix.\imageext}};
\node[anchor=center] at (2.55817980652, -0.996517071338) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-278-\imagesuffix.\imageext}};
\node[anchor=center] at (1.77005378096, 38.8781057817) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-261-\imagesuffix.\imageext}};
\node[anchor=center] at (2.09939088309, -33.0966756306) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-260-\imagesuffix.\imageext}};
\node[anchor=center] at (4.16097711195, 25.6274307554) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-243-\imagesuffix.\imageext}};
\node[anchor=center] at (2.50015389174, 26.2322701122) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-232-\imagesuffix.\imageext}};
\node[anchor=center] at (3.92108468668, 79.5385949586) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-210-\imagesuffix.\imageext}};
\node[anchor=center] at (2.07336687642, 1.30580858494) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-190-\imagesuffix.\imageext}};
\node[anchor=center] at (3.82670737471, 38.6684059563) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-174-\imagesuffix.\imageext}};
\node[anchor=center] at (1.91589684645, 54.0971628897) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-151-\imagesuffix.\imageext}};
\node[anchor=center] at (2.44268144639, 14.0660463949) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-128-\imagesuffix.\imageext}};
\node[anchor=center] at (3.52946247878, 24.5465765588) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-115-\imagesuffix.\imageext}};
\node[anchor=center] at (3.99755164416, 49.8070142566) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-107-\imagesuffix.\imageext}};
\node[anchor=center] at (3.01819599917, -7.91787306008) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-104-\imagesuffix.\imageext}};
\node[anchor=center] at (3.3963129242, 78.1561920668) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-103-\imagesuffix.\imageext}};
\node[anchor=center] at (4.20607296229, 65.7609623739) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-86-\imagesuffix.\imageext}};
\node[anchor=center] at (2.10616200204, 19.6674171113) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-65-\imagesuffix.\imageext}};
\node[anchor=center] at (3.70627822306, 58.3489037053) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-39-\imagesuffix.\imageext}};
\node[anchor=center] at (4.54929565506, 52.8298213665) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-37-\imagesuffix.\imageext}};
\node[anchor=center] at (2.91881613935, 68.6462403173) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-14-\imagesuffix.\imageext}};
\node[anchor=center] at (3.31085967524, 58.0206097863) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-10-\imagesuffix.\imageext}};
\node[anchor=center] at (2.88819278941, 56.8875169124) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-5-\imagesuffix.\imageext}};
\node[anchor=center] at (2.53172975859, 54.8143051261) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-4-\imagesuffix.\imageext}};
\node[anchor=center] at (2.31766278875, 39.6272063277) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-0-\imagesuffix.\imageext}};

information_gains_Judd.tex
\filldraw (2.75628735815, 51.7778077114) circle (1pt);
\filldraw (2.24298312186, 51.9345279198) circle (1pt);
\filldraw (2.97801211785, 53.3270188878) circle (1pt);
\filldraw (2.34572990786, 47.5288515323) circle (1pt);
\filldraw (2.61244411197, 36.6802400069) circle (1pt);
\filldraw (2.58626052495, 32.9004325156) circle (1pt);
\filldraw (2.46729941717, 59.7789744244) circle (1pt);
\filldraw (2.30747631695, 41.6777382862) circle (1pt);
\filldraw (3.33078763402, 33.6166200558) circle (1pt);
\filldraw (2.91484561608, 44.0258370413) circle (1pt);
\filldraw (2.60360118347, 21.4792056624) circle (1pt);
\filldraw (2.66507773942, 54.7511791629) circle (1pt);
\filldraw (2.31997095442, 60.8323745741) circle (1pt);
\filldraw (3.65515491734, 44.3581322062) circle (1pt);
\filldraw (3.0329520141, 52.9031563658) circle (1pt);
\filldraw (2.95968348844, 1.18273322544) circle (1pt);
\filldraw (2.95155992666, 58.3111200576) circle (1pt);
\filldraw (3.22235926758, 43.0278198769) circle (1pt);
\filldraw (2.46373672668, 54.8962643601) circle (1pt);
\filldraw (2.66197679004, 50.3248274243) circle (1pt);
\filldraw (2.78988300487, 34.0752975391) circle (1pt);
\filldraw (3.07605779985, 29.337201874) circle (1pt);
\filldraw (2.41016676761, 6.08408214439) circle (1pt);
\filldraw (2.51526082509, 53.6352449875) circle (1pt);
\filldraw (3.81813052482, 84.1023629582) circle (1pt);
\filldraw (3.30915494738, 70.515231397) circle (1pt);
\filldraw (2.33948031515, 48.3213268203) circle (1pt);
\filldraw (2.8048642717, 52.2556897449) circle (1pt);
\filldraw (2.64395860481, 50.8854858845) circle (1pt);
\filldraw (3.63572265949, 68.6278862538) circle (1pt);
\filldraw (2.457339633, 20.8192945585) circle (1pt);
\filldraw (2.92782906242, 49.0443829995) circle (1pt);
\filldraw (3.15958971312, 34.7170484911) circle (1pt);
\filldraw (2.53905930651, 39.1499429608) circle (1pt);
\filldraw (2.67685357278, 44.942209554) circle (1pt);
\filldraw (3.17318514919, 45.084277923) circle (1pt);
\filldraw (1.90385760345, 32.5587046248) circle (1pt);
\filldraw (3.46555938588, 71.5309829074) circle (1pt);
\filldraw (2.57801224683, 40.5713411677) circle (1pt);
\filldraw (2.2767703124, 38.3737490743) circle (1pt);
\filldraw (3.12969729221, 27.6739024166) circle (1pt);
\filldraw (2.95895927482, 49.8354636793) circle (1pt);
\filldraw (2.70467463589, 46.0120149144) circle (1pt);
\filldraw (2.80848882487, 38.5616884201) circle (1pt);
\filldraw (3.04135161212, 47.8970304601) circle (1pt);
\filldraw (2.97251555438, 39.1068879402) circle (1pt);
\filldraw (2.88106941369, 34.8518383257) circle (1pt);
\filldraw (2.24093688711, 38.6808894954) circle (1pt);
\filldraw (2.65369274586, 48.8871838261) circle (1pt);
\filldraw (2.80917533892, 46.9629391868) circle (1pt);
\filldraw (2.85036608977, 41.2275048429) circle (1pt);
\filldraw (2.45270722533, 35.2824077622) circle (1pt);
\filldraw (3.09639755938, 67.4081426761) circle (1pt);
\filldraw (2.88591605328, 42.4626698155) circle (1pt);
\filldraw (2.60982746578, 30.8699037146) circle (1pt);
\filldraw (2.21117352229, 33.0949682823) circle (1pt);
\filldraw (2.09361467566, 40.2387197215) circle (1pt);
\filldraw (2.53027408135, 47.6351682076) circle (1pt);
\filldraw (2.55499748005, 53.6971241993) circle (1pt);
\filldraw (2.95135185707, 53.5530991857) circle (1pt);
\filldraw (2.7834354129, 61.1711489813) circle (1pt);
\filldraw (2.4971459385, 54.4813802597) circle (1pt);
\filldraw (2.56999572106, 58.9129611378) circle (1pt);
\filldraw (2.87800355161, 22.5800219804) circle (1pt);
\filldraw (3.47940927804, 55.301861853) circle (1pt);
\filldraw (2.96895017484, 54.3579054412) circle (1pt);
\filldraw (2.41327908521, 43.5329743112) circle (1pt);
\filldraw (2.56496598565, 36.3995042651) circle (1pt);
\filldraw (2.64653803204, 61.3114627673) circle (1pt);
\filldraw (3.11256125497, 49.6928068085) circle (1pt);
\filldraw (2.4912851486, 45.3516144113) circle (1pt);
\filldraw (2.85631184337, 51.2006319053) circle (1pt);
\filldraw (2.92914629726, 39.2286198175) circle (1pt);
\filldraw (2.79746594029, 60.5833186606) circle (1pt);
\filldraw (2.48143731709, 31.4268888869) circle (1pt);
\filldraw (3.9494449262, 58.138630814) circle (1pt);
\filldraw (3.0749608549, 56.1774300378) circle (1pt);
\filldraw (3.03872454688, 56.4048563975) circle (1pt);
\filldraw (3.35075873339, 48.5645565163) circle (1pt);
\filldraw (3.21927412194, 31.3117516443) circle (1pt);
\filldraw (2.70169877705, 57.1429216395) circle (1pt);
\filldraw (2.62709222423, 42.0062561991) circle (1pt);
\filldraw (2.1453917887, 43.850008421) circle (1pt);
\filldraw (2.23419042313, 48.6242165085) circle (1pt);
\filldraw (3.34322780157, 39.4575420454) circle (1pt);
\filldraw (3.72488817418, 55.7534492897) circle (1pt);
\filldraw (3.04448850223, 37.112477003) circle (1pt);
\filldraw (3.39107048598, 19.025279193) circle (1pt);
\filldraw (2.60316719083, 64.5335340152) circle (1pt);
\filldraw (3.1521985385, 45.6850880831) circle (1pt);
\filldraw (2.40548775918, 51.8225016792) circle (1pt);
\filldraw (2.34886881669, 50.9544424636) circle (1pt);
\filldraw (2.69636621937, 42.5726668782) circle (1pt);
\filldraw (2.47722364939, 28.0982288251) circle (1pt);
\filldraw (2.26843358119, 24.1124382609) circle (1pt);
\filldraw (2.48210412197, 57.3202796027) circle (1pt);
\filldraw (2.81407573955, 33.7183921122) circle (1pt);
\filldraw (3.33659759686, 63.4312365195) circle (1pt);
\filldraw (3.49161934471, 59.2902785122) circle (1pt);
\filldraw (2.84128625467, 20.2787038053) circle (1pt);
\filldraw (2.18652098541, 42.9635293201) circle (1pt);
\filldraw (2.28160163129, 24.0994843579) circle (1pt);
\filldraw (2.69800434151, 20.3806017694) circle (1pt);
\filldraw (3.19288271646, 48.5719687105) circle (1pt);
\filldraw (2.76297904171, 39.2731277165) circle (1pt);
\filldraw (2.68062227435, 50.7158139508) circle (1pt);
\filldraw (2.82153332929, 45.6071740306) circle (1pt);
\filldraw (3.11659150638, 17.5049571484) circle (1pt);
\filldraw (3.13468662555, 47.5831413983) circle (1pt);
\filldraw (3.54607667145, 51.3803306805) circle (1pt);
\filldraw (2.29647128477, 28.7711855957) circle (1pt);
\filldraw (2.38228508563, 44.4633605788) circle (1pt);
\filldraw (3.15435965733, 33.0185569999) circle (1pt);
\filldraw (2.83774912265, 50.6339479796) circle (1pt);
\filldraw (2.81871652392, 52.1147394385) circle (1pt);
\filldraw (2.82441855463, 60.8354599637) circle (1pt);
\filldraw (2.78150939745, 5.77170961451) circle (1pt);
\filldraw (3.90084227816, 58.736770005) circle (1pt);
\filldraw (2.76323718248, 23.6566978896) circle (1pt);
\filldraw (3.06804589243, 46.0931565326) circle (1pt);
\filldraw (3.30631134059, 27.8762717198) circle (1pt);
\filldraw (2.98492162199, 31.6470050827) circle (1pt);
\filldraw (2.75285773065, 53.37876822) circle (1pt);
\filldraw (2.68918909611, 51.3781383754) circle (1pt);
\filldraw (4.09781856656, 42.1239032523) circle (1pt);
\filldraw (2.6180121029, 37.7061193162) circle (1pt);
\filldraw (3.77904424789, 46.9769596376) circle (1pt);
\filldraw (2.95777341877, 50.9911110422) circle (1pt);
\filldraw (2.38934020862, 26.9326411724) circle (1pt);
\filldraw (3.17122607663, 45.4653284143) circle (1pt);
\filldraw (4.17575852741, 45.6451184543) circle (1pt);
\filldraw (2.49199406207, 24.2292918314) circle (1pt);
\filldraw (2.92435861971, 46.1658605973) circle (1pt);
\filldraw (3.10809272865, 34.7755814149) circle (1pt);
\filldraw (2.86020579265, 60.7200686357) circle (1pt);
\filldraw (3.26637437632, 37.663769776) circle (1pt);
\filldraw (3.2133085336, 23.2297528507) circle (1pt);
\filldraw (2.36901958624, 50.1686725666) circle (1pt);
\filldraw (2.92052275717, 52.5769909299) circle (1pt);
\filldraw (2.77900464806, 30.5733401403) circle (1pt);
\filldraw (2.29729335467, 39.1151571888) circle (1pt);
\filldraw (2.76635762228, 13.9116377156) circle (1pt);
\filldraw (3.76452829744, 69.9965118366) circle (1pt);
\filldraw (3.05335508826, 40.1606421726) circle (1pt);
\filldraw (3.22181726739, 49.5492072345) circle (1pt);
\filldraw (2.95637804486, 45.7062340876) circle (1pt);
\filldraw (2.61274754154, 56.6892688004) circle (1pt);
\filldraw (2.33694479808, 41.1136922606) circle (1pt);
\filldraw (3.50821376546, 57.4305396916) circle (1pt);
\filldraw (2.324765685, 46.033426571) circle (1pt);
\filldraw (2.5021883186, 50.5323938577) circle (1pt);
\filldraw (3.08781745451, 54.8722063944) circle (1pt);
\filldraw (2.43583394796, 61.3353797786) circle (1pt);
\filldraw (3.08916204695, 35.8934333506) circle (1pt);
\filldraw (2.55755129544, 52.1497972463) circle (1pt);
\filldraw (2.74417990566, 63.3683109156) circle (1pt);
\filldraw (2.44202478994, 29.5656872194) circle (1pt);
\filldraw (3.1795299594, 56.0184551476) circle (1pt);
\filldraw (2.32936807493, 52.2449028532) circle (1pt);
\filldraw (3.204625918, 55.00603313) circle (1pt);
\filldraw (3.24176822975, 24.8739584051) circle (1pt);
\filldraw (2.77567353362, 4.99458508444) circle (1pt);
\filldraw (2.46424967258, 14.3599224289) circle (1pt);
\filldraw (2.57849895356, 65.1988504923) circle (1pt);
\filldraw (3.53481742706, 53.8118454592) circle (1pt);
\filldraw (2.42944163452, -1.60217467825) circle (1pt);
\filldraw (2.35113463883, 49.1545969211) circle (1pt);
\filldraw (3.86309982107, 28.0645602087) circle (1pt);
\filldraw (3.71369348777, 57.4091157094) circle (1pt);
\filldraw (2.63934227042, 61.4444744496) circle (1pt);
\filldraw (2.702170052, -15.1975209832) circle (1pt);
\filldraw (2.76545639724, 47.9330703361) circle (1pt);
\filldraw (3.33962827315, 30.4712204137) circle (1pt);
\filldraw (2.92544900927, 68.8117257136) circle (1pt);
\filldraw (2.39583582869, 30.2753923096) circle (1pt);
\filldraw (2.74174749, 65.5986550941) circle (1pt);
\filldraw (2.69296637046, 51.2472126592) circle (1pt);
\filldraw (3.0855296822, 49.6578120618) circle (1pt);
\filldraw (2.11591504049, 30.8967087536) circle (1pt);
\filldraw (3.14852808913, 50.2686973668) circle (1pt);
\filldraw (2.79445584341, 58.2065389475) circle (1pt);
\filldraw (2.79599905623, 61.6125406359) circle (1pt);
\filldraw (1.85726770026, 52.1028897823) circle (1pt);
\filldraw (2.69981616984, 38.919138532) circle (1pt);
\filldraw (2.25890958952, 9.15035106925) circle (1pt);
\filldraw (2.2669950975, 29.6202320985) circle (1pt);
\filldraw (3.15540188569, 36.1017918764) circle (1pt);
\filldraw (2.50188485039, 31.1148455642) circle (1pt);
\filldraw (3.44037931922, 43.4989033514) circle (1pt);
\filldraw (2.92390830976, 42.9899387108) circle (1pt);
\filldraw (3.11674458299, 37.1623574899) circle (1pt);
\filldraw (2.36098134693, 39.1753352415) circle (1pt);
\filldraw (3.39673105057, 57.0565391307) circle (1pt);
\filldraw (2.79249615531, 54.9551451575) circle (1pt);
\filldraw (2.63144322244, 23.9138946701) circle (1pt);
\filldraw (2.79443179581, 24.1598960904) circle (1pt);
\filldraw (2.56705273233, 42.3431366818) circle (1pt);
\filldraw (2.6916933733, 40.9631134711) circle (1pt);
\filldraw (2.86363739968, 61.3656836345) circle (1pt);
\filldraw (2.80006180504, 40.1269259319) circle (1pt);
\filldraw (3.04818473348, 37.1070854196) circle (1pt);
\filldraw (2.77699908715, 59.0964275479) circle (1pt);
\filldraw (3.8015607397, 25.8048540135) circle (1pt);
\filldraw (2.48281194638, 21.4883539909) circle (1pt);
\filldraw (2.65659881831, 52.7375639101) circle (1pt);
\filldraw (2.30168222691, 45.9906515127) circle (1pt);
\filldraw (2.43436823546, 36.23687731) circle (1pt);
\filldraw (2.25662724973, 30.6401354568) circle (1pt);
\filldraw (2.57887876166, 28.6922349634) circle (1pt);
\filldraw (2.71654738539, 19.6453784618) circle (1pt);
\filldraw (3.06319590806, 47.1353698364) circle (1pt);
\filldraw (2.19869388505, 24.5126365799) circle (1pt);
\filldraw (3.55555396812, 68.1281850447) circle (1pt);
\filldraw (3.11704792023, 62.3193150891) circle (1pt);
\filldraw (3.2352143723, 51.0777280516) circle (1pt);
\filldraw (2.15334701869, 40.0365517736) circle (1pt);
\filldraw (2.79650190679, 58.4891789986) circle (1pt);
\filldraw (2.76732034248, 42.9495912199) circle (1pt);
\filldraw (2.86662875205, 55.3246579414) circle (1pt);
\filldraw (3.09855643238, 22.9615104155) circle (1pt);
\filldraw (3.35362812086, 42.8841104332) circle (1pt);
\filldraw (3.30158312847, 54.6879105053) circle (1pt);
\filldraw (2.41234898302, 59.539540513) circle (1pt);
\filldraw (3.04933414907, 36.5318913155) circle (1pt);
\filldraw (3.1514403271, 36.0581288968) circle (1pt);
\filldraw (2.49876679641, 25.0258207103) circle (1pt);
\filldraw (3.04317580738, 15.2064255262) circle (1pt);
\filldraw (3.98942212466, 56.6146473035) circle (1pt);
\filldraw (2.67003144792, -38.2016046207) circle (1pt);
\filldraw (2.57219326496, 43.3447993386) circle (1pt);
\filldraw (3.10064490939, 47.0680810651) circle (1pt);
\filldraw (2.89130020701, 44.3481413838) circle (1pt);
\filldraw (4.21482792059, 63.6749275978) circle (1pt);
\filldraw (2.98808396391, 56.1491502142) circle (1pt);
\filldraw (2.8667541359, 43.7989241969) circle (1pt);
\filldraw (2.38897425327, 51.8091993789) circle (1pt);
\filldraw (3.55701301457, 65.9712909132) circle (1pt);
\filldraw (2.90372417752, 28.2626916806) circle (1pt);
\filldraw (3.24352323956, 45.3958213531) circle (1pt);
\filldraw (3.0794856707, 42.4876650171) circle (1pt);
\filldraw (2.90682756203, 39.2761799412) circle (1pt);
\filldraw (2.6584328992, 42.50952501) circle (1pt);
\filldraw (3.02438462868, 63.8058035365) circle (1pt);
\filldraw (2.29298655025, 36.62654631) circle (1pt);
\filldraw (2.68213757045, 23.5176464008) circle (1pt);
\filldraw (3.45742828884, 48.6674363595) circle (1pt);
\filldraw (3.4943148838, 5.69646149549) circle (1pt);
\filldraw (3.41681721143, 42.811299521) circle (1pt);
\filldraw (2.51380355887, 50.9444309605) circle (1pt);
\filldraw (2.84805957288, 53.2655864637) circle (1pt);
\filldraw (1.9923127202, 49.1866597379) circle (1pt);
\filldraw (2.81408542949, 34.2609165091) circle (1pt);
\filldraw (2.89517648226, 25.5385261576) circle (1pt);
\filldraw (3.07466586169, 67.6197070729) circle (1pt);
\filldraw (2.53002213323, 44.2306390302) circle (1pt);
\filldraw (2.7127547372, 50.4998405144) circle (1pt);
\filldraw (2.53320257013, 20.8190790895) circle (1pt);
\filldraw (2.24148953834, -9.03829213326) circle (1pt);
\filldraw (2.31539909856, 42.7859668141) circle (1pt);
\filldraw (2.2689990907, 35.5201234688) circle (1pt);
\filldraw (2.63574670666, 55.009660465) circle (1pt);
\filldraw (3.46587805049, 67.6626898858) circle (1pt);
\filldraw (3.00742348204, 32.2870190042) circle (1pt);
\filldraw (3.69600466214, 63.6290229318) circle (1pt);
\filldraw (2.8779700098, 15.0901834556) circle (1pt);
\filldraw (2.48161775132, 53.8471888823) circle (1pt);
\filldraw (2.13691668521, 47.1510754199) circle (1pt);
\filldraw (1.85360670891, 1.39319697026) circle (1pt);
\filldraw (3.02445201106, 54.1627661176) circle (1pt);
\filldraw (2.84908920095, 28.0833760851) circle (1pt);
\filldraw (3.54775233476, 20.9237117271) circle (1pt);
\filldraw (3.71148900205, 43.3220131388) circle (1pt);
\filldraw (2.92452587279, 69.7598789247) circle (1pt);
\filldraw (2.67757696734, -0.630042696579) circle (1pt);
\filldraw (2.04051854342, 40.8525132262) circle (1pt);
\filldraw (3.231479136, 68.2313975678) circle (1pt);
\filldraw (2.68650596684, 43.1258937153) circle (1pt);
\filldraw (3.29891944332, 63.4146059814) circle (1pt);
\filldraw (2.79971815649, 20.2455629768) circle (1pt);
\filldraw (2.96533058946, 46.9477287802) circle (1pt);
\filldraw (2.54957640485, 49.482089502) circle (1pt);
\filldraw (2.70499912965, 55.5638414958) circle (1pt);
\filldraw (2.50093315963, 27.2236129204) circle (1pt);
\filldraw (3.06235061038, 60.5887330348) circle (1pt);
\filldraw (1.96131531408, 40.480448114) circle (1pt);
\filldraw (2.79866322861, -21.3906081122) circle (1pt);
\filldraw (2.63714362481, 18.4815674149) circle (1pt);
\filldraw (3.28693598115, 37.5476373251) circle (1pt);
\filldraw (3.90878239968, 59.7737845676) circle (1pt);
\filldraw (2.76491782628, 42.2719693947) circle (1pt);
\filldraw (2.50879054867, 42.5304976422) circle (1pt);
\filldraw (2.31029140575, 54.0187247885) circle (1pt);
\filldraw (3.20079429964, 44.4956190337) circle (1pt);
\filldraw (3.57633033988, 50.5102952834) circle (1pt);
\filldraw (2.29268959165, 37.7444022889) circle (1pt);
\filldraw (2.94967417629, 52.197844384) circle (1pt);
\filldraw (3.00894124182, 32.7284340226) circle (1pt);
\filldraw (2.73384957096, 29.0318333575) circle (1pt);
\filldraw (1.97096492885, 27.0894404925) circle (1pt);
\filldraw (3.01912063727, 53.9583888333) circle (1pt);
\filldraw (2.33943210156, 61.8518228136) circle (1pt);
\filldraw (2.83421691085, 25.2247218128) circle (1pt);
\filldraw (2.37405008767, 17.3517332792) circle (1pt);
\filldraw (3.84481353269, 61.5239120388) circle (1pt);
\filldraw (3.24120424267, 50.6388051588) circle (1pt);
\filldraw (2.61527779345, 23.3705946918) circle (1pt);
\filldraw (2.61885074379, 51.0212834268) circle (1pt);
\filldraw (2.83347546389, 57.6007391895) circle (1pt);
\filldraw (2.59875621755, 43.7406486778) circle (1pt);
\filldraw (2.41199812649, 54.236021244) circle (1pt);
\filldraw (2.27786707343, 27.5874466855) circle (1pt);
\filldraw (2.75118687756, 52.9046203057) circle (1pt);
\filldraw (2.35543509058, 48.2090891376) circle (1pt);
\filldraw (3.23648097861, 65.5535512155) circle (1pt);
\filldraw (2.40787106924, 58.8758204983) circle (1pt);
\filldraw (2.06017594745, 37.2626522912) circle (1pt);
\filldraw (3.62416199961, 48.6560103766) circle (1pt);
\filldraw (2.9873891576, 57.4737883817) circle (1pt);
\filldraw (2.29672095846, 40.8282404556) circle (1pt);
\filldraw (2.14627504496, 39.7355921944) circle (1pt);
\filldraw (3.39428630244, 16.1498946879) circle (1pt);
\filldraw (3.28408801105, 59.7802900813) circle (1pt);
\filldraw (4.02850252866, 56.1808536085) circle (1pt);
\filldraw (2.66156506891, -7.32005452297) circle (1pt);
\filldraw (3.69840764902, 45.4213876458) circle (1pt);
\filldraw (2.94640229489, 54.1147962154) circle (1pt);
\filldraw (3.52114417998, 45.9757963595) circle (1pt);
\filldraw (2.14410382816, 36.4800581434) circle (1pt);
\filldraw (2.55189276993, 60.9715991309) circle (1pt);
\filldraw (2.80588338969, 46.8584538547) circle (1pt);
\filldraw (2.90245948241, 59.2814487384) circle (1pt);
\filldraw (2.91971582302, 27.7752298434) circle (1pt);
\filldraw (3.54045553507, 60.8030188412) circle (1pt);
\filldraw (2.65424295239, 44.8822864909) circle (1pt);
\filldraw (3.55316057098, 80.8900242337) circle (1pt);
\filldraw (2.91511499028, 30.5349275458) circle (1pt);
\filldraw (3.32732512581, 60.7818439737) circle (1pt);
\filldraw (3.13031258211, 45.1521316739) circle (1pt);
\filldraw (3.20871215834, 27.6601494552) circle (1pt);
\filldraw (2.23984975581, 46.957661259) circle (1pt);
\filldraw (3.25168069769, 37.9188155378) circle (1pt);
\filldraw (2.89974684568, 48.7814990998) circle (1pt);
\filldraw (3.8256890776, 45.0067502997) circle (1pt);
\filldraw (3.23682278301, 62.4201670283) circle (1pt);
\filldraw (3.92853399209, 60.1933073239) circle (1pt);
\filldraw (2.89262444449, 61.0924148107) circle (1pt);
\filldraw (2.51864538702, 34.2988267055) circle (1pt);
\filldraw (3.31607704486, 53.2469690417) circle (1pt);
\filldraw (3.50537624892, 60.1737907212) circle (1pt);
\filldraw (2.74735410631, 41.5922043901) circle (1pt);
\filldraw (2.76922081749, 22.785923174) circle (1pt);
\filldraw (2.157402006, 34.0507944119) circle (1pt);
\filldraw (2.92782237247, 51.1627055724) circle (1pt);
\filldraw (2.02181383895, 10.1716131535) circle (1pt);
\filldraw (2.82431838692, 38.9321434923) circle (1pt);
\filldraw (1.84653816245, 35.4203642181) circle (1pt);
\filldraw (3.19106479257, 50.8641231134) circle (1pt);
\filldraw (2.17453489689, 22.3806408826) circle (1pt);
\filldraw (2.53573459029, 0.731642284453) circle (1pt);
\filldraw (3.35851749561, 51.205318918) circle (1pt);
\filldraw (2.97496102155, 24.7505556208) circle (1pt);
\filldraw (2.71942153331, 44.1355963595) circle (1pt);
\filldraw (2.31381755548, 36.0075629974) circle (1pt);
\filldraw (2.26876792855, 54.9227426118) circle (1pt);
\filldraw (3.05539507234, 64.9032817803) circle (1pt);
\filldraw (2.99308552961, 15.1057155948) circle (1pt);
\filldraw (3.1746101815, 53.4287627221) circle (1pt);
\filldraw (3.21119479014, 26.1221268455) circle (1pt);
\filldraw (4.0132934313, 58.1820313609) circle (1pt);
\filldraw (3.21488227676, 37.4810228834) circle (1pt);
\filldraw (2.97330203063, 19.7722523619) circle (1pt);
\filldraw (3.06202662121, 45.8555497707) circle (1pt);
\filldraw (2.73538672009, 5.73339944234) circle (1pt);
\filldraw (2.35335472434, 52.4045817156) circle (1pt);
\filldraw (3.04164239987, 50.7761675406) circle (1pt);
\filldraw (2.29477976596, 44.4222262775) circle (1pt);
\filldraw (2.12387012284, 42.1575300774) circle (1pt);
\filldraw (2.16160112439, 51.8430339519) circle (1pt);
\filldraw (2.30100862588, 35.1495393998) circle (1pt);
\filldraw (2.89637599973, 24.1415126688) circle (1pt);
\filldraw (2.61887914155, 50.5287542355) circle (1pt);
\filldraw (2.55414320729, 13.9098947361) circle (1pt);
\filldraw (3.3623696642, 45.5157078877) circle (1pt);
\filldraw (2.48912437378, 56.0914415892) circle (1pt);
\filldraw (3.27781042045, 63.7399422432) circle (1pt);
\filldraw (2.73181144779, 37.1120081269) circle (1pt);
\filldraw (3.52252679333, 35.6192471459) circle (1pt);
\filldraw (2.68183313451, 49.4774665547) circle (1pt);
\filldraw (2.56079590474, 48.4249798214) circle (1pt);
\filldraw (2.43338356017, 30.6953080244) circle (1pt);
\filldraw (2.99329834299, 25.8915417946) circle (1pt);
\filldraw (3.55583077077, 62.506197298) circle (1pt);
\filldraw (2.54143998755, 45.6960824592) circle (1pt);
\filldraw (3.03080050658, 61.8223206117) circle (1pt);
\filldraw (2.84764374411, 32.3797367079) circle (1pt);
\filldraw (3.08252574367, 54.30124633) circle (1pt);
\filldraw (2.99312743759, -2.37598280113) circle (1pt);
\filldraw (3.02461917829, 30.5738154779) circle (1pt);
\filldraw (3.29902319005, 21.2357162379) circle (1pt);
\filldraw (2.28112205287, 21.0463032378) circle (1pt);
\filldraw (3.50358160432, 71.7018892236) circle (1pt);
\filldraw (2.46664046462, 52.3208083699) circle (1pt);
\filldraw (3.2792660596, 43.9352182384) circle (1pt);
\filldraw (3.5734069816, 30.6389684662) circle (1pt);
\filldraw (3.147558292, 71.1403642784) circle (1pt);
\filldraw (2.48626458371, 36.7815519455) circle (1pt);
\filldraw (2.38600923153, 41.0930771814) circle (1pt);
\filldraw (2.93470092347, 30.4498024949) circle (1pt);
\filldraw (2.32460514053, 19.5785781028) circle (1pt);
\filldraw (2.42436988427, 41.2764272945) circle (1pt);
\filldraw (1.99848079644, 34.7185954423) circle (1pt);
\filldraw (3.8181809271, 41.203109065) circle (1pt);
\filldraw (3.21164546973, 37.198829161) circle (1pt);
\filldraw (3.51453311653, 61.3888881545) circle (1pt);
\filldraw (2.89090932861, 68.3311832807) circle (1pt);
\filldraw (2.77839963241, 49.7901216749) circle (1pt);
\filldraw (3.31642587774, 42.8681458386) circle (1pt);
\filldraw (2.56327550259, 50.9100824366) circle (1pt);
\filldraw (2.15490305821, 34.718470707) circle (1pt);
\filldraw (2.3265349552, 57.1358310957) circle (1pt);
\filldraw (2.68975099478, 45.3349538845) circle (1pt);
\filldraw (3.63496855861, 59.4135147108) circle (1pt);
\filldraw (2.42081116075, 46.202886139) circle (1pt);
\filldraw (3.29978479372, 39.89808016) circle (1pt);
\filldraw (2.90244527698, 55.8105345934) circle (1pt);
\filldraw (2.8827943808, 65.9095429431) circle (1pt);
\filldraw (3.66329162502, 33.6858545129) circle (1pt);
\filldraw (3.16545862029, 32.7654507412) circle (1pt);
\filldraw (2.30542466009, 52.5361747416) circle (1pt);
\filldraw (2.76640025672, 50.3682923712) circle (1pt);
\filldraw (3.19037453367, 66.9546682254) circle (1pt);
\filldraw (3.03900312698, 33.237598339) circle (1pt);
\filldraw (2.74259996029, 63.9774213793) circle (1pt);
\filldraw (2.22802320985, 53.5097376861) circle (1pt);
\node[anchor=center] at (1.8353666626, 37.8474861957) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-424-\imagesuffix.\imageext}};
\node[anchor=center] at (2.59706232542, 61.5330411455) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-361-\imagesuffix.\imageext}};
\node[anchor=center] at (2.25339483575, -3.98864867671) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-360-\imagesuffix.\imageext}};
\node[anchor=center] at (1.80003102238, 46.002064304) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-353-\imagesuffix.\imageext}};
\node[anchor=center] at (5.26196144276, 88.7433393301) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-327-\imagesuffix.\imageext}};
\node[anchor=center] at (3.19897276416, -7.53444286166) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-318-\imagesuffix.\imageext}};
\node[anchor=center] at (2.55817980652, 16.4313384265) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-278-\imagesuffix.\imageext}};
\node[anchor=center] at (1.77005378096, 41.2165232998) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-261-\imagesuffix.\imageext}};
\node[anchor=center] at (2.09939088309, 2.92356136148) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-260-\imagesuffix.\imageext}};
\node[anchor=center] at (4.16097711195, 42.099307114) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-243-\imagesuffix.\imageext}};
\node[anchor=center] at (2.50015389174, -17.276230832) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-232-\imagesuffix.\imageext}};
\node[anchor=center] at (3.92108468668, 68.3812157154) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-210-\imagesuffix.\imageext}};
\node[anchor=center] at (2.07336687642, -15.2368622023) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-190-\imagesuffix.\imageext}};
\node[anchor=center] at (3.82670737471, 18.8483111854) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-174-\imagesuffix.\imageext}};
\node[anchor=center] at (1.91589684645, 37.3667764851) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-151-\imagesuffix.\imageext}};
\node[anchor=center] at (2.44268144639, 49.9439026367) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-128-\imagesuffix.\imageext}};
\node[anchor=center] at (3.52946247878, 20.7362289314) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-115-\imagesuffix.\imageext}};
\node[anchor=center] at (3.99755164416, 42.343650431) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-107-\imagesuffix.\imageext}};
\node[anchor=center] at (3.01819599917, -19.2296312063) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-104-\imagesuffix.\imageext}};
\node[anchor=center] at (3.3963129242, 64.3807797132) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-103-\imagesuffix.\imageext}};
\node[anchor=center] at (4.20607296229, 82.1539161289) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-86-\imagesuffix.\imageext}};
\node[anchor=center] at (2.10616200204, 24.491145643) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-65-\imagesuffix.\imageext}};
\node[anchor=center] at (3.70627822306, 76.6481289338) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-39-\imagesuffix.\imageext}};
\node[anchor=center] at (4.54929565506, 67.6557906467) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-37-\imagesuffix.\imageext}};
\node[anchor=center] at (2.91881613935, 67.3418896109) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-14-\imagesuffix.\imageext}};
\node[anchor=center] at (3.31085967524, 55.5106176272) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-10-\imagesuffix.\imageext}};
\node[anchor=center] at (2.88819278941, 50.2471223975) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-5-\imagesuffix.\imageext}};
\node[anchor=center] at (2.53172975859, 56.4218695397) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-4-\imagesuffix.\imageext}};
\node[anchor=center] at (2.31766278875, 31.799099256) {\includegraphics[width=\pwidth\tikzunit]{information_gains/dataset/\imageprefix-0-\imagesuffix.\imageext}};

modeling_temporal.pgf.tex
\ifdefined\noktikz
  \input{figures/model_colors}
\else
  \usetikzlibrary{calc}
  \usetikzlibrary{shapes}
  \graphicspath{{/kyb/agmb/mkuemmerer/Documents/Uni/Bethge/Saliency/TPAMI/figures/}}

  \input{/kyb/agmb/mkuemmerer/Documents/Uni/Bethge/Saliency/TPAMI/figures/model_colors}
\fi


\newcommand\loglik[1]{\textsf{$\mathsf{#1}$\,bit/fix}}
\newcommand\auc[1]{\textsf{$\mathsf{#1}$\,\%}}
%\newcommand\imgwidth{5cm}
%\newlength\imgwidth
%\setlength\imgwidth{5cm}
%\newlength\abstikzunit
%\setlength\abstikzunit{1cm}

%\newcommand\scalingfactor{0.8}
%\newlength\tikzunit
%\setlength\tikzunit{\scalingfactor\abstikzunit}

\ifdefined\abstikzunit
	\setlength\abstikzunit{1cm}
	\newcommand\scalingfactor{0.8}
	\newcommand\imgwidth{5}
	\setlength\tikzunit{\scalingfactor\abstikzunit}
\else
	\newlength\abstikzunit
	\setlength\abstikzunit{1cm}
	\newlength\tikzunit
	\newcommand\scalingfactor{0.8}
	\newcommand\imgwidth{5}
	\setlength\tikzunit{\scalingfactor\abstikzunit}
\fi


\begin{tikzpicture}[scale=\scalingfactor,font=\sffamily]
  \definecolor{fillcolor}{rgb}{0.9, 0.9, 0.9};
  \tikzset{boxstyle/.style={fill=fillcolor, rounded corners=2mm}};

  % Data left column

  \coordinate (leftcolwidth) at (6.7,0);
  \coordinate (leftcolheight) at (0,-1.2);
  \newcommand{\leftcolimgheight}{1.0}

  \coordinate (leftcolsize) at ($ (leftcolwidth) + (leftcolheight) $);

  \coordinate (leftcolsep) at (0, -0.75);
  \coordinate (leftcolop) at ($ (leftcolheight) + 0.5*(leftcolsep) + 0.5*(leftcolwidth) $);

  \coordinate (lefttotalcolheight) at ($ (leftcolheight) + (leftcolsep) $);

  \coordinate (leftcolimage) at (0.2,-0.1);
  \coordinate (leftcoltext) at ($ 0.5*(leftcolheight) + (1.5,0) $);

  \coordinate (leftcoltemporalsep) at (0,-0.1);

  \coordinate (col2) at (5.9, 0);
  \coordinate (row2) at (0, -4.2);
  \coordinate (label_rel_subfig) at (-0.6, 0);

  % Data right column

  \coordinate (rightcol) at ($ (leftcolwidth) + (1,0) $);
  \newcommand\rightcolwidth{9}

  \coordinate (rightcolheight) at (0,-6);

  \node[anchor=north west] at ($ (label_rel_subfig) + (-0.2, 0) $) {a)};

  % Blur

  \path[draw, boxstyle, fill=colorblur]
    ($ 0*(lefttotalcolheight) $) rectangle ($  0*(lefttotalcolheight) + (leftcolsize) $);
  \node[anchor=north west, inner sep=0] at ($ 0*(lefttotalcolheight) + (leftcolimage) $)
    {\includegraphics[height=\leftcolimgheight\tikzunit]{saliencymap_nonlinearity_blur_centerbias.png}};
  \node[anchor=west] at ($ 0*(lefttotalcolheight) + (leftcoltext) $) {Stationary model};

  \draw[->,thick] ($ (leftcolheight) + 0.5*(leftcolwidth) + (0, -0.1) $) -- +($ (leftcolsep) + (0, 0.2) $);


  %\draw[thick, gray] ($ 0*(lefttotalcolheight) +(leftcolheight) + 0.35*(leftcolsep) $) -- +(leftcolwidth) ;

%  \node at ($ 3*(lefttotalcolheight) + (leftcoltemporalsep) + (leftcolop) $) {+};

  % Temporal  effects

  \path[draw, boxstyle, fill=colorexcitation]
    ($ 1*(lefttotalcolheight) + (leftcoltemporalsep) $) rectangle +($ (leftcolsize) $);
  \node[anchor=north west, inner sep=0] at ($ 1*(lefttotalcolheight) + (leftcoltemporalsep) + (leftcolimage) $)
    {\includegraphics[height=\leftcolimgheight\tikzunit]{self_excitation_symbol.pdf}};
  \node[anchor=west] at ($ 1*(lefttotalcolheight) + (leftcoltemporalsep) + (leftcoltext) $) {Self excitation};

  %\node at ($ 1*(lefttotalcolheight) + (leftcoltemporalsep) + (leftcolop) $) {+};

  \draw[->,thick] ($ 1*(lefttotalcolheight) + (leftcolheight) + (leftcoltemporalsep) + 0.5*(leftcolwidth) + (0, -0.1) $) -- +($ (leftcolsep) + (0, 0.2) $);


  % Temporal  effects

  \path[draw, boxstyle, fill=colorexcentric]
    ($ 2*(lefttotalcolheight) + (leftcoltemporalsep) $) rectangle +(leftcolsize);
  \node[anchor=north west, inner sep=0] at ($ 2*(lefttotalcolheight) + (leftcoltemporalsep) + (leftcolimage) $)
    {\includegraphics[height=\leftcolimgheight\tikzunit]{excentric_self_excitation_symbol.pdf}};
  \node[anchor=west] at ($ 2*(lefttotalcolheight) + (leftcoltemporalsep) + (leftcoltext) $) {Elliptical self excitation};


  %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  %%%% Right column

  \node[anchor=north west] at ($ (rightcol) + (label_rel_subfig) $) {b)};

  \node[anchor=north west] at ($ (rightcol) $)
   {\includegraphics[width=\rightcolwidth\tikzunit]%
     {log_likelihoods_with_IoR_rel_centerbias_JuddEtAl2009Cor.pdf}};



%  \path[draw, boxstyle] ($ (leftcolheight) + (leftcolsep) $)
%       rectangle  ($ (leftcolheight) + (leftcolsep) + (leftcolwidth) + (leftcolheight) $);



\end{tikzpicture}

% Clean up
\let\abstikzunit\undef
\let\scalingfactor\undef
\let\imgwidth\undef


auc_problems.pgf.tex
\ifdefined\noktikz
  \input{figures/auc_examples}
\else
  \usetikzlibrary{calc}
  \usetikzlibrary{shapes}
  \graphicspath{{/kyb/agmb/mkuemmerer/Documents/Uni/Bethge/Saliency/TPAMI/figures/}}

  \input{/kyb/agmb/mkuemmerer/Documents/Uni/Bethge/Saliency/TPAMI/figures/auc_examples}
\fi


\newcommand\loglik[1]{\textsf{$\mathsf{#1}$\,bits/fix}}
\newcommand\auc[1]{\textsf{$\mathsf{#1}$\,\%}}

%\newlength\imgwidth
%\setlength\imgwidth{4cm}

\ifdefined\abstikzunit
\else
	\newlength\abstikzunit
	\newlength\tikzunit
\fi

\setlength\abstikzunit{1cm}
\newcommand\scalingfactor{0.6}
\newcommand\imgwidth{5}
\setlength\tikzunit{\scalingfactor\abstikzunit}


\setlength\abstikzunit{1cm}
\setlength\tikzunit{\scalingfactor\abstikzunit}


%\newcommand\imgwidth{5cm}
%\newcommand\imgwidth{4cm}

\begin{tikzpicture}[scale=\scalingfactor,font=\small\sffamily]
  \coordinate (col2) at (5.5, 0);
  \coordinate (row2) at (0, -4);
  \coordinate (label_rel_fig) at (-4.5,3.5);
  \coordinate (ll_rel) at (2.5,-2);

  \node (labela) at (label_rel_fig) {\textsf{a)}};

  \node (modelpredictions) at ($ (col2) + (0, 4.3) $) {Model predictions};
  \node[rotate=90] (fixations) at ($ (row2) + (-5.7, 0) $) {Fixations};

  \node[anchor=north east] (fix_high)  at (0,0)
    {\includegraphics[width=\imgwidth\tikzunit]{fixations_high_contrast.pdf}};
  \node[anchor=north east] (fix_low)  at (row2)
    {\includegraphics[width=\imgwidth\tikzunit]{fixations_low_contrast.pdf}};

  \node[anchor=south west] (fix_high)  at (0,0)
    {\includegraphics[width=\imgwidth\tikzunit]{saliencymap_high_contrast.pdf}};

  \node[anchor=south west] (fix_high)  at (col2)
    {\includegraphics[width=\imgwidth\tikzunit]{saliencymap_low_contrast.pdf}};

  \node[rectangle split, rectangle split parts=2, anchor=center] at (ll_rel) {\loglik\contrastHighFixHighSmapLoglikelihood\nodepart{second}\auc\contrastHighFixHighSmapAUC};

  \node[rectangle split, rectangle split parts=2, anchor=center] at ($ (col2) + (ll_rel) $) {\loglik\contrastHighFixLowSmapLoglikelihood\nodepart{second}\auc\contrastHighFixLowSmapAUC};

  \node[rectangle split, rectangle split parts=2, anchor=center] at ($ (row2) + (ll_rel) $) {\loglik\contrastLowFixHighSmapLoglikelihood\nodepart{second}\auc\contrastLowFixHighSmapAUC};

  \node[rectangle split, rectangle split parts=2, anchor=center] at ($ (row2) + (col2) + (ll_rel) $) {\loglik\contrastLowFixLowSmapLoglikelihood\nodepart{second}\auc\contrastLowFixLowSmapAUC};

  % hline
  \coordinate (fig2) at (0,-13);
  \coordinate (hline_rel_fig2) at (0,4.5);

  \draw ($ (fig2) - (col2) + (hline_rel_fig2) $) -- ($ (fig2) +(hline_rel_fig2) + 2*(col2) $);

  % Figure2: Ordering


  \node (labela) at ($ (fig2) + (label_rel_fig) $) {\textsf{b)}};

  \node[anchor=north east] (fix)  at ($(fig2) + (0,0) $)
    {\includegraphics[width=\imgwidth\tikzunit]{fixations_ordering.pdf}};

  \node[anchor=south west] (sal_good)  at ($ (fig2)  $)
    {\includegraphics[width=\imgwidth\tikzunit]{saliencymap_ordering_good.pdf}};

  \node[anchor=south west] (sal_bad)  at ($ (fig2) + (col2) $)
    {\includegraphics[width=\imgwidth\tikzunit]{saliencymap_ordering_bad.pdf}};


  \node[rectangle split, rectangle split parts=2, anchor=center] at ($ (fig2)  + (ll_rel) $) {\loglik\orderingGoodLoglikelihood\nodepart{second}\auc\orderingGoodAUC};

  \node[rectangle split, rectangle split parts=2, anchor=center] at ($ (fig2) + (col2) + (ll_rel) $) {\loglik\orderingBadLoglikelihood\nodepart{second}\auc\orderingBadAUC};



  % colorbar

  \coordinate (colorbar_label_correction) at (0, 0.2);

  \node[anchor=south west] (colorbar)  at ($ (fig2) +(row2) +(col2)$)
    {\includegraphics[width=\imgwidth\tikzunit]{auc_examples_colorbar.pdf}};


  \node[anchor=north west] at ($ (colorbar.south west) + (colorbar_label_correction) $)
    {\textsf{\tiny{low density}}};

  \node[anchor=north east] at ($ (colorbar.south east) + (colorbar_label_correction) $)
    {\textsf{\tiny{high density}}};

\end{tikzpicture}

% Clean up
\let\abstikzunit\undef
\let\scalingfactor\undef
\let\imgwidth\undef
\let\tikzunit\undef

auc_examples.tex

\newcommand{\contrastHighFixHighSmapLoglikelihood}{2.93}
\newcommand{\contrastHighFixLowSmapLoglikelihood}{0.08}
\newcommand{\contrastLowFixHighSmapLoglikelihood}{-1.82}
\newcommand{\contrastLowFixLowSmapLoglikelihood}{0.00}

\newcommand{\contrastHighFixHighSmapAUC}{86}
\newcommand{\contrastHighFixLowSmapAUC}{86}
\newcommand{\contrastLowFixHighSmapAUC}{50}
\newcommand{\contrastLowFixLowSmapAUC}{50}

\newcommand{\orderingGoodLoglikelihood}{2.23}
\newcommand{\orderingBadLoglikelihood}{0.77}

\newcommand{\orderingGoodAUC}{81}
\newcommand{\orderingBadAUC}{85}


kl_divergence.pgf.tex
\ifdefined\noktikz
%  \input{figures/auc_examples}
\else
  \usetikzlibrary{calc}
  \usetikzlibrary{shapes}
  \graphicspath{{/kyb/agmb/mkuemmerer/Documents/Uni/Bethge/Saliency/TPAMI/figures/}}

%  \input{/kyb/agmb/mkuemmerer/Documents/Uni/Bethge/Saliency/TPAMI/figures/auc_examples}
\fi


\ifdefined\abstikzunit
\else
    \newlength\abstikzunit
    \newlength\tikzunit
\fi

\setlength\abstikzunit{1cm}
\newcommand\scalingfactor{1.3}
\newcommand\imgwidth{5.3}
\newcommand\smallimgwidth{2.5}
\setlength\tikzunit{\scalingfactor\abstikzunit}


\setlength\abstikzunit{1cm}
\setlength\tikzunit{\scalingfactor\abstikzunit}


\begin{tikzpicture}[scale=\scalingfactor]
  \coordinate (col2) at (5.6, 0);
  \coordinate (row2) at (0, -4.6);

  \coordinate (smallcol2) at (2.9, 0);
  \coordinate (smallrow2) at (0, -2.1);


  \coordinate (label_rel_subfig) at (-0.45, -0.0);
  \coordinate (label_rel_subfig2) at (-0.1, -0.0);

%  \node (labela) at (label_rel_fig) {\textsf{a)}};

  \node[anchor=north west] at (0,0)
    {\includegraphics[width=\smallimgwidth\tikzunit]{fixation_based_kl_divergence_saliency_map.pdf}};

  \node[anchor=north west] at (smallcol2)
    {\includegraphics[width=\smallimgwidth\tikzunit]{fixation_based_kl_divergence_negative_saliency_map.pdf}};

  \node[anchor=north west] at (smallrow2)
    {\includegraphics[width=\smallimgwidth\tikzunit]{fixation_based_kl_divergence_binned_saliency_map.pdf}};

  \node[anchor=north west] at ($ (smallrow2) + (smallcol2) $)
    {\includegraphics[width=\smallimgwidth\tikzunit]{fixation_based_kl_divergence_shuffled_saliency_map.pdf}};


%   \node[anchor=north west] at (label_rel_subfig) {\textsf{a)}};
% 
% 
%   \node[anchor=north west] at (col2)
%     {\includegraphics[width=\imgwidth\tikzunit]{fixation_based_kl_divergence_bin_dependency.pdf}};
%   \node[anchor=north west] at ($ (col2) + (label_rel_subfig2) $) {\textsf{b)}};
% 
% 
%   \node[anchor=north west] at ($ 2*(col2) $)
%     {\includegraphics[width=\imgwidth\tikzunit]{ellr_estimators.pdf}};
%   \node[anchor=north west] at ($ 2*(col2) + (label_rel_subfig2) $) {\textsf{c)}};

\end{tikzpicture}

% Clean up
\let\abstikzunit\undef
\let\scalingfactor\undef
\let\imgwidth\undef
\let\tikzunit\undef

other_measures.pgf.tex
\ifdefined\noktikz
%  \input{figures/auc_examples}
\else
  \usetikzlibrary{calc}
  \usetikzlibrary{shapes}
  \graphicspath{{/kyb/agmb/mkuemmerer/Documents/Uni/Bethge/Saliency/TPAMI/figures/}}

%  \input{/kyb/agmb/mkuemmerer/Documents/Uni/Bethge/Saliency/TPAMI/figures/auc_examples}
\fi


\ifdefined\abstikzunit
\else
    \newlength\abstikzunit
    \newlength\tikzunit
\fi

\setlength\abstikzunit{1cm}
\newcommand\scalingfactor{1}
\newcommand\imgwidth{8.5}
\newcommand\smallimgwidth{2.5}
\setlength\tikzunit{\scalingfactor\abstikzunit}


\setlength\abstikzunit{1cm}
\setlength\tikzunit{\scalingfactor\abstikzunit}


\begin{tikzpicture}[scale=\scalingfactor]
  \coordinate (col2) at (8, 0);
  \coordinate (row2) at (0, -4.6);

  \coordinate (smallcol2) at (2.9, 0);
  \coordinate (smallrow2) at (0, -2.1);


  \coordinate (label_rel_subfig) at (-0.0, -0.4);
  \coordinate (label_rel_subfig2) at (label_rel_subfig);% (-0.1, -0.4);

%  \node (labela) at (label_rel_fig) {\textsf{a)}};

  \node[anchor=north west] at (0,0)
    {\includegraphics[width=\imgwidth\tikzunit]{other_measures_no_fitting_scatter.pdf}};



  \node[anchor=north west] at (label_rel_subfig) {\textsf{a)}};


  \node[anchor=north west] at (col2)
    {\includegraphics[width=\imgwidth\tikzunit]{other_measures_scatter.pdf}};
  \node[anchor=north west] at ($ (col2) + (label_rel_subfig2) $) {\textsf{b)}};

\end{tikzpicture}

% Clean up
\let\abstikzunit\undef
\let\scalingfactor\undef
\let\imgwidth\undef
\let\tikzunit\undef

temporal_effects.pgf.tex
\ifdefined\noktikz
\else
  \usetikzlibrary{calc}
  \usetikzlibrary{shapes}
  \graphicspath{{/kyb/agmb/mkuemmerer/Documents/Uni/Bethge/Saliency/TPAMI/figures/}{/home/matthias/Documents/Uni/Bethge/Saliency/TPAMI/figures/}}

\fi


\newcommand\loglik[1]{\textsf{$\mathsf{#1}$\,bit/fix}}
\newcommand\auc[1]{\textsf{$\mathsf{#1}$\,\%}}


\ifdefined\abstikzunit
\else
        \newlength\abstikzunit
        \newlength\tikzunit
\fi
        \setlength\abstikzunit{1cm}
        \newcommand\scalingfactor{0.4}
        \newcommand\imgwidth{5}
        \setlength\tikzunit{\scalingfactor\abstikzunit}


\begin{tikzpicture}[scale=\scalingfactor,font=\tiny\sffamily,every node/.style={inner sep=0,outer sep=0}]
  \coordinate (center) at (5.45, 0);
  \coordinate (col2) at (5.9, 0);
  \coordinate (row2) at (0, -4.6);
  \coordinate (row3) at ($ 2*(row2) + (0, 0) $);
  \coordinate (horiz_text_rel_row) at  (center);
  \coordinate (vertical_text_rel_col) at  (-0.1, -2);

  \node[anchor=south,rotate=90] at (vertical_text_rel_col) {spatial density};  

  \node[anchor=north] (orig)  at (center)
    {\includegraphics[width=\imgwidth\tikzunit]{inhibition_of_return_before.pdf}};


  \node at ($  (2.5, 0.4) $) {inhibition of return};  
  \node at ($ (col2) + (2.5, 0.4) $) {self excitation};  

  \node[anchor=south,rotate=90] at ($ (row2) + (vertical_text_rel_col)$) {correcting factor};  



  \node[anchor=north west] (inhib_factor)  at (row2)
    {\includegraphics[width=\imgwidth\tikzunit]{inhibition_of_return_factor.pdf}};

  \node[anchor=north west] (excitation_factor)  at ($ (row2) + (col2) $)
    {\includegraphics[width=\imgwidth\tikzunit]{self_excitation_factor.pdf}};


  \node[anchor=south,rotate=90] at ($ (row3) + (vertical_text_rel_col) $) {final density};  

  \node[anchor=north west] (inhib_after)  at (row3)
    {\includegraphics[width=\imgwidth\tikzunit]{inhibition_of_return_after.pdf}};

  \node[anchor=north west] (excitation_after)  at ($ (row3) + (col2) $)
    {\includegraphics[width=\imgwidth\tikzunit]{self_excitation_after.pdf}};

  \draw[->] (orig.south) -- (inhib_factor.north);
  \draw[->] (orig.south) -- (excitation_factor.north);

  \draw[->] (inhib_factor.south) -- (inhib_after.north);
  \draw[->] (excitation_factor.south) -- (excitation_after.north);

\end{tikzpicture}

% Clean up to make including this file tidy
\let\abstikzunit\undef
\let\scalingfactor\undef
\let\imgwidth\undef
\let\tikzunit\undef

kl_divergence_table.tex
%\newcommand{\mycite}[1]{\cite{#1}: \citetitle{#1}}
\newcommand{\mycite}[1]{\cite{#1}}

\begin{tabular}{p{1cm}p{3cm}p{8cm}}
Paper & KL-Divergence & estimate of true distribution \\ \hline
%%% Fixation based

% Bayesian surprise attracts human attention.
\mycite{itti2005bayesian}
    & fixation-based
    & \\

% A principled approach to detecting surprising events in video
\mycite{itti2005principled}
    & fixation-based
    & \\

\mycite{itti2005quantifying}
    & fixation-based
    & \\

% Attention: Bits versus Wows
\mycite{baldi2005attention}
    & fixation-based
    & \\

 %SUN: A Bayesian framework for saliency using natural statistics
\mycite{Zhang2008} 
    & fixation-based
    & \\ 

% Saliency, attention, and visual search: An information theoretic approach
\mycite{Bruce2009} 
    & fixation-based
    & \\


% Of bits and wows: A Bayesian theory of surprise with applications to attention.
\mycite{baldi2010bits}
    & fixation-based
    & \\

% Measuring visual saliency by Site Entropy Rate
\mycite{wang2010measuring}
    & fixation-based
    & \\


\mycite{Borji2013}
    & fixation based
    & \\
\mycite{Borji2013b}
    & fixation-based
    & \\
\mycite{Borji2013d}
    & fixation-based
    & \\
% The contribution of low-level features at the centre of gaze to
%  saccade target selection
%\mycite{Dorr2009}
%    & ???
%    & ``To assess how closely the original distribution of saccade length
%and direction was approximated, we computed the Kullback–Leibler divergence between the original distribution and those generated by the baseline conditions. For a reference point, we also computed the KLD of one half of the original data set to the other half. Results were 1.19, 0.4, 0.08, 0.07, and 0.05, respectively (for ``random'', ``scrambled'', ``lengths'', ``synthetic'', and ``original'').''\\
%
%% Variability of eye movements when viewing dynamic natural scenes.
%\mycite{Dorr2010} 
%    & They cite \mycite{Tatler2005} who uses image-based but use NSS (the use leave-one-out
%      cross-validation to quantify variablity, but no hyperparameter optimization).
%    & Gaussian with radius $1.2^\circ$\\


%%% Image based



% Point-of-gaze analysis reveals visual search strategies
\mycite{rajashekar2004point}
    & image-based
    %, symmetric
    & Gaussian kernel, width of fovea\\


% Visual correlates of fixation selection: effects of scale and time
\mycite{Tatler2005} 
    & image-based
    & 2d-histograms, bins of $2^\circ \times 2^\circ$ and $10^{-5}$ added as prior\\

% Predicting visual fixations
% on video based on low-level visual features
\mycite{lemeur2007predicting}
    & image-based
    & precision of the eye tracking.\\

% Measures and limits of models of fixation selection
\mycite{Wilming2011} 
    & image-based
    & Gaussian with $2^\circ$, motivated by fovea + eye tracker.\\

\mycite{lemeur2013methods}
    & image-based, fixation based
    & Gaussian kernel density estimate, kernel size $1^\circ$ of visual angle.\\
    


% Saliency and Human Fixations: State-of-the-Art and Study of
%  Comparison Metrics
\mycite{Riche2013}
    & image-based
    & not stated\\

% Spatial statistics and attentional dynamics in scene viewing
\mycite{Engbert2014} 
    & image-based
    & ``kernel-density estimates with bandwidth parameters chosen according to Scott's rule''\\






%% Visual search in noise: revealing the influence of structural cues by gaze-contingent classification image analysis.
%\mycite{rajashekar2006visual}
%    & none
%    & n.a.\\




%\mycite{chen2010preattentive}
%    & TODO: understand
%    & \\


\end{tabular}

saliency_manuscript_v2.tex
\RequirePackage{snapshot}
\documentclass[11pt, a4paper, twocolumn]{article}
\pdfoutput=1

%\documentclass[11pt,a4paper]{article}
%\usepackage[marginparsep=0cm,marginparwidth=7cm,left=20pt,top=90pt,textwidth=12.5cm,textheight=22cm]{geometry}


\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{dsfont}
\usepackage{amssymb}
\usepackage[per-mode = symbol]{siunitx}
\usepackage{hyperref}
\usepackage{graphicx}
\usepackage{longtable}
\usepackage{todonotes}

%\usepackage{showframe}
\bibliographystyle{abbrv}

% Make figure captions small:
\usepackage[font=small]{caption}

%to make the pgf imports work in ktikz
\newcommand*{\noktikz}{}
\usepackage{tikz}
\usetikzlibrary{calc}
\usetikzlibrary{shapes}



\graphicspath{{figures/}}

\newcommand\mitodo[1]{\todo[inline,author=Matthias]{#1}}
\newcommand\mtodo[1]{\todo{Matthias: #1}}
\newcommand\titodo[1]{\todo[inline,author=Tom]{#1}}
\newcommand\ttodo[1]{\todo{Tom: #1}}

\newcommand\EE{\mathds{E}}
\newcommand\RR{\mathds{R}}
%\DeclareMathOperator{\DKL}{\mathrm{D}_{\mathrm{KL}}}
\DeclareMathOperator{\DKL}{D_{KL}}
\newcommand\pfix{p_\text{fix}}
\newcommand\pnonfix{p_\text{nonfix}}
\newcommand\xfix{x_\text{fix}}
\newcommand\xnonfix{x_\text{nonfix}}


\DeclareSIUnit\fix{fix}
\DeclareSIUnit\bits{bits}

\include{figures/data}
\include{figures/data_kienzle}

\author{Matthias Kümmerer, Thomas Wallis, Matthias Bethge}
\title{How close are we to understanding image-based saliency?}


\begin{document}
\maketitle

\abstract{
Within the set of the many complex factors driving gaze placement, the properities of an image that are associated with fixations under ``free viewing'' conditions have been studied extensively.
There is a general impression that the field is close to understanding this particular association. 
Here we frame saliency models probabalistically as point processes, allowing the calculation of log-likelihoods and bringing saliency evaluation into the domain of information.
We compared the information gain of state-of-the-art models to a gold standard and find that only one third of the explainable spatial information is captured.
We additionally provide a principled method to show where and how models fail to capture information in the fixations.
Thus, contrary to previous assertions, purely spatial saliency remains a significant challenge.

\textbf{Keywords}: saliency | visual attention | information theory | eye movements | likelihood | point processes | model comparison
}


\section{Introduction}

The properties of an image that attract fixations during free viewing are said to be \textit{salient}, and models that attempt to capture these image features are saliency models (Figure \ref{fig:fixations_saliency_point_processes}a and b). 
Predicting where people fixate is relevant to both understanding visual information processing in biological systems and also to applications in computer vision and engineering. 
Beginning with the influential model of \cite{Itti1998}, there are now over 50 models of saliency as well as around 10 models that seek to incorporate top-down effects (see \cite{Borji2013, Borji2013b, Borji2013d} for recent reviews and analyses of this extensive literature).
There exists a general impression that these models have captured most of the association between image properties and fixation structure under free viewing conditions \cite{Einhauser2010, Borji2013b}.
For example, \cite{Einhauser2010} write that ``Recent elaborations of such stimulus-driven models are now approaching the limits imposed by intersubject variability''.
But how can we assess this question in a quantitatively precise way?


\begin{figure}
  \center
  \include{figures/intro_figure.pgf}
  \caption{
  \small
Fixations, saliency and point processes.
\textbf{(a)} Sample image with fixations from several subjects (black points) and the fixation train of one specific subject (red lines).
\textbf{(b)} Saliency map produced by the AIM model for the same image.
Red and blue denote higher and lower saliency respectively.
\textbf{(c)} The fixation train from (a) represented as a sample of a spatiotemporal point process.
Note that spatiotemporal point processes take into account both the spatial position and inter-fixation durations.
}
\label{fig:fixations_saliency_point_processes}
\end{figure}


To judge our level of understanding, we must measure the distance between our model predictions and some upper bound of possible prediction performance using an interpretable scale.
The field is currently drowning in model comparison metrics, indicating uncertainty about what the right measure is.
For example, a recent paper \cite{Riche2013} included a comparison of 12 metrics with the recommendation that researchers use three of them to avoid the pitfalls of any one.
Following this recommendation would mean that measuring saliency and comparing saliency models is inherently ambiguous, since it is impossible to define a unique ranking if any two of the considered rankings are inconsistent.
Thus, one could never answer the question we pose.
Since no comparison of existing metrics can resolve this issue, we instead advocate a return to first principles.

Saliency is operationalised by measuring fixation densities. 
How well a model predicts these densities can be directly assessed using information theory.
As originally shown by Shannon \cite{Shannon1949}, information theory provides a unique and universally accepted measure of information gain from three basic axioms.
\textit{Information gain} is defined as the entropic difference between the prior and the posterior distribution.
In the context of image-based saliency, this quantifies the reduction in uncertainty (intuitively, the scatter of predicted fixations) about where people look.

To reiterate, understanding saliency is the \textit{distance} between model predictions and a gold standard.
By one of Shannon's axioms, information is a linear scale.
This means we can simply take the difference between a model and the gold standard to judge our level of understanding, making information gain distinct from any other metric.


%This difference score tells us how much saliency models understand about the spatial structure of fixations, compared to what they could know.

%(see \cite{Wilming2011,Riche2013} and Appendix \ref{sec:app_other_metrics}).


%The most prominent metric is the \textit{AUC}: the area under a curve of model hit rate against false positive rate for each threshold.
%It is equivalent to the performance in a two-alternative forced-choice (2AFC) task where the model is ``presented'' with two image locations: one at which an observer fixated and another from a nonfixation distribution.
%The thresholded saliency value is the model's decision, and the percent correct of the model in this task across all possible thresholds is the AUC score.
%
%To reiterate, understanding is the \textit{distance} between model predictions and a gold standard.
%Unfortunately, the AUC does not provide a useful measure of distance because the meaning of AUC differences depends on the absolute value of performance, which has an upper bound of 1.
%For example, it is not the case that increasing AUC from 0.6 to 0.7 is the same gain in knowledge as increasing from 0.9 to 1.
%We therefore require a measure of distance that is linear, in that differences along the number line mean the same thing no matter where they fall on the line.

%Information theory provides such a metric.
%\textit{Information} is usually expressed in terms of the difference to some baseline model (\textit{information gain}), which tells you how much more information your model knows about the data than the baseline.
%Because information is a linear scale, we can also simply take the difference between a model and the gold standard to judge our level of understanding.
%This difference score tells us how much state-of-the-art saliency models understand about the spatial structure of fixations, compared to what they could know.

To evaluate saliency models in terms of information we must use a probabalistic model.
It is natural to consider fixations and saliency as a probabalistic process, because humans do not make the same fixations as each other, or as themselves if viewing an image twice.
Probabalistic models can distinguish the inherent stochasticity of the fixation paths from uncertainty in the predictions. 
As recognised and applied by \cite{Barthelme2013}, an ideal framework to consider fixation distributions probabalistically is that of spatial point processes: the class of probability distributions of sets of points in space and time.


For probabalistic models, information gain is computed as the difference in average log-likelihood\footnote{Strictly, information gain refers to the information difference between prior and posterior; here we include the information difference between baseline and image-based saliency-model}.
While average log-likelihood can be rewritten in terms of Kullback-Leibler divergence (KL-divergence), our approach is fundamentally different from how KL-divergence has previously been used to compare saliency models.
Most importantly, we point out that average log-likelihoods are not invariant under reparameterization of the scale used to measure saliency.
Therefore, we emphasize that it is important to find the optimal scale for measuring saliency rather than just relying on its rank.
The theoretical underpinnings of our approach and its relationship to the KL-divergence measures used previously are discussed in detail in the Appendix (\ref{sec:kl_divergence}).

%For probabalistic models, information gain is computed as the difference in average log-likelihood.
%It is important to note that log-likelihood is related to, but distinct from, another commonly-used evaluation metric: Kullback-Leibler divergence.
%In theory, the difference between these metrics is negligible; in practice and depending on the actual estimators used, there can be significant differences between the outcomes.
%These practical differences in estimating information are discussed in the Appendix (\ref{sec:kl_divergence}).

Here we evaluate saliency models using log-likelihood within a point process framework and compare each model's information gain relative to a baseline model to a gold standard.
Contrary to the impression that image-based saliency is essentially solved, our results show that current state-of-the-art models capture only one third of the explainable image-based spatial information in fixation locations.


\section{Methods}

\subsection{Point processes and log-likelihoods}
\label{sec:point_processes_methods}

We define a \textit{fixation train} as a three-dimensional point process.
A fixation train consists of $N$ fixations with positions $x_i$, $y_i$, $t_i$, where $x_i$ and $y_i$ denote the spatial position of the fixation in the image and $t_i$ denotes the time of the fixation (see Figure \ref{fig:fixations_saliency_point_processes}c).
Conceiving of fixation trains as 3D point processes allows us to model the joint probability distribution of all fixations of a subject on an image.
In general, a model's log-likelihood is $\frac1N\sum_k \log p(x_k)$, where $p$ is the probability function of the model and $x_k$, $k=1, \dots, N$ are samples from the probabilistic process that we would like to model.
Our likelihoods are therefore of the form
\[
  p(x_1, y_1, t_1, \dots, x_N, y_N, t_N, N),
\]
where $N$ is part of the data distribution, not a fixed parameter.
By chain rule, this is decomposed into conditional likelihoods

\begin{align*}
  &p(x_1, y_1, t_1, \dots, x_N, y_N, t_N, N)\\
  &\quad =p(N)\prod_{i=1}^N p(x_i, y_i, t_i
%  &\qquad
  \mid N, x_1, y_1, t_1,
 \\ &\qquad \qquad \qquad \qquad
   \dots, x_{i-1}, y_{i-1}, t_{i-1})
\end{align*}


The above holds true for any three-dimensional point process.
We make the additional assumption that the conditional likelihoods do not depend on $N$\footnote{
To be precise, we condition on $N \geq i$, which the notation $p(x_i, y_i, t_i \mid x_1, y_1, t_1, \dots, x_{i-1}, y_{i-1}, t_{i-1})$ already implies.
Therefore the condition $N \geq i$ is omitted by abuse of notation.
}.
Furthermore, we assume that all models account for the factor $p(N)$ in the same way.
In the calculation of the log-likelihood, this gives rise to an additive constant that is the same for each model.
Since we are only interested in differences of log-likelihoods, we can omit it completely.

Here we use the logarithm to base two, meaning that the log-likelihoods we report are in bits.
If one model's log-likelihood exceeds another model's log-likelihood by 1~bit, the first model assigns on average double the likelihood to the data: it predicts the data twice as well.
Model comparison within the framework of likelihoods is well-defined and the standard of any statistical model comparison enterprise.

\subsection{A definition of ``image-based saliency''}
\label{sec:saliency_definition}

Image-based saliency is usually considered as the properties of the image that are associated with fixation selection, independent of image-independent factors like centre bias.
However, a precise definition of this term is lacking.
Now that we have defined a probabalistic framework for saliency, we can provide this precise definition because the influences of image-independent factors can be divided out.
This is not possible as long as saliency maps use only a range scale (i.e. are invariant under monotonic transformations).

We define the image-based saliency $s(x,y \mid I)$ for an image $I$ to be

\[
  s(x, y \mid I) = \frac{p(x, y \mid I)}{p(x,y)},
\]

where $p(x,y \mid I)$ is the fixation distribution of this image while $p(x,y)$ is the image independent prior distribution.
Consequently, the prediction of image-based saliency for a model $\hat p(x, y \mid I)$ is $\hat s(x,y \mid I) = \frac{\hat p(x,y \mid I)}{p(x,y)}$.
This definition of image-based saliency has the property of having the best possible shuffled AUC (area under the receiver operating characteristic curve; \cite{Barthelme2013}).

%% Might have to be moved later
\begin{figure*}[!ht]
  \center
  \include{figures/modeling_stationary.pgf}

  \caption{
  \small
Modelling procedure and results. 
\textbf{(a)} For each saliency model we optimised three spatial, image-independent factors that increased the complexity of each model: a pointwise nonlinearity, the centre bias and the blur kernel.
\textbf{(b)} Log likelihood (bits/fixation) for each model considered relative to a null model which assumes uniform fixation density over the image. 
The total bar height represents the optimised performance of each model; the coloured sections show the relative contributions of each model factor from (a).
The baseline model is a nonparametric model of all fixations from other images (thus capturing purely image-independent spatial biases).
The gold standard model is the (Gaussian blurred) empirical fixation distributions for all other observers for each image: it represents all the spatial structure of fixations that can be captured (image-dependent information).
\textbf{(c)} Fully-optimised model performance, expressed as a percentage of the possible information gain from baseline (0) to gold standard (100).
The best performing model explains only \SI{\BestModelPercentOfGoldRelBaseline}{\percent}.
}
\label{fig:modelling_and_results}
\end{figure*}

\subsection{Upper and lower bounds}

In evaluating the performance of a model we would like to know the best performance a model \textit{could possibly} have given some constraints, so that we can better judge how well a model does.
We created three baseline models to facilitate comparison of the saliency models.

The \textbf{maximum entropy model} assumes that fixations are uniformly distributed over the image (i.\,e. the gaze density is constant).
All log-likelihoods are calculated relative to this model.
Likelihoods greater than zero therefore represent an improvement (in bits/fixation) over the prediction performance of constant gaze density.

The \textbf{lower-bound model} is a 2D histogram model crossvalidated \textit{between} images (trained on all fixations for all observers on all other images).
Thus, it captures the image-independent spatial information in the fixations, and in this paper is used as an estimate of $p_{prior}$. 
Bincount and regularization parameters were optimised by gridsearch.
If a saliency model captured all the behavioural fixation biases but nothing about what causes parts of an image to attract fixations, it would do as well as the lower-bound model.

Fixation preferences that are inconsistent between observers are by definition unpredictable from fixations alone.
If we have no additional knowledge about inter-observer differences, the best predictor of an observer's fixation pattern on a given image is therefore to average the fixation patterns from all other observers and add regularization. 
This is our \textbf{gold standard model}. 
It was created by blurring the fixations with a Gaussian kernel, learned by 10-fold crossvalidation between subjects.
It accounts for the amount of information in the spatial structure of fixations to a given image that can be explained while averaging over the biases of individual observers.
This model is the upper bound on prediction in the dataset (see \cite{Wilming2011} for a thorough comparison of this gold standard and other upper bounds capturing different constraints).


\subsection{Saliency models}

We converted a range of influencial saliency map models into point processes and calculated their likelihoods, relative to the maximum entropy model. 
We included all models from the saliency benchmark in \cite{Judd2012} and the top performing models that have been added later on the web page belonging to this benchmark\footnote{\url{http://saliency.mit.edu/}} up to July 2014. 
For all models, the original source code and default parameters have been used unless stated otherwise. 
The included models are \textbf{Itti \& Koch}
\cite{Itti1998} (here, two implementations have been used: The one from
the Saliency Toolbox\footnote{\url{http://www.saliencytoolbox.net}} and the
variant specified in the GBVS paper),
%
\textbf{Torralba}\footnote{\url{http://people.csail.mit.edu/tjudd/SaliencyBenchmark/Code/torralbaSaliency.m}}
\cite{Torralba2006},
%
\textbf{GBVS}\footnote{\url{http://www.klab.caltech.edu/~harel/share/gbvs.php}} \cite{Harel2006},
%
\textbf{SUN} \cite{Zhang2008} (we used a scale parameter of $0.64$, corresponding to the pixel size of $2.3^\prime$ of visual angle of the dataset used to learn the filters. This model will be called ``SUN, orig'' in the following. Additionally, we did a grid search over the scale parameter. This resulted in a scale parameter of $0.15$ for the model called ``SUN, optim''),
%
\textbf{Kienzle}\footnote{Code provided by Simon Barthelm\'{e}}\cite{Kienzle2007a, Kienzle2009} (with a patch size of $195$px corresponding to the optimal patchsize of $5.4^\circ$ reported by them).
%
\textbf{Hou \&
Zhang}\footnote{\url{http://www.klab.caltech.edu/~xhou/projects/spectralResidual/spectralresidual.html}}
\cite{Hou2007},
%
\textbf{AIM}\footnote{\url{http://www-sop.inria.fr/members/Neil.Bruce/}}
\cite{Bruce2009},
%
\textbf{Judd}\footnote{\url{http://people.csail.mit.edu/tjudd/WherePeopleLook/index.html}}
\cite{Judd2009},
%
\textbf{Context-Aware
saliency}\footnote{\url{http://webee.technion.ac.il/labs/cgm/Computer-Graphics-Multimedia/Software/Saliency/Saliency.html}}
\cite{Goferman2010, Goferman2012},
%
\textbf{CovSal}\footnote{\url{http://web.cs.hacettepe.edu.tr/~erkut/projects/CovSal/}} \cite{Erdem2013},
%(we used two variants of the model, the \textit{CovariancesOnly} type and the \textit{SigmaPoints} type), %% Now only sigmapoints
%
\textbf{RARE2012}\footnote{\url{http://www.tcts.fpms.ac.be/attention/?categorie17/rare2012}}\cite{Riche2013a},
%
Boolean Map-based Saliency \textbf{(BMS)}\footnote{\url{http://cs-people.bu.edu/jmzhang/BMS/BMS.html}},
\cite{Zhang2013, Zhang2014}
 and finally
%
the Ensemble of Deep Networks \textbf{(eDN)}\footnote{\url{http://github.com/coxlab/edn-cvpr2014}}
\cite{Vig2014}.


\subsubsection{Conversion into point processes}

The models we consider above are all explicitly spatial: they do not include any temporal dependencies.
Therefore, here the general point process formula simplifies via marginalisation over time.
Each model was converted into a point process by treating the normalized saliency map as conditional gaze density for the next fixation (notation from Section \ref{sec:point_processes_methods}; $s(x,y)$ denotes the saliency at point $(x, y)$):

\begin{align*}
&p(x_j, y_j, t_j \mid (x_0, y_0, t_0) \dots (x_{j-1}, y_{j-1}, t_{j-1}))\\
  &\qquad= p(x_j, y_j, t_j)\\
  &\qquad = p(x_j, y_j)p(t_j)\\
  &\qquad \propto s(x_j, y_j)
\end{align*}

The first equality expresses the fact that the saliency models assume independence of the previous fixation history; the second equality states that they also do not take the actual time of the fixations into account.

Since many of these models were optimised for AUC, and since AUC is invariant to monotonic transformations whereas log-likelihood is not, simply comparing the models' raw saliency map would not be fair.
The saliency map for each model was therefore transformed by a \textit{pointwise monotonic nonlinearity} that was optimised to give the best log-likelihood for that model (see Figure \ref{fig:modelling_and_results}a).
This corresponds to picking the model with the best log-likelihood from all models that are equivalent (under AUC) to the original model.

Every saliency map was jointly rescaled to range from 0 to 1 (i.e. over all images at once, not per image, keeping contrast changes from image to image intact).
The pointwise monontonic nonlinearity was then applied.
This nonlinearity was modeled by a pointwise linear and continous function that was parametrized as a continuous piecewise linear function supported in 20 equidistant points $x_i$ between $0$ and $1$ with values $y_i$ with $0 \leq x_0 \leq \dots \leq x_{19}$:
$p_{\text{nonlin}}(x, y) \propto f_{\text{nonlin}}(s(x, y))$ with

\[
  f_{\text{nonlin}}(x) = \frac{y_{i+1}-y_{i}}{x_{i+1}-x_{i}} (x-x_i)+y_i
\]
for $x_i \leq x \leq x_{i+1}$.

\subsubsection{Including additional factors}

We iteratively increased the complexity of the basic saliency-as-point-process model (described above) by adding components that included additional factors (see Figure \ref{fig:modelling_and_results}a).
First, we optimised a \textit{centre bias} term that accounts for the fact that human observers tend to look towards the centre of the screen.
Second, we optimised a \textit{blur} term that compensates for models that make overly-precise, confident predictions of fixation locations \cite{Judd2012}.

The center bias was modeled as 

\begin{align*}
&p_{\text{centerbias}}(x,y) \\
&\qquad \propto f_{\text{centerbias}}(d(x,y))p_{\text{nonlin}}(x,y)
\end{align*}

Here, $d(x,y) = \sqrt{(x-x_{\text{c}})^2+\alpha(y-y_{\text{c}})^2}/d_{\text{max}}$ is the normalized distance of $(x,y)$ to the center of the image $(x_\text{c}, y_\text{c})$ with eccentricity $\alpha$, $f_{\text{centerbias}}(d)$ is again a continuous piecewise linear function that was fitted in 12 points.

The blurring was modeled by blurring $s(x, y)$ with a Gaussian kernel with radius $\sigma$ before the nonlinearity and the center bias were applied.

\subsubsection{Optimization}

For each case of spatial optimization described above (nonlinearity, nonlinearity+centerbias, nonlinearity+centerbias+blur), all parameters were optimized jointly using the L-BFGS SLSQP algorithm from \texttt{scipy.optimize} \cite{Scipy}. 

\subsubsection{Dataset}

We evaluated model log-likelihoods using a subset of the MIT-1003 fixation dataset \cite{Judd2009}. 
To better estimate the nonparametric baseline model we used only the most common image size (1024 $\times$ 768), resulting in 463 images included in the evaluation. 
We also compared model performance in the full dataset of Kienzle \cite{Kienzle2009} (see Discussion).


\section{Results}

\subsection{Whole database performance}

Figure \ref{fig:modelling_and_results}b shows the results of the model fitting procedure for spatial point processes using the Judd dataset \cite{Judd2009}.
The total bar heights are the optimised performance of each model (in bits/fixation) relative to the mutual entropy null model.

The gold standard model shows that the total mutual information between the image and the spatial structure of the fixations amounts to \SI{\MutualInformationImageSpatial}{\bits\per\fix}.
To give another intuition for this number, a model that would for every fixation always correctly predict the quadrant of the image in which it falls would also have a likelihood of \SI{2}{\bits\per\fix}. 

The lower-bound model is able to explain \SI{\BaselinePerformance}{\bits\per\fix} of this mutual information.
That is, \SI{\BaselinePercentOfGold}{\percent} of the information in spatial fixation distributions can be accounted for by behavioural biases (e.g. the bias of human observers to look at the centre of the image).

The \BestModel\ model performs best of all the saliency models compared, with \SI{\BestModelPerformance}{\bits\per\fix}, capturing \SI{\BestModelPercentOfGold}{\percent} of the total mutual information.
It accounts for \SI{\BaselineToBestPercentOfGold}{\percent} more than the lower-bound model, or \SI{\BestModelPercentOfGoldRelBaseline}{\percent} of the possible information gain (\SI{\PossibleInformationGain}{\bits\per\fix}) between baseline and gold standard.
Of the approximately \SI{\BestModelPerformance}{\bits\per\fix} captured by the \BestModel\ model, \SI{\BestModelNonlinearityPercent}{\percent} of this is due to the optimised nonlinearity, \SI{\BestModelNonlinearityToCenterbiasPercent}{\percent} is added by the centre bias, and \SI{\BestModelCenterbiasToBlurPercent}{\percent} by spatial blurring.

Considering only model performance (i.e. without also including centre bias and blur factors; the pink sections in Figure \ref{fig:modelling_and_results}b) shows that many of the models perform worse than the lower-bound model.
This means that the centre bias is more important than the portion of image-based saliency that these models do capture \cite{Tatler2005}.

Readers will also note that the centre bias and blurring factors account for very little of the performance of the Judd model relative to most other models.
This is because the Judd model already includes a centre bias that is optimised for the Judd dataset.



\subsection{Image-level comparisons}

\begin{figure*}[!ht]
  \center
  \include{figures/information_gains_schema.pgf}
  \caption{Calculation of information gain in the image space.
  The first column shows an example image.
  Hypothetical fixation densities of the gold standard (``true'') and model predictions are shown in the second column.
  These are divided by the baseline model (prior) to get the image-based saliency map.
  Both saliency maps are then log-transformed and multiplied by the gold standard density to calculate the information gain in the image space.
  Subtracting the gold standard information gain from the model's information gain yields a difference map of the possible information gain: that is, 
  where and by how much the model's predictions fail.
  In this case, the model overestimates (blue contours) the fixation density in the left (red) spot in the image, underestimates (red contours) the centre bias and predicts the right-most (yellow) spot almost perfectly.
  }
\end{figure*}


\begin{figure*}
   \center
   \include{figures/information_gains.pgf}
  
\caption{
Information gain in the image space.
\textbf{(a)} The left column displays the image for which the eDN model explains most of the information gain.
The second column shows the model density (where the model expects fixations to be), for the gold standard, eDN, BMS and AIM in each row.
The third column shows the ratio of the model density to the baseline density (where the model believes the image-based saliency to be).
The fourth column shows the information gain of the model relative to the baseline.
Contours are equally spaced and consistent over the column (more dense contour lines indicate a peak in information gain).
The fifth column shows how this differs from the possible information gain estimated by the gold standard.
The log-likelihood of the model for each image is shown to the right of the fifth column.
\textbf{(b)} Same as (a) for the image on which the AIM and BMS models perform best.
\textbf{(c)} Same as (a) for the image on which the AIM and BMS models perform worst.
}
\label{fig:information_gain_images}
\end{figure*}

\begin{figure*}[!ht]

  \input{figures/information_gains_scatter_combined.pgf}

  \caption{
    Distribution of information gains and explained information over
    all images in the dataset. 
    Each black dot represents an image from the dataset. 
    For some example cases, in the left column
    the images themselves are shown. In the right column for these cases
    the information gain difference to the gold standard is shown.
    These plots allow a modeller to assess the performance of a model on all images in the dataset.
    In the lower right of the plot are images where a lot of information could be explained but is not; these are where the model could be best improved.
  }
  \label{fig:information_gain_scatter}
\end{figure*}


One major advantage of using a probabalistic framework rather than AUC is that the information gain of a model can be compared with the gold standard in the image space.
Here we visualise where the information that saliency models fail to capture is located in several images.

To choose example images in a principled way, we computed the images for which the best model (\BestModel) explained most and least of the possible information gain. 
We additionally included the image that the two second and third ranking models (BMS and AIM) performed worst on (this happened to be the same image).

First, we plot the model density for each model (second column in Figure \ref{fig:information_gain_images}). 
This is $\hat p(x,y \mid I)$, and shows where the model expects fixations to occur in the given image.
Then we plot the model's prediction of image-based saliency (see above) in the third column of Figure \ref{fig:information_gain_images}.
It tells us where and how much the model believes the fixation distribution in a given image is different from the prior $p(x,y)$ (centre bias).
If the ratio is greater than one, the model predicts there should be more fixations than the centre bias expects.

Now we wish to understand, for each point in the image, how much more the model knows about the fixations than the centre bias.
This can be achieved by separating the information gain (an integral over space) into its constituent pixels (shown in Figure \ref{fig:information_gain_images} column four), as $p(x,y \mid I) \log(\hat p(x,y \mid I) / p(x,y))$.
The information gain quantifies how much more efficient the model is in describing the fixations than the centre bias, by quantifying how much channel capacity it could save (in \si{\bits\per\fix}).
These plots quantify \textit{where} in the image the information is saved.
Note also that weighting by the gold standard $p(x,y \mid I)$ results in a weaker penalty for incorrect predictions in areas where there are fewer fixations.


Finally, the fifth column in Figure \ref{fig:information_gain_images} shows the difference between the model's information gain and the possible information gain, estimated by the gold standard, resulting in $p(x,y \mid I) \log(\hat p(x,y \mid I) / p(x,y \mid I))$.
It tells us where and by how much the model's belief is wrong: where and how much information (bits) is being wasted that could be used to describe the fixations more efficiently. 
If the contour plots in the last column of Figure \ref{fig:information_gain_images} are integrated over the image, we get exactly the negative image-based Kullback-Leibler divergence. 
The advantage of our approach is obvious: we can see not only how much a model fails, but exactly where it fails.
We believe this metric can be used to make informed decisions about how to improve a saliency model.

For example, in Figure \ref{fig:information_gain_images}c, it is important to capture the text in the lower part of the image.
The BMS and AIM models fail to capture this difference, while eDN does noticably better.
In addition, all models in these images appear to underestimate the spread of fixations.
Take for example the first image: the eDN model's prediction could be improved by placing more density around the ship rather than on it.

To extend this image-based analysis to the level of the entire dataset, we display each image in the dataset according to its possible information gain and the percentage of that information gain explained by the model (Figure \ref{fig:information_gain_scatter}).
In this space, points to the bottom right represent images that contain a lot of explainable information in the fixations that the model fails to capture.
Points show all images in the dataset, and for a subset of these we have displayed the image itself (left column) and the information gain difference to the gold standard (right column).
For the eDN model (top row), the images in the bottom-right of the plot tend to contain human faces.
The Judd model contains an explicit face detection module, and as can be seen in the bottom row of Figure \ref{fig:information_gain_scatter}, it tends to perform better on these images.
In terms of the whole dataset however, the eDN model generally performs better on images with a moderate level of explainable information (around 3 bit/fixation).





\section{Discussion}

\subsection{How close are we to understanding image-based saliency?}

We have evaluated a number of saliency models within the framework of point processes using log-likelihoods.
For the dataset examined here, the total amount of mutual information that can be extracted from an image about the spatial structure of fixations was 2.1 bits/fixation.
This represents all variation in fixation patterns that is consistent between observers (the gold standard model); a better prediction is not possible without additional knowledge about inter-observer differences in fixation behaviour.
The best saliency model (\BestModel) accounts for \SI{\BestModelPercentOfGold}{\percent} of this total information, while a model that ignores image content and captures only observers' centre bias accounts for \SI{\BaselinePercentOfGold}{\percent} of this information.
Partialling out the mutual information explained by spatial behavioural biases shows that the best existing saliency models explain only \SI{\BestModelPercentOfGoldRelBaseline}{\percent} of the mutual information. 
To examine the generality of these findings across eye movement datasets, we also ran our analysis on the Kienzle dataset \cite{Kienzle2009}, which has been designed to remove photographer bias (placing salient objects in the centre of the photograph) such that salient points are roughly equally distributed across the image.
In this dataset (see see Appendix \ref{sec:app_kienzle}), even less \SI{\KienzleBestModelPercentOfGoldRelBaseline}{\percent} of the possible information gain is covered by the best model (here, \KienzleBestModel).
Our results show that there remains a significant amount of information that image-based saliency models could explain but do not.

In order to improve models it is useful to know where this unexplained information is located.
We developed methods to assess not only model performance on a database level, but also to show where and by how much model predictions fail in individual images (Figures \ref{fig:information_gain_images} and \ref{fig:information_gain_scatter}).
We expect these tools will be useful for the model development community, and provide them in our free software package.

%Our contribution above depends on having a precise definition of image-based saliency, which we provided in Section \ref{sec:saliency_definition}.
%Note that this definition includes anything that can be known from the image alone.
%This may be different to how some in the field have defined ``low-level saliency'', since it can include more abstract features such as faces, text, and other objects.
%It excludes effects specific to the observer, such as memory components, serial dependencies, motivation and task-driven expectations.
%These undoubtedly have a large influence on where people look in images \cite{Koehler2014, Tatler2011}. 
%--- understanding fixation selection in humans at a basic level, versus applied goals (.e.g advertising).


%% TODO: we also resolve the inconsistencies in model rankings (compare non-optimised scales to optimised scales)...

We compared a number of models on their performance over two datasets.
While the primary goal of this paper was to show the distance between state-of-the-art models and the gold standard, benchmarking and model comparison is important to gauge progress \cite{Borji2012,Judd2012,Borji2013d,Borji2013b}.
Happily, the ranking of models by their log-likelihoods does not differ substantially from rankings according to AUC (see Appendix, Figure \ref{fig:metric_comparison}), indicating that the results of previous model comparison efforts would be unlikely to change qualitatively under log-likelihoods.
Furthermore, metrics like AUC and KL-divergence are highly correlated with log-likelihoods if correctly computed (though often they are not; see Appendix \ref{sec:kl_divergence} and Figure \ref{fig:metric_comparison}).
Nevertheless, log-likelihood is the only metric that is linear in information, and therefore the only one we could use to answer the question ``how close are we'':
having a linear information scale allows us to judge not only the ranking between models, but quantitatively how much one model is better than another.




\subsection{Point processes and probabalistic modelling}

It is appealing to evaluate saliency models using log-likelihoods because it is mathematically principled: it is the standard of all statistical model comparison.
When modeling fixations, point processes are the appropriate probability distributions to use \cite{Barthelme2013}.
While in the traditional framework the saliency map output by a model gives values on some unknown scale, considering fixation selection as a point process allows a precise definition of a saliency map: $\frac{p(x,y\mid I)}{p(x,y)}$ (see Section \ref{sec:saliency_definition}).
Previously saliency was loosely defined as a spatial combination of image features that can predict eye fixation patterns.
We make this connection precise.

We suggest that the biggest practical advantage in using a probabalistic framework is its generality.
Once a model is formulated in a probabalistic way many kinds of ``task performance'' can be calculated, depending on problems of applied interest.
The AUC is one such task. 
It is the performance of a model in a 2AFC task ``which of these two points was fixated?''.
There are other cases where different tasks may be a better metric for model evaluation than log-likelihood, depending on the application.
For example, we might be interested in whether humans will look at an advertisement on a website, or whether the top half of an image is more likely to be fixated than the bottom half.
These predictions are a simple matter of integrating over the probability distribution.
Without a definition of the scale of saliency values this type of evaluation is not well defined.
In addition, a probabalistic model allows the examination of any statistical moments of the probability distribution that might be of practical interest.
For example, Engbert et al \cite{Engbert2014} examine the properties of second-order correlations between fixations in one scanpath.
Using a probabalistic framework does not restrict us from using any type of task-based evaluation metric; on the contrary, it enables the evaluation of multiple tasks for the same model.

Using information to evaluate saliency models has several other desirable properties.
First, it allows the contribution of different factors in explaining data variance to be quantified. 
For example, it is possible to show how much the centre bias contributes to explaining fixation data independent of image-based saliency contributions, as we have done here.
Second, it is differentiable in the point process density, allowing models to be numerically-optimised using off-the-shelf techniques like gradient descent or quasi-newton methods.
Third, it is readily extended to include other factors such as temporal effects or task dependencies, the first of which we do below.


\subsection{Extension to temporal effects} 

\begin{figure*}
  \center
  \include{figures/modeling_temporal.pgf}

  \caption{
  \small
Temporal modelling and results.   
\textbf{(a)} For each optimised spatial saliency model (from Figure \ref{fig:modelling_and_results}) we optimised two temporal image-independent factors: self-excitation captures the tendency of observers to fixate in a circular region nearby their previous fixation point, and aspect-optimised self-excitation is where we additionally optimised the shape of the self-excitation process.
\textbf{(b)} Log likelihood (bits/fixation) for each model considered relative to a null model which assumes uniform fixation density over the image. 
The total bar height represents the optimised performance of each model; the coloured sections show the relative contributions of each model factor from (a).
Note the y-axis has been cropped to the raw baseline model performance.
All green bars show the fully-optimised spatial models from Figure \ref{fig:modelling_and_results}, blue bars show the additional influence of temporal factors.
}
\label{fig:temporal_modelling_and_results}
\end{figure*}

The saliency field has recognised that free viewing static images is not the most representative condition to study naturalistic eye movement behaviour \cite{Tatler2008, Tatler2009, Tatler2011, Ehinger2009, Dorr2010}.
Understanding image based saliency is not only a question of ``where?'', but of ``when?'' and ``in what order?''.
It is the spatiotemporal pattern of fixation selection that is increasingly of interest to the field, rather than purely spatial predictions of fixation locations.

The point process framework we outline in this paper is easily extended to study spatiotemporal effects.
We demonstrate this by modelling a simple temporal correlation between fixations.
Figure \ref{fig:temporal_modelling_and_results} shows the model fitting with the additional inclusion of temporal effects (see Appendix \ref{sec:temporal_effects_methods} for model fitting details).
With \SI{\BaselineSelfExcitationPerformance}{\bits\per\fix}, the baseline model including this optimised temporal component predicted fixations better than the best saliency model (\BestModel) without temporal effects (\SI{\BestModelPerformance}{\bits\per\fix}).
While the gold standard shows that there is still a lot of spatial structure in the fixations left to explain, currently a simple image agnostic model of eye movement dynamics performs better than the best saliency maps.

Optimising the magnitude, scale and shape of the temporal component produced a weighting that assigned higher likelihoods to points near to the last fixation.
That is, human observers were more likely to look near to where they had previously fixated (see also \cite{Hooge2005, Tatler2008, Smith2009, Engbert2014}).
Furthermore, the temporal weighting had an elliptical shape with a bias towards horizontal saccades improved performance (see also recent work by \cite{Clarke2014}).
This is consistent with previous literature showing that the spatial distribution of saccades when viewing natural scenes can be horizontally biased \cite{Tatler2009, Dorr2009, Dorr2010}.

``Inhibition of return'' has been posited as a biologically-plausible way to select new fixations \cite{Klein2000, Itti2000, Itti2001, Borji2013d}.  
In part this was necessitated by the decision stages of these models (winner-takes-all): since the saliency map for a given static image does not change, these models could only predict a single fixation without some mechanism to avoid the point of maximum saliency.
Framing saliency maps probabalistically does not require this mechanism because the probability of fixating a given point of the image is usually much less than one, meaning that the spatial distribution of fixations in an image will tend to spread over the image rather than remaining at the location of maximum saliency.
Our results examining temporal correlations show that a local self-reinforcement effect provides a more complete description of human eye movement behaviour for this dataset.

Finally, this self-reinforcement effect reveals that fixation selection is an example of a \textit{self-exciting point process}.
%\ttodo{insert a sentence relating to eye movements. Check Dirk Brockmann paper on levy flights and eye movements.}
Self-exciting point processes have already proven a powerful modelling tool for a variety of other spatial phenomena, including earthquakes \cite{Ogata1998}, civilian deaths in Iraq \cite{Lewis2010}, and burglaries in Los Angeles \cite{Mohler2011}.
An awareness of this connected literature may lead to new ideas for approaches to saliency models in the future.


\subsection{Conclusions}

Our results show that almost two-thirds of the information (once behavioural biases are partialled out) in fixations that \textit{could} be explained purely from the image still remains to be explained by image-based saliency models.
The use of log-likelihoods to evaluate and optimise saliency models holds a number of advantages over existing metrics. 
To facilitate the use of this metric, we will make a free and open source software framework available (\url{www.bethgelab.org}).

Of course, accounting for the entirety of human eye movement behaviour in naturalistic settings will require incorporating information about the task, high-level scene properties, and mechanistic constraints on the eye movement system \cite{Tatler2011, Tatler2009, Ehinger2009, Engbert2014}.
Our gold standard contains the influence of high-level (but still purely image-dependent) factors to the extent that they are consistent across observers.
Successful image-based saliency models will therefore need to use such higher-level features, combined with task relevant biases, to explain how image features are associated with the spatial distribution of fixations over scenes.

\begin{figure}
  \begin{center}
    \includegraphics[width=7cm]{log_likelihoods_with_IoR_KienzleEtAl2009.pdf}
  \end{center}
  \caption{
	Spatial and temporal results (as in Figure \ref{fig:temporal_modelling_and_results}) assessed on the Kienzle dataset.  
  }
  \label{fig:kienzle_results}
\end{figure}

\newpage


\section{Appendices}

\subsection{Kienzle data set}
\label{sec:app_kienzle}

We repeated the full evaluation on the dataset of Kienzle et al in \cite{Kienzle2009}. It consists of 200 grayscale images of size $1024\times678$ and $15$ subjects. This dataset is of special interest, as the authors removed the photographer bias by using random crops from larger images. The results are shown in Figure \ref{fig:kienzle_results}.



In this dataset, with \SI{\KienzleBestModelPercentOfGoldRelBaseline}{\percent} even less of the possible information gain is covered by the best model (here, \KienzleBestModel\footnote{Note that we were not yet able to include eDN into this comparision, as the source code was not yet released at the time of the analysis.}).
Removing the photographer bias leads to a smaller contribution (\SI{\KienzleBaselinePercentOfGold}{\percent}) of the nonparametric model compared to the increase in log-likelihood by saliency map based models. The possible information gain is with \SI{\KienzlePossibleInformationGain}{\bits\per\fix} smaller than for the Judd dataset (\SI{\PossibleInformationGain}{\bits\per\fix}) There are multiple possible reasons for this. Primarily, this dataset contains no pictures of people, but a lot of natural images. In addition, the images are in grayscale.
The increase by considering simple temporal correlations is still higher than the increase of the best spatial saliency map model, the effect is even stronger than in the Judd dataset.


\subsection{Other evaluation metrics}

Here we consider the relationship between log-likelihoods and prominent existing saliency metrics: AUC and KL-divergence.

\label{sec:app_other_metrics}

\subsubsection{AUC}

\begin{figure*}[!ht]
  \center
  \include{figures/auc_problems.pgf}
  \caption{
  \small
The AUC is invariant to monotonic transformations of the saliency maps, and this can produce counterintuitive results. 
Several hypothetical saliency maps (coloured images in columns) and fixation distributions (point scatters in rows) are shown in a grid; 
each cell shows the log-likelihood (bits/fixation) and AUC score (\%) for that model/fixation combination.
\textbf{(a)} Two saliency maps with the same spatial pattern of predictions but different prediction strengths.
The two fixation distributions are sampled from the first and second models respectively.
The first model makes a strong prediction that fixations should cluster in the centre; when they do (first row), this model receives a much higher likelihood than the alternative model.
When the prediction is wrong and fixations are nearly uniformly distributed across the image (second row), the first model is penalised with a much lower likelihood.
AUC does not differentiate these models because it is insensitive to prediction strength, only rank order matters for the performance.
\textbf{(b)} The property of AUC demonstrated in (a) can lead to counterintuitive results.
If higher saliency values are intuitively associated with more fixations then the left model is the better model.
Counterintuitively, the left model has a lower AUC score than the right model. 
Likelihoods are more in line with our intuitions in this case.
}
\label{fig:auc_problems}
\end{figure*}


The most prominent metric used in the saliency literature is the area under the receiver operating characteristic curve (AUC).
The AUC is the area under a curve of model hit rate against false positive rate for each threshold.
It is equivalent to the performance in a two-alternative forced-choice (2AFC) task where the model is ``presented'' with two image locations: one at which an observer fixated and another from a \textit{nonfixation distribution}.
The thresholded saliency value is the model's decision, and the percent correct of the model in this task across all possible thresholds is the AUC score.
The different versions of AUC used in saliency research differ primarily in the nonfixation distribution used. 
This is usually either a uniformly-selected distribution of not-fixated points across the image (e.g. in \cite{Judd2012}), or the distribution of fixations for other images in the database (the shuffled AUC, \cite{Tatler2005, Borji2013b, Borji2013d}).
The latter provides an effective control against \textit{centre bias} (a tendency for humans to look in the centre of the screen, irrespective of the image content), assuming both fixation and nonfixation distributions have the same image-independent bias\footnote{It is important to bear in mind that this measure will penalise models that explicitly try to model the centre bias.}.
The AUC therefore depends critically on the definition of the nonfixation distribution.
In the case of the uniform nonfixation distribution, AUC is tightly related to \textit{area counts}: optimizing for AUC with uniform nonfixation distribution is equivalent to finding for each percentage $0\leq r \leq 100$ the area consisting of $r\%$ of the image which includes most fixations \cite{Barthelme2013}.

One characteristic of the AUC that is often considered an advantage is that it is sensitive only to the rank-order of saliency values, not their scale (i.e. it is invariant under monotonic pointwise transformations) \cite{Tatler2005}.
This allows the modelling process to focus on the shape (i.e. the geometry of iso-saliency points) of the distribution of saliency without worrying about the scale, which is argued to be less important for understanding saliency than the contour lines \cite{Tatler2005}.
However, in certain circumstances the insensitivity of AUC to differences in saliency can lead to counterintuitive behaviour, if we accept that higher saliency values are intuitively associated with more fixations (Figure \ref{fig:auc_problems}).





By using the likelihood of points as a classifier score, one can compute the AUC for a probabilistic model just as for saliency maps. 
This has a principled connection with the probabilistic model itself: 
if the model performed the 2AFC task outlined above using maximum likelihood classification, then the model's performance is exactly the AUC.
Given the real fixation distribution, it can also be shown that the best saliency
map in terms of AUC with uniform nonfixation distribution is exactly the gaze
density of the real fixation. 
However, this does not imply that a better AUC score will yield a better log-likelihood or vice versa.

For more details and a precise derivation of these claims, see \cite{Barthelme2013}. In Figure \ref{fig:metric_comparison} we compare our log-likelihood evaluation to AUC performances.


\subsubsection{Kullback-Leibler divergence}
\label{sec:kl_divergence}

\begin{figure}[!ht]
  \center
  \include{figures/kl_divergence.pgf}
  \caption{Fixation-based Kullback-Leibler divergence for saliency maps.
  The top-left image shows a real saliency map (from eDN); the top-right is inverted,
  bottom-left is the same map with binned saliency values, and in the bottom-right map, the saliency assigned to 
  each bin is shuffled.
  These maps have identical fixation-based KL-divergence (and very different log-likelihoods).
  }
  \label{fig:kl_divergence}
\end{figure}


\begin{table*}[!ht]
  \center
  \include{kl_divergence_table}
  \caption{
	Papers using KL-divergence to evaluate saliency models. We describe what was used as the estimate of the true distribution for image-based KL-divergence. 
  }
  \label{tab:kl_divergence}
\end{table*}




Kullback-Leibler divergence (KL-divergence) is tightly related to
log-likelihoods.
However, KL-divergence as used in practice in the saliency literature
is not the same as the approach we advocate.

In general, the KL-divergence between two probability distributions
$p$ and $q$ is given by
\[
 D_{KL}[p \| q] = \int \log\left(\frac{p(x)}{q(x)}\right)p(x)dx,
\]

and is a popular measure of the difference between two probability
distributions.
In the saliency literature, there are at least two different model
comparison metrics that have been called ``Kullback-Leibler divergence''.
Thus, when a study reports a KL metric, one needs to check how this
was computed.
The first variant treats the saliency map as a two dimensional
probability distribution and computes the KL-divergence between this
predicted distribution and the empirical density map of fixations
\cite{Tatler2005, Wilming2011}; we will call this \textit{image-based
KL-divergence}.
The second metric referred to as ``Kullback-Leibler divergence'' is
the KL-divergence between the distribution of saliency values at
fixations and the distribution of saliency values at nonfixation
locations; we will call this \textit{fixation-based KL-divergence}
\cite{itti2005bayesian}.
This is calculated by binning 
the saliency values at 
fixations and nonfixations into a
histogram  and then computing the KL-divergence of these histograms.
Like AUC, it depends critically on the definition of the nonfixation
distribution, and additionally on the histogram binning.
In Table \ref{tab:kl_divergence} we list a number of papers using one
of these two definitions of KL-divergence.

We now precisely show the relationship between these measures and our
information theoretic approach. Very generally, Information theory can be derived from the task of assigning code words to different events that occur with different probabilities such that their average code word length becomes minimal. It turns out that the negative log-probability is a good approximation to the optimal code word length possible which gives rise to the definition of the log-loss:
$$
l(x) = - \log p(x)
$$
In case of a discrete uniform distribution $p(x)=\frac{1}{n}$ the log-loss for any possible $x$ is simply $\log n$, i.e. the log of the number of possible values of $x$. Accordingly, the more ambiguous the possible values of a variable are, the larger its average log-loss which is also known as its entropy 
$$
H[X]= \EE[-\log p(x)]
$$
If $p(x)$ denotes the true distribution which accurately describes the variable behavior of $x$ and we have a model $q(x)$ of that distribution then we can think of assigning code words to different values of $x$ that are of length $-\log q(x)$ and compute the average log-loss for the model distribution
\begin{align*}
  &\EE[-\log q(x)] \\
  &\quad = - \int p(x) \log q(x) dx\\
  &\quad = H[X] + D_{KL}[p(x)||q(x)]
\end{align*}
That is the KL-divergence measures how much the average log-loss of a model distribution $q(x)$ exceeds the average log-loss of the true distribution. The KL-divergence is also used to measure the information gain of an observation if p(x) denotes a posterior distribution that correctly describes the variability of $x$ after the observation has been made while q(x) denotes the prior distribution. In a completely analog fashion we can measure how much more or less information one model distribution $q_1(x)$ provides about $x$ than an alternative model $q_2(x)$ does by computing how much the average log-loss of model 1 is reduced (or increased) relative to the average log-loss of model 2. This can also be phrased as an expected log-likelihood ratio \footnote{The concept of log-likelihood ratios is familiar to readers with knowledge of model comparison using e.g. $\chi^2$ tests.} (ELLR):
\begin{align*}
ELLR & = \EE[- \log q_2(x)] - \EE[- \log q_1(x)]\\
& =  \EE[\log q_1(x)] - \EE[ \log q_2(x)]\\
& = \int p(x) \log \frac{q_1(x)}{q_2(x)} dx.
\end{align*}
In other words, very generally, the amount of information model 2 provides about a variable relative to model 1 can be measured by asking how much more efficiently the variable can be encoded when assuming the corresponding model distribution $q_2(x)$ instead of $q_1(x)$ for the encoding. Note, that this reasoning does not require any of the two model distributions to be correct. For example, in the context of saliency maps we can ask what the best possible model distribution is which does not require any knowledge of the actual image content. This baseline model can capture general biases of the subjects such as the center bias. In order to evaluate the information provided by a saliency map that can be assigned to the specific content of an image we thus have to ask how much more the model distribution of that saliency model provides relative to the baseline model.

Our information gain metric reported in the paper is exactly the ELLR,
where $q_1$ is the model, $q_2$ is the baseline, and we estimated the
expectation value using the sampling estimator.
The ELLR can be rewritten as a difference between KL-divergences:

\begin{align*}
\text{ELLR} :=& \EE[ \log (q_1(x)/q_2(x))]\\
            =& \EE[ \log q_1(x)] - \EE[\log q_2(x)]\\
            =& \DKL[p(x) \| q_2(x)] - \DKL[p(x) \| q_1(x)]
\end{align*}


This naturally raises the question: is our measure equivalent to the
KL-divergence that has been used in the saliency literature?
The answer is no.

%The term KL-divergence is ambiguous because it is critical for which
%probability distributions the KL divergence is computed.

It is crucial to note that in the past the scale used for saliency
maps was only a rank scale.
This was the case because AUC was the predominant performance measure,
and is invariant under such transformations.
That is, two saliency maps $S_1(x)$ and $S_2(x)$ were considered
equivalent if a strictly monotonic increasing function $g: \RR \to
\RR$ exists such that $S_1(x)$ = $g(S_2(x))$.
In contrast, in the equation for ELLR, the two distributions $q_1$ and
$q_2$ are directly proportional to the saliency map times the center
bias distribution, and well-defined only if the scale used for
saliency maps is meaningful.
In other words, if one applies a nonlinear invertible function to a
saliency map the ELLR changes.








%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%












%
%
%
%
%
%
%
%
%
%Kullback-Leibler divergence (KL-divergence) is tightly related to log-likelihoods.
%However, KL-divergence as used in practice in the saliency literature is not the same as the approach we advocate.
%
%In general, the KL-divergence between two probability distributions $p$ and $q$ is given by
%\[
%  D_{KL}[p \| q] = \int \log\left(\frac{p(x)}{q(x)}\right)p(x)dx,
%\]
%
%and is a popular measure of the difference between two probability distributions.
%In the saliency literature, there are at least two different model comparison metrics that have been called ``Kullback-Leibler divergence''.
%Thus, when a study reports a KL metric, one needs to check how this was computed.
%The first variant treats the saliency map as a two dimensional probability distribution and computes the KL-divergence between this predicted distribution and the empirical density map of fixations \cite{Tatler2005, Wilming2011}; we will call this \textit{image-based KL-divergence}.
%The second metric referred to as ``Kullback-Leibler divergence'' is the KL-divergence between the distribution of saliency values at fixations and the distribution of saliency values at nonfixation locations; we will call this \textit{fixation-based KL-divergence} \cite{itti2005bayesian}.
%This is calculated by binning fixations and nonfixations into a histogram then computing the KL-divergence of these histograms.
%Like AUC, it depends critically on the definition of the nonfixation distribution, and additionally on the histogram binning.
%In Table \ref{tab:kl_divergence} we list a number of papers using one of these two definitions of KL-divergence.
%
%We now precisely show the relationship between these measures and our measure of information gain.
%Information gain for any given fixation equals the log-likelihood ratio (LLR) between model distribution $q_1$ to model distribution $q_2$\footnote{The concept of log-likelihood ratios is familiar to readers with knowledge of model comparison using e.g. $\chi^2$ tests.}
%The expected LLR (ELLR) is obtained by averaging over the true distribution $p(x)$ and thus measures how much more information $q_1$ provides about $x$ than $q_2$.
%The ELLR can be rewritten as a difference between KL-divergences:
%
%\begin{align*}
%\text{ELLR} :=& \EE[ \log (q_1(x)/q_2(x))]\\
%             =& \EE[ \log q_1(x)] - \EE[\log q_2(x)]\\
%             =& \DKL[p(x) \| q_2(x)] - \DKL[p(x) \| q_1(x)]
%\end{align*}
%
%Our information gain metric reported in the paper is exactly the ELLR, where $q_1$ is the model, $q_2$ is the baseline, and we estimated the expectation value using the sampling estimator.
%This naturally raises the question: is our measure equivalent to the KL-divergence that has been used in the saliency literature?
%The answer is no.
%%The term KL-divergence is ambiguous because it is critical for which probability distributions the KL divergence is computed. 
%
%It is crucial to note that in the past the scale used for saliency maps was only a rank scale.
%This was the case because AUC was the predominant performance measure, and is invariant under such transformations.
%That is, two saliency maps $S_1(x)$ and $S_2(x)$ were considered equivalent if a strictly monotonic increasing function $g: \RR \to \RR$ exists such that $S_1(x)$ = $g(S_2(x))$.
%In contrast, in the equation for ELLR, the two distributions $q_1$ and $q_2$ are directly proportional to the saliency map times the center bias distribution, and well-defined only if the scale used for saliency maps is meaningful.
%In other words, if one applies a nonlinear invertible function to a saliency map the ELLR changes.

% paragraph on fixation-based:
\textit{Fixation-based KL-divergence} is the more common variant in the literature: researchers wanted to apply information theoretic measures to saliency evaluation while remaining consistent with the rank-based scale of AUC \cite{itti2005bayesian}.
Therefore they did not interpret saliency maps themselves as probability distributions, but applied the KL-divergence to the distribution of saliency values obtained when using the fixations to that obtained when using non-fixations.
%That is, they first sample $x \thicksim \pfix$ and determine a histogram over $s=S(x)$ where $S(x)$ is the saliency map, and then they sample $x \thicksim \pnonfix$ and determine another histogram over $s=S(x)$ and then they compute a KL-divergence between the two resulting histograms.
%For distinction, we refer to this as ``fixation-based'' KL-divergence and
We emphasize that this measure has an important conceptual caveat:
rather than being invariant under only monotonic increasing transformations, KL-divergence is invariant under \textit{any} reparameterisation.
%In practice with histogram binning, this means that any transformation that preserves which saliency values belong to the same bin will leave the KL-divergence unchanged.
%We demonstrate this in Figure \ref{fig:kl_divergence}a.
%We can invert the saliency map shown in the upper right, or even rearange the binned saliency values randomly, without changing the fixation-based KL-divergence.
This implies that the measure only cares about which areas are of equal saliency, but does not care which of any two areas is actually the more salient one.
For illustration, for any saliency map $S(x, y)$, its negative counterpart $\bar S(x,y) := \sup(S) - S(x,y)$ is completely equivalent with respect to the fixation-based KL metric, even though for any two image regions $\bar S$ would always make the opposite prediction about their salience (see Figure \ref{fig:kl_divergence}a).
Furthermore, the measure is sensitive to the histogram binning used, and in the limit of small bin width all models have the same KL-divergence: the model-independent KL-divergence between $p(\xfix)$ and $p(\xnonfix)$.

%Fixation-based KL-divergence also depends critically on the bin width (or equivalently, the number of bins) used in the histogram, in a non-principled way (Figure \ref{fig:kl_divergence}b).
%With too few bins both distributions become uniform (the KL-divergence converges to zero with one bin).
%With sufficiently many bins, the saliency map itself is a reparameterisation ($S: \RR^2 \to \RR$), and therefore the KL-divergence converges to the true KL-divergence between $p(\xfix)$ and $p(\xnonfix)$. 
%That is, in the infinite bin limit, the lines in Figure \ref{fig:kl_divergence}b will converge to the same value.
%Our numerical simulations show that there is a plateau between these limits where the estimated KL-divergence seems to be constant --- but the location of this region is arbitrary.
%Finally, there is also a selection of bin size for which a random model (with uniformly sampled saliency values) performs better than the eDN model.
%Fixation-based KL-divergences can show many artifacts, depending on the number of bins selected.

%Because of this, we can rearange the bins of the histogram as we like while keeping the fixation-based KL-divergence constant
%That is, for example if we first look at image stripes only, then a monotonically increasing saliency map would always have exactly the same KL divergence as a monotonically decreasing one, since both must be equal to $\DKL [ \pfix \| \pnonfix]$.
%Furthermore, if one would approximate a saliency map $S(x)$ by a piecewise constant function $S^*(x)$ such that $S^*(x)$ takes a unique value within each bin of the two histograms used for computing the KL-divergence then it holds that again the KL-divergence of all saliency maps are identical as long as they take (infinitesimally) different values in each bin (i.e. the $S^*$ provides a one-to-one mapping of the index of each bin).
%Again they would all have to be equal to the KL-divergence between the two discretized distributions $\pfix$ and $\pnonfix$.
%Therefore, all differences observed between different saliency maps are artifacts.
%\mitodo{Reformulate with negative saliencymap example, refer Figure \ref{fig:kl_divergence} a and b}

\textit{Image-based KL-divergence} requires that the saliency maps are interpreted as probability distributions.
Previous studies employing this method (Table \ref{tab:kl_divergence}) simply divided the saliency values by their sum to obtain such probability distributions. 
However, they did not consider that this measure is sensitive to the scale used for the saliency maps.
Optimization of the pointwise nonlinearity (i.e. the scale) has a huge effect on the performance of the different models (see below in \ref{sec:app_metric_comparison}).
More generally, realising that image-based KL-divergence treats saliency maps as probability distributions means that other aspects of density estimation, like centre bias and regularisation strategies (blurring), must also be taken into account.

The only conceptual difference between image-based KL-divergence and log-likelihoods is that for estimating expected log-likelihood ratios, it is not necessary to have a gold standard.
One can simply use the unbiased sample mean estimator (see \ref{sec:app_estimation_considerations}).
Furthermore, by conceptualising saliency in an information-theoretic way, we can not only assign meaning to expected values (such as ELLR or DKL) but we also know how to measure the information content of an individual event (here, a single fixation) using the notion of its log-loss (see our application on the individual image level in Figure \ref{fig:information_gain_images}).
Thus, while on a theoretical level log-likelihoods and image-based KL-divergence are tightly linked, on a practical level a fundamental reinterpretation of saliency maps as probability distributions is necessary \ref{sec:app_metric_comparison}.

%Thus, if one does not optimize the scale, the assessment of the different models becomes completely inconsistent with AUC (as in \cite{Riche2013}), while our approach of optimizing the scale leads to results that are highly consistent with AUC.





\subsubsection{Metric comparison}
\label{sec:app_metric_comparison}

\begin{figure*}[!ht]
  \center
  %\includegraphics[width=7.5cm]{other_measures_scatter.pdf}
  \include{figures/other_measures.pgf}
%  \include{figures/kl_divergence.pgf}
  \caption{Possible information gain explained (x-axis) compared to evaluation metrics (y-axis). 
  All metrics have been rescaled to yield zero for the baseline and one for the gold standard. 
  Each column of points corresponds to one model (e.g. the right-most column is the eDN).
  The dashed diagonal line shows model log-likelihoods, which are linear in information.
  \textbf{(a)}: 
  Metrics (other than log-likelihoods) calculated on raw model output.
  The legend shows the Pearson correlation coefficient $r$ and Spearman rank order correlation $\rho$ between each metric and information gain explained.
  Many metrics are inconsistent with information, in particular showing rank order changes.
  \textbf{(b)}: 
  Included models are all optimised for nonlinearity, blur and centre bias.
  All metrics are highly correlated with information gain explained.
  Even though log-likelihoods should be preferred for the reasons outlined in our paper, results using other metrics are unlikely to change qualitatively so long as they are appropriately converted to probabalistic models.
  However, note also that since all metrics must converge to the gold-standard model at (1, 1), all metrics apart from log-likelihoods are nonlinear in information.
   }
  \label{fig:metric_comparison}
\end{figure*}

While many saliency metrics have been shown to provide inconsistent model rankings in the past \cite{Riche2013}, we now show how treating saliency maps in a stringent probabalistic fashion can resolve these discrepancies, resulting in a fully principled and unambiguous evaluation framework (see Figure \ref{fig:metric_comparison}).

In Figure \ref{fig:metric_comparison} we show a comparison between a number of metrics and information gain.
Crucially, we do this both for the raw model output and for models that have been appropriately converted to a probabalistic framework (as we do in the paper, by jointly optimising for nonlinearity, centre bias and blurring).
To allow comparison between the metrics, we have rescaled the performance metrics to yield zero for the baseline (centre bias) model and one for the gold standard.
This corresponds to the ratio of the explainable information gain (gold standard) to that explained by the models.
That is, for each metric, the baseline model is always at point (0, 0) and the gold standard at point (1, 1; not shown in the plot).

Figure \ref{fig:metric_comparison}a shows this space for all metrics computed on the raw model output (except for log-likelihoods, which must be optimised for nonlinearities to be meaningful).
Log-likelihood is a straight line running through the space, because its differences are exactly information gain.
The metrics strongly disagree, showing inconsistent rankings --- this is similar to what has been previously shown when comparing saliency metrics \cite{Riche2013}.
In addition, they are inconsistently correlated with information gain, from almost perfectly rank-order correlated (e.g. AUC with a uniform nonfixation distribution) to not at all (fixation-based KL-divergence with a centre bias nonfixation distribution).

In contrast, when the saliency maps are appropriately translated into probabalistic models (Figure \ref{fig:metric_comparison}b), all metrics become highly correlated with log-likelihood (and thus information gain) in both value and ranking.
This shows that model comparisons that have used other metrics would be unlikely to change qualitatively if log-likelihoods were compared instead --- provided that the models are properly converted into a probabalistic framework.

%We believe that the reason that benchmarks using different metrics give inconsistent results \cite{Judd2012, Borji2013b, Riche2013} is that they use metrics that have incompatible definitions of what a saliency map is, and apply them to models with incompatible definitions of what a saliency map is.
%It is therefore no surprise that the metrics produce inconsistent rankings.
%For example, in \cite{Judd2012}, the Judd model \cite{Judd2009} performs significantly better than AIM \cite{Bruce2009}.

%The problem to this point has been that model comparison metrics each expect the models to be doing different things: each metric applied its own definition of what a saliency map is.
%Because the models do different things, this results in inconsistencies.
%For example, consider one model that defines the saliency map as excluding centre bias (e.g. by using shuffled AUC) and a second model that is exactly the same except that it expects the centre bias and tries to model it.
%From the point of view of understanding the association between image features and fixations, these models are exactly the same.
%A saliency map from the first model will win by shuffled AUC, whereas a saliency map from the second model will win on normal AUC. 
%However, from the perspective of saliency understanding, these models should be equal --- and should be under the right comparison metric.

Additionally, this visualisation serves to demonstrate two desirable features of using log-likelihoods.
First, note that for four of the metrics the model ranks with information do change slightly, whereas the model ranking according to log-likelihood remains consistent with information.
Second, recall that all metrics must converge to the gold-standard model at (1, 1).
This means that all metrics aside from log-likelihoods are nonlinear in information, since they diverge from the line linking (0, 0) and (1, 1).
These metrics will answer the question of ``how close are we to understanding image-based saliency'' in a distorted way.

However, we can see that all of these metrics will nevertheless provide an answer of ``not very'' for the dataset here (for example, AUC would say ``40\%!''). 
To reach this conclusion, it is essential to use the correct baseline model.
If you are interested in image-based saliency, the appropriate baseline is the centre bias model, not the uniform distribution. 


%\subsubsection{Comparison of log-likelihood to other saliency metrics}
%
%Several authors have considered desiderata for saliency evaluation metrics \cite{Wilming2011, Tatler2005, Riche2013}.
%How does log-likelihood fair against other metrics?
%First, log-likelihoods contain no free parameters: the real fixation distribution does not need to be estimated, a nonfixation distribution does not need to be chosen, it does not assume any particular error distribution, and there are no binning schemes, thresholds or bandwidths.
%For the same reason it is not sensitive to low data (in fact, log-likelihood is an unbiased estimator for how close the model is to the real distribution --- see Appendix \ref{sec:kl_divergence}). 
%Unlike AUC, log-likelihood \textit{is} sensitive to monotonic transformations and biases (such as the centre bias), but this invariance keeps us from understanding the fixation distribution to its full extent.
%We believe it is better to explicitly model these biases and nonmonotonicities, as we have done in this paper.
%Finally, likelihoods \textit{are} sensitive to outliers in the data, unlike AUC.
%For example, if a fixation falls in an area where the prediction was extremely low, the average model likelihood will be heavily penalised.
%However, we argue this is in fact a desirable feature for a model comparison metric.
%Unless there is good reason to think that a data point is erroneous, it should be modelled like everything else.
%If a model gets some predictions badly wrong, this provides useful information about where a model fails and how it can be improved.
%
%More generally, we believe that log-likelihood is an intuitive scale because it measures information.
%If one model's log-likelihood exceeds another model's log-likelihood by 1~bit, the first model predicts the data twice as well ($2^1 = 2$).
%The first model is ``less surprised'' by seeing the data.
%Relative likelihood values give more than just the model ranking, they also quantify how much more one model knows about the fixations than another.
%This is not true for AUC. 
%
%\ttodo{mention possibility of regularisation for optimisation?}

\subsection{Estimation considerations}
\label{sec:app_estimation_considerations}

%\begin{figure}[!ht]
%  \center
%  \includegraphics[width=7.8cm]{ellr_estimators.pdf}
%%  \include{figures/kl_divergence.pgf}
%  \caption{The influence of different estimates for the real distribution on the expected log-likelihood ratio (ELLR). See text for details.
%   }
%  \label{fig:estimation_considerations}
%\end{figure}

%we do not need gold standard for all model comparisons but the gold-model because of sample mean 
One principle advantage of using log-likelhoods instead of image-based KL-divergence is that for all model comparisons but comparing against the gold standard we do not have to rely on the assumptions made for the gold standard but can simply use the unbiased sample mean estimator:
$$
\hat \EE[\log q_1(x)/q_2(x)] = \frac{1}{N} \sum_{k=1}^N \log q_1(x_k)/q_2(x_k)
$$
This is why we used the sample mean estimator for all model comparisons rather than the gold standard to estimate the ELLR. 

% crossvalidated kernel size
However, estimating the upper limit on information gain still requires a gold standard (an estimate of the true distribution $p(x)$). 
Image-based KL-divergence requires this not only for estimating the upper bound, but for calculating the performance of any model.
There, it has usually been done using a 2D histogram or Gaussian kernel density estimate (see Table \ref{tab:kl_divergence}), and
the hyper parameters (e.g. bin size, kernel size) have commonly been chosen based on fovea size or eye tracker precision.
In our framework of interpreting saliency maps as probability distributions, a principled way of choosing these hyper parameters is to cross-validate over them to get the best possible estimate of the true distribution.

For our dataset, the optimal cross-validated kernel size was $27$ pixels which is relatively close to the commonly used kernel size of $1^\circ$ (37 pixels).
However, with more fixations in the dataset the optimal cross-validated kernel sizes will shrink, because the local density can be estimated more precisely.
Therefore, choosing these hyperparameters on criteria other than cross-validation will produce inaccurate estimates of the ELLR in the large data limit.





% 34% understood is a conservative estimate


Since we conclude that our understanding of image-based saliency is surprisingly limited, we have been using a conservative strategy 
for estimating the information gain of the gold standard that is downward biased such that we obtain a conservative upper bound on the fraction of how much we understand about image-based saliency.
To this end, we not only used the unbiased sample estimator for averaging over the true distribution but we resorted to a cross-validation strategy for estimating the gold standard that takes into account how well the distributions generalize across subjects:
$$
\hat E[p_{gold}] = \sum_{j=1}^M \frac{1}{N_j} \sum_{k=1}^{N_j} \log p_{gold}(x_{jk}|j)
$$
where the first sum runs over all subjects j and $p_{gold}(x_{jk}|j)$ denotes a kernel density estimator which uses all fixations but the one of subject $j$.
For comparison, if one would simply use the plain sample mean estimator for the gold standard the fraction explained would drop to an even smaller value of only 22\%.
Our approach guarantees that it is very likely that the true vale falls into the range between 22 and 34\%.

\subsection{Modelling temporal effects}
\label{sec:temporal_effects_methods}

We quantified the extent to which humans use inhibition of return when viewing static images by including these temporal effects into the model comparison.
If humans use inhibition of return then adding this mechanism to the model should lead to higher log-likelihoods than models without IoR.

The temporal effects were optimized independently of the other factors detailed above.
We modeled inhibition of return by multiplying the likelihood with a factor depending on $\Delta = \sqrt{(x_i-x_{i-1})^2+(y_i-y_{i-1})^2}$:

\begin{align*}
&p(x_i, y_i | x_1, y_1, \dots, x_{i-1}, y_{i-1}) \\
&\qquad  \propto (1+f(\Delta))p(x_i, y_i)
\end{align*}

If $f(\Delta)=0$ then the saliency map prediction is unchanged by temporal information.
If $f(\Delta)<0$ then saliency values at this distance ($\Delta$) to the previous fixation are decreased: they become less likely, reflecting inihibition of return.
Conversely, if $f(\Delta)>0$ this reflects an attraction or self-excitation effect.

Notice however that in order to make $(1+f(\Delta))p(x_i, y_i)$ a density once more, it must be renormalized to sum to $1$. 
This results in a shift of the threshold $\theta$ that separates excitation ($f(\Delta)>\theta$) from inhibition ($f(\Delta)<\theta$).

\begin{figure}[!ht]
  \center
  \include{figures/temporal_effects.pgf}

  \caption{
	\small  
  Modelling temporal effects.
  In all images, yellow depicts higher density and green lower density.  
  We weight the spatial point process density (top image) by a correction factor centred over the previous fixation (red point) that either decreases (inhibition of return, left column) or increases local saliency (self-excitation, right column).
  The final density is then used to predict the next fixation (blue point).
  }
\end{figure}

We modeled $f$ using a Gaussian, i.e. $f(\Delta) = -\delta\exp(-\tfrac12 \Delta^2/\sigma^2)$. 
Inspired by results of \cite{Tatler2008, Tatler2009} we extended the temporal effect to include eccentricity effects (i.e. allow the region of temporal inhibition / excitation to be elliptical) by setting $\Delta = \sqrt{(x_i-x_{i-1})^2+\alpha (y_i-y_{i-1})^2}$.
We also tested a difference of Gaussians model with separable inhibitory and self-reinforcing components, but omit these results here since they produced the same results as the simple Gaussian (i.e. no evidence for an inhibitory component).

To optimise these temporal parameters the fully optimized spatial density (nonlinearity, centre bias and blur) was used as a base. 
All temporal parameters where again optimized jointly with L-BFGS SLSQP, but independent of the already optimized spatial parameters.


%%% Information gain scatterplots %%%

%\begin{figure*}[!ht]
%
%  \input{figures/information_gains_scatter_combined_appendix.pgf}
%
%  \caption{
%    Image-based information gain plots as in Figure \ref{fig:information_gain_scatter} for the BMS and AIM models.
%  }
%\end{figure*}


\section*{Acknowledgements}

MK, TSAW and MB conceived of the experiments.
MK analysed the data.
MK, TSAW and MB wrote the paper.
We thank Lucas Theis for his suggestions, and Eleonora Vig for helpful comments on an earlier draft of this manuscript.
TSAW was supported by a Humboldt Postdoctoral Fellowship from the Alexander von Humboldt Foundation.

%\bibliography{../../../Mendeley_Papers/library.bib}
\bibliography{library.bib,own_literature.bib}

\end{document}

%%% Local Variables:
%%% TeX-source-correlate-mode: t
%%% TeX-parse-self: t
%%% TeX-auto-save: t
%%% eval: (flyspell-mode 1)
%%% End:

% LocalWords:  saliency

saliency_manuscript_v2.bbl
\begin{thebibliography}{10}

\bibitem{baldi2005attention}
P.~Baldi and L.~Itti.
\newblock Attention: bits versus wows.
\newblock In {\em Neural Networks and Brain, 2005. ICNN\&B'05. International
  Conference on}, volume~1, pages PL--56. IEEE, 2005.

\bibitem{baldi2010bits}
P.~Baldi and L.~Itti.
\newblock Of bits and wows: a bayesian theory of surprise with applications to
  attention.
\newblock {\em Neural Networks}, 23(5):649--666, 2010.

\bibitem{Barthelme2013}
S.~Barthelm\'{e}, H.~Trukenbrod, R.~Engbert, and F.~Wichmann.
\newblock {Modeling fixation locations using spatial point processes.}
\newblock {\em {Journal of vision}}, {13}(12):1--34, Jan. 2013.

\bibitem{Borji2013}
A.~Borji and L.~Itti.
\newblock {State-of-the-art in visual attention modeling.}
\newblock {\em {IEEE transactions on pattern analysis and machine
  intelligence}}, {35}(1):185--207, Jan. 2013.

\bibitem{Borji2012}
A.~Borji, D.~Sihite, and L.~Itti.
\newblock {Salient object detection: A benchmark}.
\newblock In {\em {Computer Vision{\textendash}ECCV 2012}}, pages 414--429,
  2012.

\bibitem{Borji2013b}
A.~Borji, D.~N. Sihite, and L.~Itti.
\newblock {Quantitative analysis of human-model agreement in visual saliency
  modeling: a comparative study.}
\newblock {\em {IEEE transactions on image processing : a publication of the
  IEEE Signal Processing Society}}, {22}(1):55--69, Jan. 2013.

\bibitem{Borji2013d}
A.~Borji, H.~R. Tavakoli, D.~N. Sihite, and L.~Itti.
\newblock {Analysis of Scores, Datasets, and Models in Visual Saliency
  Prediction}.
\newblock {\em {2013 IEEE International Conference on Computer Vision}}, pages
  921--928, Dec. 2013.

\bibitem{Bruce2009}
N.~Bruce and J.~Tsotsos.
\newblock {Saliency, attention, and visual search: An information theoretic
  approach}.
\newblock {\em {Journal of vision}}, {9}(3):5, 1--24, 2009.

\bibitem{Clarke2014}
A.~D. Clarke and B.~W. Tatler.
\newblock {Deriving an appropriate baseline for describing fixation behaviour}.
\newblock {\em {Vision Research}}, {102}:41--51, Sept. 2014.

\bibitem{Dorr2009}
M.~Dorr, K.~R. Gegenfurtner, and E.~Barth.
\newblock {The contribution of low-level features at the centre of gaze to
  saccade target selection}.
\newblock {\em {Vision Research}}, {49}(24):2918--2926, 2009.

\bibitem{Dorr2010}
M.~Dorr, T.~Martinetz, K.~R. Gegenfurtner, and E.~Barth.
\newblock {Variability of eye movements when viewing dynamic natural scenes.}
\newblock {\em {Journal of Vision}}, {10}(10):28, 2010.

\bibitem{Ehinger2009}
K.~A. Ehinger, B.~Hidalgo-Sotelo, A.~Torralba, and A.~Oliva.
\newblock {Modeling Search for People in 900 Scenes: A combined source model of
  eye guidance.}
\newblock {\em {Visual Cognition}}, {17}(6-7):945--978, 2009.

\bibitem{Einhauser2010}
W.~Einh\"{a}user and P.~K\"{o}nig.
\newblock {Getting real{\textemdash}sensory processing of natural stimuli}.
\newblock {\em {Current Opinion in Neurobiology}}, {20}(3):389--395, June 2010.

\bibitem{Engbert2014}
R.~Engbert, H.~A. Trukenbrod, S.~Barthelm\'{e}, and F.~A. Wichmann.
\newblock {Spatial statistics and attentional dynamics in scene viewing}.
\newblock {\em {arXiv}}, page~23, May 2014.

\bibitem{Erdem2013}
E.~Erdem and A.~Erdem.
\newblock {Visual saliency estimation by nonlinearly integrating features using
  region covariances}.
\newblock {\em {Journal of Vision}}, {13}(4):11--11, Mar. 2013.

\bibitem{Goferman2010}
S.~Goferman, L.~Zelnik-Manor, and A.~Tal.
\newblock {Context-aware saliency detection}.
\newblock pages 2376--2383. {IEEE}, June 2010.

\bibitem{Goferman2012}
S.~Goferman, L.~Zelnik-Manor, and A.~Tal.
\newblock {Context-Aware Saliency Detection}.
\newblock {\em {IEEE Transactions on Pattern Analysis and Machine
  Intelligence}}, {34}(10):1915--1926, Oct. 2012.

\bibitem{Harel2006}
J.~Harel, C.~Koch, and P.~Perona.
\newblock {Graph-based visual saliency}.
\newblock In {\em {Advances in neural information processing systems}}, pages
  545--552, 2006.

\bibitem{Hooge2005}
I.~T.~C. Hooge, E.~A. Over, R.~J. van Wezel, and M.~A. Frens.
\newblock {Inhibition of return is not a foraging facilitator in saccadic
  search and free viewing}.
\newblock {\em {Vision Research}}, {45}(14):1901--1908, 2005.

\bibitem{Hou2007}
X.~Hou and L.~Zhang.
\newblock {Saliency detection: A spectral residual approach}.
\newblock In {\em {IEEE Conference on Computer Vision and Pattern Recognition,
  2007.}}, pages 1--8. {IEEE}, 2007.

\bibitem{itti2005quantifying}
L.~Itti.
\newblock Quantifying the contribution of low-level saliency to human eye
  movements in dynamic scenes.
\newblock {\em Visual Cognition}, 12(6):1093--1123, 2005.

\bibitem{itti2005principled}
L.~Itti and P.~Baldi.
\newblock A principled approach to detecting surprising events in video.
\newblock In {\em Computer Vision and Pattern Recognition, 2005. CVPR 2005.
  IEEE Computer Society Conference on}, volume~1, pages 631--637. IEEE, 2005.

\bibitem{itti2005bayesian}
L.~Itti and P.~F. Baldi.
\newblock {Bayesian surprise attracts human attention}.
\newblock In {\em {Advances in neural information processing systems}}, pages
  547--554, 2005.

\bibitem{Itti2000}
L.~Itti and C.~Koch.
\newblock {A saliency-based search mechanism for overt and covert shifts of
  visual attention.}
\newblock {\em {Vision Research}}, {40}(10-12):1489--1506, 2000.

\bibitem{Itti2001}
L.~Itti and C.~Koch.
\newblock {Computational modelling of visual attention}.
\newblock {\em {Nature reviews neuroscience}}, {2}(3):194--203, 2001.

\bibitem{Itti1998}
L.~Itti, C.~Koch, and E.~Niebur.
\newblock {A model of saliency-based visual attention for rapid scene
  analysis}.
\newblock {\em {IEEE Transactions on Pattern Analysis and Machine
  Intelligence}}, {20}(11):1254--1259, 1998.

\bibitem{Scipy}
E.~Jones, T.~Oliphant, P.~Peterson, and {others}.
\newblock {\em {SciPy: Open source scientific tools for Python}}.
\newblock 2001.
\newblock {[}Online; accessed 2014-09-12{]}.

\bibitem{Judd2012}
T.~Judd, F.~Durand, and A.~Torralba.
\newblock {A benchmark of computational models of saliency to predict human
  fixations}.
\newblock {\em {CSAIL Technical Reports}}, Jan. 2012.

\bibitem{Judd2009}
T.~Judd, K.~Ehinger, F.~Durand, and A.~Torralba.
\newblock {Learning to predict where humans look}.
\newblock In {\em {Computer Vision, 2009 IEEE 12th international conference
  on}}, pages 2106--2113. {IEEE}, 2009.

\bibitem{Kienzle2009}
W.~Kienzle, M.~O. Franz, B.~Scholkopf, and F.~A. Wichmann.
\newblock {Center-surround patterns emerge as optimal predictors for human
  saccade targets}.
\newblock {\em {Journal of Vision}}, {9}(5):7, 2009.

\bibitem{Kienzle2007a}
W.~Kienzle, F.~A. Wichmann, B.~Sch\"{o}lkopf, and M.~O. Franz.
\newblock {A Nonparametric Approach to Bottom-Up Visual Saliency}.
\newblock In B.~Sch\"{o}lkopf, J.~Platt, and T.~Hoffman, editors, {\em
  {Advances in Neural Information Processing Systems 19}}, pages 689--696,
  {Cambridge, MA}, 2007. {MIT Press}.

\bibitem{Klein2000}
R.~M. Klein.
\newblock {Inhibition of return}.
\newblock {\em {Trends in cognitive sciences}}, {4}(4):138--147, 2000.

\bibitem{lemeur2013methods}
O.~Le~Meur and T.~Baccino.
\newblock Methods for comparing scanpaths and saliency maps: strengths and
  weaknesses.
\newblock {\em Behavior research methods}, 45(1):251--266, 2013.

\bibitem{lemeur2007predicting}
O.~Le~Meur, P.~Le~Callet, and D.~Barba.
\newblock Predicting visual fixations on video based on low-level visual
  features.
\newblock {\em Vision research}, 47(19):2483--2498, 2007.

\bibitem{Lewis2010}
E.~Lewis, G.~Mohler, P.~J. Brantingham, and A.~Bertozzi.
\newblock {Self-exciting point process models of insurgency in Iraq}.
\newblock {\em {UCLA CAM Reports 10}}, {38}, 2010.

\bibitem{Mohler2011}
G.~O. Mohler, M.~B. Short, P.~J. Brantingham, F.~P. Schoenberg, and G.~E. Tita.
\newblock {Self-Exciting Point Process Modeling of Crime}.
\newblock {\em {Journal of the American Statistical Association}},
  {106}(493):100--108, Mar. 2011.

\bibitem{Ogata1998}
Y.~Ogata.
\newblock {Space-time point-process models for earthquake occurrences}.
\newblock {\em {Annals of the Institute of Statistical Mathematics}},
  {50}(2):379--402, 1998.

\bibitem{rajashekar2004point}
U.~Rajashekar, L.~K. Cormack, and A.~C. Bovik.
\newblock Point-of-gaze analysis reveals visual search strategies.
\newblock In {\em Electronic Imaging 2004}, pages 296--306. International
  Society for Optics and Photonics, 2004.

\bibitem{Riche2013}
N.~Riche, M.~Duvinage, M.~Mancas, B.~Gosselin, and T.~Dutoit.
\newblock {Saliency and Human Fixations: State-of-the-Art and Study of
  Comparison Metrics}.
\newblock {\em {2013 IEEE International Conference on Computer Vision}}, pages
  1153--1160, Dec. 2013.

\bibitem{Riche2013a}
N.~Riche, M.~Mancas, M.~Duvinage, M.~Mibulumukini, B.~Gosselin, and T.~Dutoit.
\newblock {RARE2012: A multi-scale rarity-based saliency detection with its
  comparative statistical analysis}.
\newblock {\em {Signal Processing: Image Communication}}, {28}(6):642--658,
  July 2013.

\bibitem{Shannon1949}
C.~E. Shannon and W.~Weaver.
\newblock {\em {The mathematical theory of communication}}.
\newblock {University of Illinois Press}, {Urbana}, 1949.

\bibitem{Smith2009}
T.~J. Smith and J.~M. Henderson.
\newblock {Facilitation of return during scene viewing}.
\newblock {\em {Visual Cognition}}, {17}(6-7):1083--1108, 2009.

\bibitem{Tatler2005}
B.~W. Tatler, R.~J. Baddeley, and I.~D. Gilchrist.
\newblock {Visual correlates of fixation selection: effects of scale and time}.
\newblock {\em {Vision Research}}, {45}(5):643--659, Mar. 2005.

\bibitem{Tatler2011}
B.~W. Tatler, M.~M. Hayhoe, M.~F. Land, and D.~H. Ballard.
\newblock {Eye guidance in natural vision: Reinterpreting salience}.
\newblock {\em {Journal of Vision}}, {11}(5):5, 2011.

\bibitem{Tatler2008}
B.~W. Tatler and B.~T. Vincent.
\newblock {Systematic tendencies in scene viewing}.
\newblock {\em {Journal of Eye Movement Research}}, {2}(2):1--18, 2008.

\bibitem{Tatler2009}
B.~W. Tatler and B.~T. Vincent.
\newblock {The prominence of behavioural biases in eye guidance}.
\newblock {\em {Visual Cognition}}, {17}(6-7):1029--1054, Aug. 2009.

\bibitem{Torralba2006}
A.~Torralba, A.~Oliva, M.~S. Castelhano, and J.~M. Henderson.
\newblock {Contextual guidance of eye movements and attention in real-world
  scenes: the role of global features in object search.}
\newblock {\em {Psychological Review}}, {113}(4):766--786, 2006.

\bibitem{Vig2014}
E.~Vig, M.~Dorr, and D.~Cox.
\newblock {Large-Scale Optimization of Hierarchical Features for Saliency
  Prediction in Natural Images}.
\newblock {\em {IEEE Computer Vision and Pattern Recognition}}, 2014.

\bibitem{wang2010measuring}
W.~Wang, Y.~Wang, Q.~Huang, and W.~Gao.
\newblock Measuring visual saliency by site entropy rate.
\newblock In {\em Computer Vision and Pattern Recognition (CVPR), 2010 IEEE
  Conference on}, pages 2368--2375. IEEE, 2010.

\bibitem{Wilming2011}
N.~Wilming, T.~Betz, T.~C. Kietzmann, and P.~K\"{o}nig.
\newblock {Measures and limits of models of fixation selection.}
\newblock {\em {PloS one}}, {6}(9):e24038, Jan. 2011.

\bibitem{Zhang2013}
J.~Zhang and S.~Sclaroff.
\newblock {Saliency detection: a boolean map approach}.
\newblock In {\em {Computer Vision (ICCV), 2013 IEEE International Conference
  on}}, pages 153--160. {IEEE}, 2013.

\bibitem{Zhang2014}
J.~Zhang and S.~Sclaroff.
\newblock {Exploit surroundedness for saliency detection: a boolean map
  approach}.
\newblock {\em {submitted to IEEE Trans. Pattern Anlaysis and Machine
  Intellegence (TPAMI)}}, 2014.

\bibitem{Zhang2008}
L.~Zhang, M.~H. Tong, T.~K. Marks, H.~Shan, and G.~W. Cottrell.
\newblock {SUN: A Bayesian framework for saliency using natural statistics}.
\newblock {\em {Journal of Vision}}, {8}(7):32--32, Dec. 2008.

\end{thebibliography}
