A Breakthrough in Astronomical Software Development
Raymond Plante
Doug Roberts
% R.\ M.\ Crutcher\altaffilmark{1}
% and the start of your text. \begin{abstract} % Leave intact % Place the text of your abstract here - NO BLANK LINES In this paper we describe {shape VWhere}, a S-Lang guilet which augments the computational power of {shape where} with the point-click ease of a Gtk-based visual interface. The result is a new mechanism for interacting with datasets, which unifies the constraint evaluation cycle implicit within observational analysis and provides a number of compelling advantages over traditional tool-based methods.
P1-1-17.tex
%APN3_PROCEEDINGS_FORM%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% TEMPLATE.TEX -- APN3 (2003) ASP Conference Proceedings template.
%
% Derived from ADASS VIII (98) ASP Conference Proceedings template
% Updated by N. Manset for ADASS IX (99), F. Primini for ADASS 2000,
% D.Bohlender for ADASS 2001, and H. Payne for ADASS XII and LaTeX2e.
%
% Use this template to create your proceedings paper in LaTeX format
% by following the instructions given below.  Much of the input will
% be enclosed by braces (i.e., { }).  The percent sign, "%", denotes
% the start of a comment; text after it will be ignored by LaTeX.  
% You might also notice in some of the examples below the use of "\ "
% after a period; this prevents LaTeX from interpreting the period as
% the end of a sentence and putting extra space after it.  
% 
% You should check your paper by processing it with LaTeX.  For
% details about how to run LaTeX as well as how to print out the User
% Guide, consult the README file.  You should also consult the sample
% LaTeX papers, sample1.tex and sample2.tex, for examples of including
% figures, html links, special symbols, and other advanced features.
%
% If you do not have access to the LaTeX software or a laser printer
% at your site, you can still prepare your paper following the
% instructions in the User Guide.  In such cases, the editors will
% process the file and make any necessary editorial adjustments.
% 
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% 
\documentclass[11pt,twoside]{article}  % Leave intact
\usepackage{adassconf}

% If you have the old LaTeX 2.09, and not the current LaTeX2e, comment
% out the \documentclass and \usepackage lines above and uncomment
% the following:

%\documentstyle[11pt,twoside,adassconf]{article}

\begin{document}   % Leave intact

%-----------------------------------------------------------------------
%			    Paper ID Code
%-----------------------------------------------------------------------
% Enter the proper paper identification code.  The ID code for your
% paper is the session number associated with your presentation as
% published in the official conference proceedings.  You can           
% find this number locating your abstract in the printed proceedings
% that you received at the meeting or on-line at the conference web
% site; the ID code is the letter/number sequence proceeding the title 
% of your presentation. 
%
% This will not appear in your paper; however, it allows different
% papers in the proceedings to cross-reference each other.  Note that
% you should only have one \paperID, and it should not include a
% trailing period.
%
% EXAMPLE: \paperID{O4-1}
% EXAMPLE: \paperID{P7-7}
%

\paperID{P1-1-17}

%-----------------------------------------------------------------------
%		            Paper Title 
%-----------------------------------------------------------------------
% Enter the title of the paper.
%
% EXAMPLE: \title{A Breakthrough in Astronomical Software Development}
% 
% If your title is so long as to fill the page header when you print it,
% then please supply a short form as a \titlemark.
%
% EXAMPLE: 
%  \title{Rapid Development for Distributed Computing, with Implications
%         for the Virtual Observatory}
%  \titlemark{Rapid Development for Distributed Computing}
%

\title{VWhere: A Visual, Extensible `where' Command}
%\titlemark{ }

%-----------------------------------------------------------------------
%		          Authors of Paper
%-----------------------------------------------------------------------
% Enter the authors followed by their affiliations.  The \author and
% \affil commands may appear multiple times as necessary (see example
% below).  List each author by giving the first name or initials first
% followed by the last name.  Authors with the same affiliations
% should grouped together. 
%
% EXAMPLE: \author{Raymond Plante, Doug Roberts, 
%                  R.\ M.\ Crutcher\altaffilmark{1}}
%          \affil{National Center for Supercomputing Applications, 
%                 University of Illinois Urbana-Champaign, Urbana, IL
%                 61801}
%          \author{Tom Troland}
%          \affil{University of Kentucky}
%
%          \altaffiltext{1}{Astronomy Department, UIUC}
%
% In this example, the first three authors, "Plante", "Roberts", and
% "Crutcher" are affiliated with "NCSA".  "Crutcher" has an alternate 
% affiliation with the "Astronomy Department".  The fourth author,
% "Troland", is affiliated with "University of Kentucky"

\author{Michael S. Noble}
\affil{Center for Space Research, Massachusetts Institute of Technology}

%-----------------------------------------------------------------------
%			 Contact Information
%-----------------------------------------------------------------------
% This information will not appear in the paper but will be used by
% the editors in case you need to be contacted concerning your
% submission.  Enter your name as the contact along with your email
% address.
% 
% EXAMPLE:  \contact{Dennis Crabtree}
%           \email{crabtree@cfht.hawaii.edu}
%

\contact{Michael S. Noble}
\email{mnoble@space.mit.edu}

%-----------------------------------------------------------------------
%		      Author Index Specification
%-----------------------------------------------------------------------
% Specify how each author name should appear in the author index.  The 
% \paindex{ } should be used to indicate the primary author, and the
% \aindex for all other co-authors.  You MUST use the following
% syntax: 
%
% SYNTAX:  \aindex{Lastname, F. M.}
% 
% where F is the first initial and M is the second initial (if
% used).  This guarantees that authors that appear in multiple papers
% will appear only once in the author index.  
%
% EXAMPLE: \paindex{Crabtree, D.}
%          \aindex{Manset, N.}        
%          \aindex{Veillet, C.}        
%
% NOTE: this information is also used to build the author list that
% appears in the table of contents.  Authors will be listed in the order
% of the \paindex and \aindex commmands.
%

\paindex{Noble, M.}

%-----------------------------------------------------------------------
%		      Author list for page header	
%-----------------------------------------------------------------------
% Please supply a list of author last names for the page header. in
% one of these formats:
%
% EXAMPLES:
% \authormark{Lastname}
% \authormark{Lastname1 \& Lastname2}
% \authormark{Lastname1, Lastname2, ... \& LastnameN}
% \authormark{Lastname et al.}
%
% Use the "et al." form in the case of seven or more authors, or if
% the preferred form is too long to fit in the header.

\authormark{Noble}

%-----------------------------------------------------------------------
%			Subject Index keywords
%-----------------------------------------------------------------------
% Enter a comma separated list of up to 6 keywords describing your
% paper.  These will NOT be printed as part of your paper; however,
% they will be used to generate the subject index for the proceedings.
% There is no standard list; however, you can consult the indices
% for past proceedings (http://adass.org/adass/proceedings/).
%
% EXAMPLE:  \keywords{visualization, astronomy: radio, parallel
%                     computing, AIPS++, Galactic Center}
%
% In this example, the author noticed that "radio astronomy" appeared
% in the ADASS VII Index as "astronomy" being the major keyword and
% "radio" as the minor keyword.  The colon is used to introduce another
% level into the index.

\keywords{scripting, S-Lang, module, rapid development, guilet,
   			scientific analysis}

%-----------------------------------------------------------------------
%			       Abstract
%-----------------------------------------------------------------------
% Type abstract in the space below.  Consult the User Guide and Latex
% Information file for a list of supported macros (e.g. for typesetting 
% special symbols). Do not leave a blank line between \begin{abstract} 
% and the start of your text.

\begin{abstract}          % Leave intact
% Place the text of your abstract here - NO BLANK LINES
In this paper we describe {\itshape VWhere}, a S-Lang guilet which augments the
computational power of {\itshape where} with the point-click ease of a
Gtk-based visual interface.  The result is a new mechanism for interacting
with datasets, which unifies the constraint evaluation cycle implicit within
observational analysis and provides a number of compelling advantages
over traditional tool-based methods.
\end{abstract}

%-----------------------------------------------------------------------
%			      Main Body
%-----------------------------------------------------------------------
% Place the text for the main body of the paper here.  You should use
% the \section command to label the various sections; use of
% \subsection is optional.  Significant words in section titles should
% be capitalized.  Sections and subsections will be numbered
% automatically. 
%
% EXAMPLE:  \section{Introduction}
%           ...
%           \subsection{Our View of the World}
%           ...
%           \section{A New Approach}
%
% It is recommended that you look at the sample papers, sample1.tex
% and sample2.tex, for examples for formatting references, footnotes,
% figures, equations, html links, lists, and other special features.  
\section{The Problem: Exploratory Analysis}
The act of working raw data into a form from which its innate properties
may be more readily discerned, referred to here as {\itshape exploratory
analysis}, is of fundamental importance to scientific inquiry.  For example,
from an astrophysical dataset {\itshape \bfseries D}  one may wish to examine events within
some region $\mathcal{R}$, after time $\mathcal{T}$, but only if they originated
from detectors $\mathcal{I}$ and $\mathcal{J}$, and were above energy 
$\mathcal{E}_{0}$ but below energy $\mathcal{E}_{1}$.
Such constraint sets {\itshape \bfseries C} = \{$C_{0}, ..., C_{i}$\} are
derived by
iterating through ``{\itshape What If?}'' cycles, modeled in the large as
\begin{figure}[h]
\epsscale{.78}
\plotone{P1-1-17_1.eps}
\end{figure}\\
Stages 
{\hspace{0.05in}\begin{picture}(10,10) \put(3,3){\circle{13}}\put(-2.7,0){\small S1} \end{picture} \hspace{-0.10in}}\
and 
{\hspace{0.05in}\begin{picture}(10,10) \put(3,3){\circle{13}}\put(-2.7,0){\small S3} \end{picture} \hspace{-0.10in}}\
may involve numerical and visualization tasks, such as
fitting models and plotting residuals.  Stage
{\hspace{0.05in}\begin{picture}(10,10) \put(3,3){\circle{13}}\put(-2.7,0){\small S2} \end{picture} \hspace{-0.10in}}\
may be manifested
by applying filters to columns and images, or grouping, binning, et cetera.
This process plays out whether {\itshape \bfseries D} is a single observation or multiple
observations spanning a number of wavelengths and telescopes.
\subsection{A Traditional Solution: Command Line Tools}
Many existing astronomical analysis systems support data exploration via
some combination of command line tools and interactive applications.  The
tools in this model operate as file transformers ({\itshape file in ---$>$
file out}), and characterize the exploration process as one in which
{\hspace{0.05in}\begin{picture}(10,10) \put(3,3){\circle{13}}\put(-2.7,0){\small S1} \end{picture},
{\hspace{0.05in}\begin{picture}(10,10) \put(3,3){\circle{13}}\put(-2.7,0){\small S2} \end{picture}}, and
{\hspace{0.05in}\begin{picture}(10,10) \put(3,3){\circle{13}}\put(-2.7,0){\small S3} \end{picture}}
are performed by\\\\
   {\hspace*{0.3in} $\bullet$} multiple invocations of distinct programs (e.g.
	 \verb+fcopy/dmcopy/XSpec+)\\
   {\hspace*{0.3in} $\bullet$} requiring many passes over files in {\itshape \bfseries D} \\
   {\hspace*{0.3in} $\bullet$} during which a number of intermediate files are produced \\\\
%   {\hspace*{0.3in} $\bullet$} for consumption within subsequent iterations of the cycle\\\\
\normalsize
While this approach certainly works, and provides bookkeeping advantages if
the tools record their operation (e.g. as comments in a FITS header), it is
clumsy for ad-hoc inquiry and generates file litter.  It is also needlessly
consumptive of
time and computational resources (at times enormously so, as corroborated
by Davis et al, \adassxiv).  Moreover, it is easy to mistakenly use the
filtering syntax of one system when executing tools from another.  Finally,
traditional tools are static in function, incapable of being selectively
extended by the end-user via the importation of a module or script.
\subsection{A New Solution: Interactive `where' Command}
To anyone who has analyzed data within array-oriented systems 
such as IDL(tm), CIAO/ISIS, or PyRAF, the {\itshape where} command
should need little introduction.
Couched in intuitive syntax, it provides a
powerful mechanism for filtering arrays via arbitrary expressions.  For
example, consider plotting from an event list all photons whose position
(specified by X, Y arrays) fell within a circle of radius 10 centered at
the origin
\begin{verbatim}
              isis> i = where( X^2 + Y^2 < 10);
              isis> plot(X[i], Y[i]);
\end{verbatim}
The {\itshape VWhere}\ guilet detailed herein augments this computational
power with a graphical interface that permits constraints to be specified,
manipulated, and evaluated visually.  In this model constraint sets are
represented as a combination of plots and the region filters cumulatively
applied to them.  Plots are generated from the axes of a dataset
{\itshape \bfseries D} resident in memory, or from new axes derived by
arbitrary analytic combinations of them, all without incurring any additional
I/O overhead or tool execution costs per iteration.
\section{Exploring Cygnus X-1}
Let us now walk through portions of a published research scenario, in which
{\itshape VWhere}\ was
used to correlate two observations (from Chandra and the RXTE All-Sky Monitor)
of black hole candidate Cygnus X-1. We begin by loading {\itshape VWhere}\ into ISIS, done explicitly here to underscore the fact that it is
\begin{verbatim}
              isis> require("vwhere");
\end{verbatim}
{\itshape not} linked directly into ISIS but is rather a standalone module
that can be loaded at runtime into {\itshape any} application which embeds
the S-Lang interpreter.
To maximize benefit to the community, while still promoting freedom of choice,
we've also begun to investigate means by which S-Lang modules may be
utilized within other interpreters, such as Tcl or Python.
After loading the data into a structure (not shown) we launch {\itshape VWhere}\
\begin{verbatim}
              isis> indices = vwhere(data);
\end{verbatim}
and generate a lightcurve by plotting 3 fields: {\tt time} versus
{\tt lchan + mchan}.
Our objective is to look for times where Cyg X-1 is emitting harder X-rays
during low count states.
\begin{figure}[h]
\epsscale{.9}
\plottwo{P1-1-17_2.eps} {P1-1-17_3.eps}
\caption{A lightcurve for Cygnus X-1, and its axis expressions.}
\end{figure}
Each such plot is generated in its own tabbed subwindow, and may be
panned, zoomed, deleted, printed, or
cosmetically altered in many of the usual manners one would expect within a
plotting GUI.  Arbitrarily complex filters may then be constructed by laying
regions upon plots.  Pressing {\itshape Done} returns to the
caller a list of indices representing all points selected by the
applied filters (the same semantics as {\itshape where}).

Notice that the lightcurve above exhibits a good deal of variability.  The
color intensity diagram in the first panel of Figure \ref{fig2} shows that
as count rate drops (lower X value) CygX-1 emission is getting harder (smaller
Y value), except for a spike at the very lowest rates.  So we select points
within that spike, by applying a polygonal region, and look at the lightcurve
again in the second panel.  The points filtered away are drawn
grayed out, revealing that the hard, low count-rate selection comes from a
deep dip near the end of the lightcurve.
\begin{figure}
\epsscale{.80}
\plottwo{P1-1-17_4.eps} {P1-1-17_5.eps}
\caption{A polygon filter applied to the color intensity diagram, and its incremental effect upon the original lightcurve.}
\label{fig2}
\end{figure}
\section{Benefit Analysis}
\subsection{Unifying the Constraint Cycle}
This example demonstrates how {\itshape incremental filtering} in
{\itshape VWhere}\ unifies stages
{\hspace*{0.05in}\begin{picture}(10,10) \put(3,3){\circle{13}}\put(-2.7,0){\small S1} \end{picture},
{\hspace{0.05in}\begin{picture}(10,10) \put(3,3){\circle{13}}\put(-2.7,0){\small S2} \end{picture}}, and
{\hspace{0.05in}\begin{picture}(10,10) \put(3,3){\circle{13}}\put(-2.7,0){\small S3} \end{picture}}
of the constraint evaluation cycle, dramatically shrinking the time and effort
required to discern subtle patterns within data, and without any package-specific filtering syntax.  With {\itshape VWhere}\ it is instantly clear how cuts
applied to portions of a dataset, or new axes derived from it, affect other
portions.
As a result, the process of constructing constraint sets is considerably more
fluid.  It is also far more powerful, since by virtue of the extensibility
endowed by S-Lang {\itshape VWhere}\ may import a wide range of C, C++, or
FORTRAN modules, and rather easily using the 
\htmladdnormallinkfoot{SLIRP code generator}{http://space.mit.edu/cxc/software/slang/modules/slirp/}. This is simply not feasible with traditional tools.
Finally, since {\itshape VWhere}\ operates directly upon arrays already
loaded within its host application, memory is conserved while I/O overhead
and runtime performance are improved by many factors.

\subsection{Real Modularity}{
Drawing from our experience as both user and developer of major
astronomical packages, we note a tendency in extant systems whereby
algorithms or libraries are internally coupled in ways which sharply 
curtail their use elsewhere.  Such systems, while open in the superficial
sense that sourcecode is
publicly available, are in a deeper sense closed by their complex web
of internal dependencies, which accrete to form a monolithic structure
impenetrable to the practicing scientist and difficult to navigate even
for professional programmers.

Potential users must confront the prospect of downloading, sifting
through, and picking apart tens or hundreds of thousands of lines of
software simply to obtain a small library or algorithm.  This effectively
prevents the use of novel {\itshape portions} of a system apart from its
{\itshape whole}.
Conversely, tightly coupled architectures betray a distinct evolutionary
disadvantage, in that they also tend to inhibit the incorporation of
software developed externally to the system.

{\itshape VWhere} is one of the earliest examples of our response to
this problem.  Our methodology shuns monolithic constructions in favor of
modular, extensible components that are orthogonal in function, and which
may either be woven together to form larger systems or used completely
standalone.  Other fruits of this effort are described on our
\htmladdnormallinkfoot{modules page}{http://space.mit.edu/cxc/software/slang/modules/} and in Davis et al (\adassxiv).

\subsection{SLgtk}
As argued elsewhere (e.g. Primini \& Noble, \adassxiv), S-Lang is an excellent
open-source language for scientific scripting, particularly for those with
FORTRAN, C, or IDL experience.  SLgtk augments its core strengths
(e.g. built-in support for array-based numerics) with an importable module
that makes it possible to construct sophisticated graphical interfaces
directly from the S-Lang interpreter.  Most of SLgtk is generated
automatically by SLIRP.

\subsection{Guilets}
A driving force behind the development of SLgtk has been the notion of 
the {\em guilet}, by which we mean visual software of a small, scriptable
nature which may be easily embedded within other applications, even those
with a primarily textual interface.  This fosters the use of graphical
interfaces where they are appropriate and beneficial, without committing
the entire application to such.

In code size and development cycle the typical guilet is considerably smaller
than traditional GUIs (applications designed with a primarily visual interface,
coded in a compiled language and explicitly utilizing low-level toolkits
such as Xt, Motif, or the Windows Foundation Classes).  Moreover, since
guilets are scripts they are more amenable to changing requirements, rapid
feature evolution, and even
customization by the end-user.  This can be vital to scientists\\
\begin{itemize}
\item interested in advanced analysis not yet supported by existing systems
\item or with proprietary periods who are unable to wait months for upgrades
\item or who find it advantageous to concoct working prototypes for
developers, to either supplement or supplant paper specifications\\
\end{itemize}
In point of fact, {\itshape VWhere} provides modern analysis and plotting
capability in only 1650 lines of scripts, and has been enhanced on numerous
occasions with turnaround times as short as 2 hours.
\section{Summary}
This paper describes {\em VWhere}, which extends the S-Lang {\em where}
command with a Gtk-based visual interface.  We have argued that by
integrating the specification, accumulation, and visualization of
analytical constraints {\em VWhere} presents a facile mechanism for
exploring data, one which eclipses tool-based methods for such,
particularly in terms of performance, extensibility, and ease of use.
\acknowledgments
This work was supported in part by the Chandra X-Ray Center contract
SV1-61010 from the Smithsonian Institution.  The author would like to thank
colleagues at MIT and the Harvard-Smithsonian CFA (especially Michael Nowak),
as well as those within the open source community, for various contributions
and commentary.

%-----------------------------------------------------------------------
%			      References
%-----------------------------------------------------------------------
% List your references below within the reference environment
% (i.e. between the \begin{references} and \end{references} tags).
% Each new reference should begin with a \reference command which sets
% up the proper indentation.  Observe the following order when listing
% bibliographical information for each reference:  author name(s),
% publication year, journal name, volume, and page number for
% articles.  Note that many journal names are available as macros; see
% the User Guide listing "macro-ized" journals.   
%
% EXAMPLE:  \reference Hagiwara, K., \& Zeppenfeld, D.\  1986, 
%                Nucl.Phys., 274, 1
%           \reference H\'enon, M.\  1961, Ann.d'Ap., 24, 369
%           \reference King, I.\ R.\  1966, \aj, 71, 276
%           \reference King, I.\ R.\  1975, in Dynamics of Stellar 
%                Systems, ed.\ A.\ Hayli (Dordrecht: Reidel), 99
%           \reference Tody, D.\  1998, \adassvii, 146
%           \reference Zacharias, N.\ \& Zacharias, M.\ 2003,
%                \adassxii, \paperref{P7.6}
% 
% Note the following tricks used in the example above:
%
%   o  \& is used to format an ampersand symbol (&).
%   o  \'e puts an accent agu over the letter e.  See the User Guide
%      and the sample files for details on formatting special
%      characters.  
%   o  "\ " after a period prevents LaTeX from interpreting the period 
%      as an end of a sentence.
%   o  \aj is a macro that expands to "Astron. J."  See the User Guide
%      for a full list of journal macros
%   o  \adassvii is a macro that expands to the full title, editor,
%      and publishing information for the ADASS VII conference
%      proceedings.  Such macros are defined for ADASS conferences I
%      through XI.
%   o  When referencing a paper in the current volume, use the
%      \adassxii and \paperref macros.  The argument to \paperref is
%      the paper ID code for the paper you are referencing.  See the 
%      note in the "Paper ID Code" section above for details on how to 
%      determine the paper ID code for the paper you reference.  
%
%\begin{references}
%\end{references}

% Do not place any material after the references section

\end{document}  % Leave intact
