Design and Implementation of Performance Metrics for Evaluation of Assessments Data
\IEEEauthorblockN{Irfan~Ahmed \
%~\IEEEmembership{Member
~IEEE
The objective of this paper is to design performance metrics and respective formulas to quantitatively evaluate the achievement of set objectives and expected outcomes both at the course and program levels. Evaluation is defined as one or more processes for interpreting the data acquired through the assessment processes in order to determine how well the set objectives and outcomes are being attained. Even though assessment processes for accreditation are well documented but existence of an evaluation process is assumed. This paper focuses on evaluation process to provide insights and techniques for data interpretation. It gives a complete evaluation process from the data collection through various assessment methods, performance metrics, to the presentations in the form of tables and graphs. Authors hope that the articulated description of evaluation formulas will help convergence to high quality standard in evaluation process.
dataprocessing_journal_150813.tex

%
% Also note that the "draftcls" or "draftclsnofoot", not "draft", option
% should be used if it is desired that the figures are to be displayed in
% draft mode.
%
%\documentclass[journal,transmag]{IEEEtran}
%\documentclass[peerreview,twocolumn,transmag]{IEEEtran}
\documentclass[journal,twocolumn,transmag]{IEEEtran}
%
% If IEEEtran.cls has not been installed into the LaTeX system files,
% manually specify the path to it like:
% \documentclass[journal]{../sty/IEEEtran}
%\usepackage{}              % list packages between braces
\usepackage{cite}
% *** MATH PACKAGES ***
%
\usepackage[cmex10]{amsmath}
\interdisplaylinepenalty=2500
\usepackage{amssymb}
%\usepackage{natbib}
\usepackage{lscape}
\usepackage[font={small}]{caption}
% *** SPECIALIZED LIST PACKAGES ***
\usepackage{caption}
\usepackage{algorithm}
\usepackage{algorithmic}
% type user-defined commands here
\graphicspath{{./figures/}}
\usepackage{graphicx}
\usepackage{float}
\usepackage{rotating}
%\usepackage{mwe}    % loads »blindtext« and »graphicx«
\usepackage{subfig}
\usepackage{booktabs}
 \usepackage{multirow}
 \usepackage{hyperref}
 \usepackage{url} % for reference section

 %\usepackage{biblatex}
 \newcommand{\otoprule}{\midrule[\heavyrulewidth]}

\setlength{\textfloatsep}{5pt}
% Some very useful LaTeX packages include:
% (uncomment the ones you want to load)





% correct bad hyphenation here
\hyphenation{op-tical net-works semi-conduc-tor}


\begin{document}
%
% paper title
% can use linebreaks \\ within to get better formatting as desired
% Do not put math or special symbols in the title.
%\title{Bare Demo of IEEEtran.cls for \textsc{Transactions on Magnetics}}
\title{Design and Implementation of Performance Metrics for Evaluation of Assessments Data}


% author names and affiliations
% transmag papers use the long conference author name format.

\author{\IEEEauthorblockN{Irfan~Ahmed \,%~\IEEEmembership{Member,~IEEE,}
and
Arif~Bhatti%~\IEEEmembership{Member,~IEEE,}
}
%James Kirk\IEEEauthorrefmark{3},
%Montgomery Scott\IEEEauthorrefmark{3}, and
%Eldon Tyrell\IEEEauthorrefmark{4},~\IEEEmembership{Fellow,~IEEE}}
\IEEEauthorblockA{College of Computers and Information Technology,
Taif University, Taif 21974 KSA}}

%\thanks{Manuscript received December 1, 2012; revised December 27, 2012.
%Corresponding author: M. Shell (email: http://www.michaelshell.org/contact.html).}}



% The paper headers
%\markboth{Journal of \LaTeX\ Class Files,~Vol.~11, No.~4, December~2012}%
%{Shell \MakeLowercase{\textit{et al.}}: Bare Demo of IEEEtran.cls for Journals}
% The only time the second header will appear is for the odd numbered pages
% after the title page when using the twoside option.
%
% *** Note that you probably will NOT want to include the author's ***
% *** name in the headers of peer review papers.                   ***
% You can use \ifCLASSOPTIONpeerreview for conditional compilation here if
% you desire.

\newpage

% for Transactions on Magnetics papers, we must declare the abstract and
% index terms PRIOR to the title within the \IEEEtitleabstractindextext
% IEEEtran command as these need to go into the title area created by
% \maketitle.
% As a general rule, do not put math, special symbols or citations
% in the abstract or keywords.
\IEEEtitleabstractindextext{%
\begin{abstract}
The objective of this paper is to design performance metrics and respective formulas to quantitatively evaluate the achievement of set objectives and expected outcomes both at the course and program levels. Evaluation is defined as one or more processes for interpreting the data acquired through the assessment processes in order to determine how well the set objectives and outcomes are being attained. Even though assessment processes for accreditation are well documented but existence of an evaluation process is assumed. This paper focuses on evaluation process to provide insights and techniques for data interpretation. It gives a complete evaluation process from the data collection through various assessment methods, performance metrics, to the presentations in the form of tables and graphs.  Authors hope that the articulated description of evaluation formulas will help convergence to high quality standard in evaluation process.
\end{abstract}

% Note that keywords are not normally used for peerreview papers.
\begin{IEEEkeywords}
assessment; evaluation; higher education; accreditation; student outcomes; program outcomes.
\end{IEEEkeywords}}

\newpage

% make the title area
\maketitle


% To allow for easy dual compilation without having to reenter the
% abstract/keywords data, the \IEEEtitleabstractindextext text will
% not be used in maketitle, but will appear (i.e., to be "transported")
% here as \IEEEdisplaynontitleabstractindextext when the compsoc
% or transmag modes are not selected <OR> if conference mode is selected
% - because all conference papers position the abstract like regular
% papers do.
\IEEEdisplaynontitleabstractindextext
% \IEEEdisplaynontitleabstractindextext has no effect when using
% compsoc or transmag under a non-conference mode.







% For peer review papers, you can put extra information on the cover
% page as needed:
% \ifCLASSOPTIONpeerreview
% \begin{center} \bfseries EDICS Category: 3-BBND \end{center}
% \fi
%
% For peerreview papers, this IEEEtran command inserts a page break and
% creates the second title. It will be ignored for other modes.
\IEEEpeerreviewmaketitle



\section{Introduction}
% The very first letter is a 2 line initial drop letter followed
% by the rest of the first word in caps.
%
% form to use if the first word consists of a single letter:
% \IEEEPARstart{A}{demo} file is ....
%
% form to use if you need the single drop letter followed by
% normal text (unknown if ever used by IEEE):
% \IEEEPARstart{A}{}demo file is ....
%
% Some journals put the first two words in caps:
% \IEEEPARstart{T}{his demo} file is ....
%
% Here we have the typical use of a "T" for an initial drop letter
% and "HIS" in caps to complete the first word.
\IEEEPARstart{A}{ny} educational program starts with a mission statement, objectives, and the program or student outcomes. Mission statement describes the broad goals of the program. Program educational objectives (PEO) are broad statements to describe what graduates are expected to attain within a few years of graduation.
Student outcomes (SO) describe what students are expected to know and be able to do by the time of graduation. SOs relate to the skills, knowledge, and behaviors that students acquire as they progress through the program \cite{ABET_abet:_2014}.

Assessment is defined as one or more processes that identify, collect, and prepare the data necessary for evaluation.  Evaluation is defined as one or more processes for interpreting the data acquired through the assessment processes in order to determine how well SOs are being attained \cite{ABET_abet:_2014}. The quality assurance of an academic program depends upon the quantitative and qualitative measurements of these PEOs and SOs.
The main component of the quality assurance system is the continuous improvement process. Faculty and students assess the achievement of SOs while advisory board members from industry and academia, program alumni, and employers are the main resource for accessing the PEO achievements. Continuous improvement requires regular documented assessment and evaluation of SOs. These evaluation are used as input to make changes in curriculum, teaching methodologies, and revision of SOs or PEOs.

There are many international academic accreditation bodies establishing standards, guidelines, and criteria to ensure a certain degree of quality assurance and progress towards continuous improvement. Accreditation Board for Engineering and Technology (ABET) \url{(http://www.abet.org)} is a US-based educational programs accreditation body. It is recognized as the worldwide leader in assuring quality and stimulating innovation in applied science, computing, engineering, and technology educations. Its responsibilities include ''organizing and carrying out a comprehensive process of accreditation of pertinent programs leading to degrees, and assisting academic institutions in planning their educational programs'' \cite{ABET_abet:_2014}. At the national level in Saudi Arabia, National Commission for Assessment and Accreditation (NCAAA) \url{(http://ncaaa.org.sa)} is responsible for the quality, effectiveness and continuing improvement in the quality of post secondary education. The presented evaluation methodologies are equally applicable to ABET and NCAAA accreditation processes.

Many authors have published their work on accreditation process, continuous improvement, assessment strategies but there is no work in the literature that focuses on the evaluation of the assessment data at course and program levels.
A complete procedure for ABET accreditation for Engineering programs at Qassim University has been presented in \cite{al-yahya_successful_2013}.
It gives the detailed implementation of the continuous improvement process, effecting major changes in the educational plan, curricular content, facilities, activities, teaching methodologies, and assessment practices. But this paper does not go into the details of evaluation process. In \cite{pierrakos_comprehensive_2013} authors present an assessment plan and continuous improvement process in preparation for ABET in department of engineering at James Madison University. They have introduced course assessment and continuous improvement (CACI) reports at course-level and student outcomes summary report (SOSR) at program-level. This paper shows some sample reports and assessment templates but does not discuss the evaluation process. A web-based tool has been introduced in \cite{reed_hierarchical_2013} for outcome-based open-ended and recursive hierarchical quantitative assessment. This quantitative assessment is used to structure outcomes and measures into a leveled hierarchy, with course outcomes at the bottom and more general objectives at the top. A general curriculum outcome (GCO) layer has been added between course's outcomes and program or student's outcomes.
In \cite{el-ariss_civil_2009} the authors describe the development of an Excel spreadsheet and the associated assessment tools for a technical design course to measure its success and ensure its continuous improvement to meet the requirements of the ABET engineering criteria.
In \cite{kalaani_continuous_2014} both direct and indirect measures are used to collect and analyze data to assess the attainments of the student outcomes. To ensure data integrity, a set of rubrics with benchmarks and performance indicators at both the program and curriculum levels are developed. Each outcome has been assessed for different levels (introductory, beginning, developing, proficient, exemplary) and from different sources.
The article \cite{sriraman_lessons_2013} presents discussions on writing learning outcomes and to assess soft skills in engineering education.
ABET accreditation preparations for construction engineering program at the American University in Cairo are described in \cite{ezeldin_international_2013}. It explains the mapping of department mission and objectives with university mission and objectives, respectively. Then the PEO to SO mapping and SO to course learning outcome (CLO) are provided. The paper \cite{nasir_towards_2011} describes the assessment techniques and the mapping of CLO to SO without the insight of evaluation process.
A case study \cite{Hughes_case_2013} describes the features that contribute to assessment quality at the programme, course and task level. This case study has a particular focus on the ‘technical’ such as task analysis and task relationship patterns.
Another case study \cite{Harris_Implementing_2010} presents a health science program reform and evaluation. It discusses potential for evaluation to establish responsive communication between students, teaching staff and programme administrators, ensuring a match between the intended, implemented and attained curriculum. None of these works however, provide the detail of evaluation metrics and their use in course and program assessment.
\begin{figure}[!htbp]
  \centering
  % Requires \usepackage{graphicx}
  \includegraphics[width=3.5in]{AssessmentProcess}\\
  \caption{Components of direct and indirect approaches in assessment process. Direct approach depends on computation and analytical analysis of collected data}\label{AssessmentProcess}
\end{figure}

This paper presents the evaluation of assessment data obtained from direct and indirect assessments as shown in Fig. \ref{AssessmentProcess}. It gives a complete evaluation process from the data collection through various assessment methods, performance metrics, to the presentations in the form of tables and graphs.  Authors hope that the articulated description of evaluation formulas will help convergence to high quality standard in evaluation process.
%To the best of our knowledge, there is no work in the literature that provides the detail of evaluation metrics and their use in course and program assessment.
The rest of the article is organized as follows: the next section describes the performance metrics formulation, course level and program level evaluation based on performance metrics are explained in section III and IV, analysis and interpretations are discussed in section V and conclusions are drawn in section VI.



% ----------------------------------------------------------------
\section{Performance Metrics Formulation}
Let $A_{i,j,m}$ be the marks obtained by student $m$ in question $j$ of assessment $i$ (homework, assignment, quiz, midterm, or final etc). Here $i$ can take the values, $i=1,2,...,I$, $j$  can take the values $j_{i}=1,2,...,J_{i}$, and $m$ can take the values $m=1,2,...,M$. $I,J_{i},M$ are the total number of assessments, questions in assessment $i$, and the students, respectively.

For quantitative analysis, question is a basic unit of computation for assessment. Average score of question $j$ in assessment $i$ that has $M$ students is given by
\begin{equation}\label{average}
  B_{i,j}=\frac{1}{M}\sum_{m=1}^{M}A_{i,j,m}
\end{equation}
$B_{i,j}$ can be written in vector form as
\begin{equation}\label{vectorB}
  \mathbf{\tilde{B}}=
  \left[
                    \begin{array}{ccccc}
                       \mathbf{B}_{1} & \mathbf{B}_{2} & . & . & \mathbf{B}_{I}
                     \end{array}
  \right]_{1\times L}
\end{equation}
Passing threshold (PT) could be absolute, relative or composite \cite{reed_hierarchical_2013}, such that the PT of question $j$ in assessment $i$ is given by one of the following:
\begin{equation}\label{passingThreshold1}
  PT_{i,j}=\alpha \times Q_{i,j}^{tot}
\end{equation}
\begin{equation}\label{passingThreshold2}
  PT_{i,j}=B_{i,j}
\end{equation}
\begin{equation}\label{passingThreshold}
  PT_{i,j}=\min \{B_{i,j},\alpha \times Q_{i,j}^{tot}\}
\end{equation}
where $Q_{i,j}^{tot}$ is the maximum or total marks of question $j$ in assessment $i$ and $0< \alpha <1$.
The maximum, minimum, standard deviation, and \emph{x-th} percentile of question $j$ in assessment $i$ are calculated as
\begin{eqnarray}
% \nonumber to remove numbering (before each equation)
\nonumber  A_{i,j,max} &=& \max_{\substack{m}} \mathbf{A}_{i,j,:} \\
\nonumber  A_{i,j,min} &=& \min_{\substack{m}} \mathbf{A}_{i,j,:} \\
\nonumber  A_{i,j,std} &=& stdev \mathbf{A}_{i,j,m}\\
\nonumber  A_{i,j,per} &=& percentile(\mathbf{A}_{i,j,:},x)
\end{eqnarray}

Course learning outcomes describe what students are expected to learn in a course. A mapping between CLO and assessment questions is required to compute the attainment of the course CLOs. If a course covers $N$ number of CLOs then $n^{th}$ CLO is written as $CLO_{n}, \quad n=1,2,...N$.
The three dimensional matrix $\mathbf{A}$ is converted into a two dimension matrix $\mathbf{\tilde{A}}$ as
  \begin{equation}\label{vectorA}
  \mathbf{\tilde{A}}=\left[
  \begin{array}{ccccc}
     \mathbf{A}_{1}^{T} & \mathbf{A}_{2}^{T} & . & . & \mathbf{A}_{I}^{T}
   \end{array}
   \right ]
  \end{equation}
  where $\mathbf{A}_{i}^{T}$ is the transpose matrix of $i^{th}$ assessment matrix $\mathbf{A}_{J\times M}$. Matrix $\mathbf{\tilde{A}}$ has the dimension $M\times L$, where $M$ is the total number of students and $L=\sum_{i=1}^{I}J_{i}$ is a total number of questions in all assessments.

CLO to SO mapping matrix is by
\begin{equation}\label{CLOSOmatrix}
\mathbf{CS}=\left[
  \begin{array}{ccccc}
CS_{1,a} & CS_{1,b} & \cdots & CS_{1,k}\\
CS_{2,a} & CS_{2,b} & \ddots & \vdots\\
\vdots & \ddots & \ddots & CS_{N-1,k} \\
CS_{N,a} & \cdots & CS_{N,j} & CS_{N,k}
\end{array}
\right ]
\end{equation}
The matrix element $CS_{n,a}$ is a variable. $CS_{n,a}> 0$ if $CLO_{n}$ maps to SO $a$, otherwise $CS_{n,a}=0$. A non-zero value is a relevance of the CLO to compute the SO. It can take values 1,2,3, for \emph{low, moderate}, and \emph{high} relevance, respectively.\\
Similarly, the CLO to question mapping is given by the following matrix
\begin{equation}\label{CLOquestionMatrix}
\mathbf{CQ}=\left[
  \begin{array}{ccccc}
CQ_{1,1} & CQ_{1,2} & \cdots & CQ_{1,L}\\
CQ_{2,1} & CQ_{2,2} & \ddots &  \vdots\\
\vdots & \ddots & \ddots & CQ_{N-1,L} \\
CQ_{N,1} & \cdots & CQ_{N,L-1} &  CQ_{N,L}
\end{array}
\right ]
\end{equation}
Rows of above matrix contain binary variables $CQ_{n,l}$ that represent the mapping of $n^{th}$ CLO with question $l$, where $l$ maps to $j^{th}$ question of an assessment $i$. $CQ_{n,l}=1$ if $CLO_{n}$ maps to question $l$. \\

%Difference between STDEV and STDEVP in MS EXCEL\\
%STDEV is used when the group of numbers being evaluated are only a partial sampling of the whole population. The denominator for dividing the sum of squared deviations is N-1, where N is the number of observations ( a count of items in the data set ). Technically, subtracting the 1 is referred to as "non-biased."
%
%STDEVP is used when the group of numbers being evaluated is complete - it's the entire population of values. In this case, the 1 is NOT subtracted and the denominator for dividing the sum of squared deviations is simply N itself, the number of observations ( a count of items in the data set ). Technically, this is referred to as "biased". The P in STDEVP stands for "population". Since the data set is not a mere sample, but constituted of ALL the actual values, this standard deviation function can return a more precise result \cite{guzik_standard_????}.

%\subsection{Course Level Performance Metrics}
Student marks in assessment questions are used to compute how well the students have done and what percentage of students have met a certain criteria. Every question contributes to one or more CLOs and every CLO contributes to one or more student outcomes (SO) as shown in \eqref{CLOquestionMatrix} and \eqref{CLOSOmatrix} respectively.
\subsection{CLO Attainment}
This metric is about how well the students have done, in percentage, for each CLO. Attainment of a CLO is derived from the average marks obtained divided by total marks for all questions that maps to the CLO.\\
Let $Q_{i,j}^{tot}$ be the maximum or total marks of question $j$ in assessment $i$. In general, for assessment $i$,
\begin{equation}\label{assessmentMaxMarks}
  \mathbf{Q}_{i}^{tot}=\left[
  \begin{array}{ccccc}
     \mathbf{Q}_{i,1}^{tot} & \mathbf{Q}_{i,2}^{tot} & . & . & \mathbf{Q}_{i,J}^{tot}
   \end{array}
   \right ]
\end{equation}
and
\begin{equation}\label{maxMarks}
  \mathbf{\tilde{Q}}^{tot}=\left[
  \begin{array}{ccccc}
     \mathbf{Q}_{1}^{tot} & \mathbf{Q}_{2}^{tot} & . & . & \mathbf{Q}_{I}^{tot}
   \end{array}
   \right ]_{1\times L}
\end{equation}
%\begin{equation}\label{B10}
%  D_{i,j}=\frac{B_{i,j}}{A_{i,j}^{max}}\times 10
%\end{equation}
Then, the percentage of CLO attainment for $n^{th}$ CLO is given by
\begin{equation}\label{SP_CLO}
  attainmentCLO_{n}[\%]=\frac{\sum_{l=1}^{L}\tilde{B}_{l}*CQ_{n,l}}{\sum_{l=1}^{L}\tilde{Q}_{l}^{tot}*CQ_{n,l}}\times 100
\end{equation}
The operator $\ast$  is used for element-wise multiplication.
\subsection{CLO Weightage Information}
In order to get meaningful results, one should design the CLOs such that there is a uniform distribution of the marks over CLOs in questions to CLO mapping. For example, if a course contains four CLOs then, ideally, each CLO should get $25\%$ weightage.

The ideal case of uniform distribution of marks over the CLOs is seldom realized. In these situations, the CLO weightage information renders a fair picture of $\%$ CLO attainment. The percentage weightage of $n^{th}$ CLO is given by
\begin{equation}\label{CLOweightage}
  WeightageCLO_{n}=\sum_{i}\frac{\sum_{j}^{J_{i}}w(CQ_{n,j})Q_{i,j}^{tot}}{A_{i}^{tot}}\times w(A_{i})
\end{equation}
where $w(CQ_{n,j})$ is the weight of $CLO_{n}$ in question $j$, $w(A_{i})$ is the weight of assessment $i$, and $A_{i}^{tot}$ is the total marks of assessment $i$.
\subsection{Student Achievement per CLO}
Student Achievement per CLO is defined as the percentage of students who are above the expected level as shown in \eqref{passingThreshold}. Expectation or target is a design parameter, one choice of the target could be $\min (B_{i,j},0.7\times Q_{i,j}^{tot})$, i.e., the minimum of the average obtained marks and the $70\%$ of the total marks \cite{reed_hierarchical_2013}.

It counts the number of students who met the criteria by comparing each student marks in a question $j$ of an assessment $i$. If the marks obtained $A_{i,j,m}$ are greater than the passing threshold $PT_{i,j}$, then it increments the counting variable $CPS$ (count pass student) by $1$. Finally, $CPS_{i,j}$ or $CPS_{l}$\footnote{$CPS_{l}$ is a row vector form of $CPS_{i,j}$, similar to \eqref{vectorA} or \eqref{vectorB}} contains number of passed students for each question $j$ in assessment $i$.
Therefore, the average student achievement per CLO is given by
\begin{equation}\label{SA_CLO}
  SA\_CLO_{n}=\frac{\sum_{i=1}^{I}\frac{1}{M_{i}}\sum_{j=1}^{J_{i}}CPS_{i,j}\times CQ_{n,i,j}\times Q_{i,j}^{tot}}{\sum_{i=1}^{I}\sum_{j=1}^{J_{i}}Q_{i,j}^{tot}\times CQ_{n,i,j}}
\end{equation}
where $M_{i}$ is the number of students that participated in the assessment $i$.


\subsection{Student Perception of CLO Attainment}
A course survey is conducted at the end of each semester to gauge students' perception of how well the CLOs were covered in the course. It is the average  of CLO perception from the students.
For each CLO, students provide their input on the scale of $1-5$ where $1$ means CLO is not achieved and $5$ means CLO is achieved completely. Summary of responses is given in following matrix.
\begin{equation}\label{SCmatrix}
\mathbf{SC}=  \begin{bmatrix}
  SC_{1,1} & SC_{1,2} & \cdots & SC_{1,N}  \\
  SC_{2,1} & SC_{2,2} & \cdots & \vdots  \\
  \vdots & \ddots & \ddots & SC_{M-1,N}  \\
  SC_{M,1} & \cdots & SC_{M,N-1} & SC_{M,N} \\
  \end{bmatrix}
\end{equation}
Student's perception of $n^{th}$ CLO attainment is given by
\begin{equation}\label{studentCLO}
  SE\_CLO_{n}=\frac{1}{M}\sum_{m=1}^{M}SC_{m,n}
\end{equation}
\subsection{\emph{x-th} Percentile Marks per CLO}
\emph{x-th} Percentile Marks per CLO is defined as the weighted average of $x-th$ percentile marks divided by total marks of the questions that map to particular CLO.\\
Let $xP_{i,j}$ be the \emph{x-th} percentile marks of question $j$ in assessment $i$. In general, for assessment $i$ we have
\begin{equation}\label{assessmentPercentileMarks}
  \mathbf{xP}_{i}=\left[
  \begin{array}{ccccc}
     \mathbf{xP}_{i,1} & \mathbf{xP}_{i,2} & . & . & \mathbf{xP}_{i,J}
   \end{array}
   \right ]
\end{equation}
and
\begin{equation}\label{percentileMarks}
  \mathbf{\tilde{xP}}=\left[
  \begin{array}{ccccc}
     \mathbf{xP}_{1} & \mathbf{xP}_{2} & . & . & \mathbf{xP}_{I}
   \end{array}
   \right ]_{1\times L}
\end{equation}
Then, the average percentage \emph{x-th} percentile marks per CLO is given by
\begin{equation}\label{xPercentile}
  xPercentileCLO_{n}=\frac{\sum_{l=1}^{L}xP_{l}\ast CQ_{n,l}}{\sum_{l=1}^{L}Q_{l}^{tot}\ast CQ_{n,l}}
\end{equation}
\subsection{SO Attainment}
By using CLO-SO mapping in \eqref{CLOSOmatrix}, course level SO assessment can be achieved. SO attainment for an SO is computed from the CLO attainment of all CLOs that map to the SO. SO attainment is defined as the weighted average of CLOs attainment (in \%).
\begin{equation}\label{weightedAvgSOAttain}
  attainmentSO_{n}=\frac{\sum_{i\in \mathcal{C}^{n}}attainmentCLO_{i}w_{i}}{\sum_{i\in \mathcal{C}^{n}}w_{i}}
\end{equation}
where $\mathcal{C}^{n}$ is the set of CLOs that map to $SO_{n}$ and $w_{i}$ is the weight (or relevance) of $i^{th}$ mapping between CLO and SO.
%need to move Arif
%After calculating the CLO attainment, a Table \ref{tab:courseAssessmentMatrixSO} has been created where %every SO that maps to a CLO (according to CLO-SO Table \ref{courseAssessmentMatrix}) takes the same values. %The last row in Table \ref{tab:courseAssessmentMatrixSO} shows relevance of course to particular SO. It is %the ceiling of the average of relevances in a corresponding column of Table \ref{courseAssessmentMatrix}.
\subsection{Student Achievement per SO}
It is defined as weighted average of student achievement of CLOs (in \%) that map to a particular SO.
\begin{equation}\label{weightedAvgSASO}
  SA\_SO_{n}=\frac{\sum_{i\in \mathcal{C}^{n}}SA\_CLO_{i}w_{i}}{\sum_{i\in \mathcal{C}^{n}}w_{i}}
\end{equation}
\subsection{Student Perception of SOs Attainment}
Student perception of SOs attainment gives an indirect measurement of SO attainment. This metric is derived from student perception of CLOs attainment in \eqref{studentCLO} and the CLO-SO mapping in \eqref{CLOSOmatrix}.
\begin{equation}\label{weightedAvgSESO}
  SE\_SO_{n}=\frac{\sum_{i\in \mathcal{C}^{n}}SE\_CLO_{i}w_{i}}{\sum_{i\in \mathcal{C}^{n}}w_{i}}
\end{equation}
\subsection{\emph{x-th} Percentile per SO}
\emph{x-th} Percentile per SO uses CLO-Average x-th Percentile \% marks with CLO-SO mapping in \eqref{CLOSOmatrix}.
\begin{equation}\label{weightedAvgxPerSO}
  xPercentileSO_{n}=\frac{\sum_{i\in \mathcal{C}^{n}}xPercentileCLO_{i}w_{i}}{\sum_{i\in \mathcal{C}^{n}}w_{i}}
\end{equation}
\begin{table}[!htbp]
%\tiny
  \centering
  \caption{Evaluation of CLOs for the sample course}
  % \begin{tabular}{|c|c|c|c|c|}
    \begin{tabular}{|c| p{1.3cm}| p{1.3cm}| p {1.3cm}| p {1.3cm}|}
    \toprule
      CLO    & Attainment & Achievement & 50th-Percentile & Student perception\\
    \midrule
    1  & 79.08 & 85    & 82.76 & 85\\
    2  & 82.91 & 91.67 & 87.71 & 72.5 \\
    3  & 78.78 & 78.33 & 82.97 & 72.5\\
    4  & 87.62 & 95    & 89.92 & 67.5\\
    5  & 74.18 & 66    & 76.44 & 77.5\\
    6  & 81.94 & 85    & 79.48 & 77.5\\
    \bottomrule
    \end{tabular}%
  \label{tab:directAssessment}%
\end{table}%

\begin{table*}[t]
\tiny
  \centering
  \caption{Basic Variables to compute evaluations metrics for a sample course. CLOs and questions mapping \eqref{CLOquestionMatrix} is shown in first two rows}
    \begin{tabular}{cccccccccccccccccccc}
    \toprule
          & \multicolumn{4}{c}{midterm}   & Quiz1 & Quiz4 & Quiz3 & \multicolumn{5}{c}{HW1}               & Quiz2 & \multicolumn{5}{c}{Final}             & Class participation \\
    \midrule
    \textbf{CLOs Covered} & \textbf{1,2,3} & \textbf{1} & \textbf{2} & \textbf{3,4} & \textbf{1} & \textbf{6} & \textbf{5} & \textbf{2} & \textbf{2} & \textbf{3} & \textbf{3} & \textbf{3} & \textbf{2} & \textbf{5} & \textbf{5} & \textbf{5} & \textbf{6} & \textbf{6} & \textbf{1-6} \\
    \textbf{Question No.} & \textbf{1} & \textbf{2} & \textbf{3} & \textbf{4} & \textbf{1} & \textbf{1} & \textbf{1} & \textbf{1} & \textbf{2} & \textbf{3} & \textbf{4} & \textbf{5} & \textbf{1} & \textbf{1} & \textbf{2} & \textbf{3} & \textbf{4} & \textbf{5} & \textbf{1} \\
    \textbf{Question Marks} & \textbf{8} & \textbf{8} & \textbf{8} & \textbf{8} & \textbf{4} & \textbf{4} & \textbf{4} & \textbf{0.4} & \textbf{0.4} & \textbf{0.4} & \textbf{0.4} & \textbf{0.4} & \textbf{4} & \textbf{5} & \textbf{10} & \textbf{10} & \textbf{10} & \textbf{10} & \textbf{5} \\
    \hline \\
    Actual Average & 5.22  & 6.6   & 6.99  & 6.69  & 3.25  & 3.664 & 3.12  & 0.4   & 0.4   & 0.4   & 0.352 & 0.128 & 3.68  & 1.9   & 8.3   & 7.2   & 6.1   & 9.3   & 4.7 \\
    Passing Threshold (PT) & 5.22  & 5.6   & 5.6   & 5.6   & 2.8   & 2.8   & 2.8   & 0.28  & 0.28  & 0.28  & 0.28  & 0.128 & 2.8   & 1.9   & 7     & 7     & 6.1   & 7     & 3.5 \\
     No. of Students Above PT& 6     & 9     & 10    & 9     & 9     & 10    & 6     & 10    & 10    & 10    & 8     & 4     & 9     & 7     & 6     & 4     & 4     & 10    & 10 \\
    Minimum Marks & 1.2   & 4.2   & 6     & 4.5   & 1.3   & 3.04  & 2.4   & 0.4   & 0.4   & 0.4   & 0.16  & 0     & 2.4   & 0     & 5     & 5     & 1     & 8     & 4.27 \\
    Maximum Marks & 7.35  & 7.5   & 7.5   & 7.5   & 4     & 3.84  & 3.84  & 0.4   & 0.4   & 0.4   & 0.4   & 0.32  & 4     & 4     & 10    & 10    & 10    & 10    & 4.9 \\
    Standard Deviation & 1.78  & 0.95  & 0.62  & 0.92  & 0.78  & 0.25  & 0.47  & 0.00  & 0.00  & 0.00  & 0.10  & 0.12  & 0.44  & 1.37  & 1.95  & 1.60  & 2.91  & 0.64  & 0.21 \\
    50th Percentile Marks & 5.85  & 6.75  & 7.35  & 6.9   & 3.3   & 3.76  & 3.2   & 0.4   & 0.4   & 0.4   & 0.4   & 0.08  & 3.84  & 2     & 9.5   & 6.5   & 5.5   & 9     & 4.79 \\
    \bottomrule
    \end{tabular}%
  \label{tab:basicVariables}%
\end{table*}%
%\twocolumn
%\scriptsize
\begin{table*}[t]
  \centering
  \caption{Mapping of Course Learning Outcomes (CLOs) to Student Outcomes (SOs) \eqref{CLOSOmatrix} as Course Assessment Matrix}\label{courseAssessmentMatrix}
  \begin{tabular}{|c|p {10cm}|c|c|c|c|c|c|c|c|c|c|c|}
\hline
\textbf{CLO} & \textbf{Course Learning Outcomes Description} & \textbf{a} & \textbf{b} & \textbf{c} & \textbf{d} & \textbf{e} & \textbf{f} & \textbf{g} & \textbf{h} & \textbf{i} & \textbf{j} & \textbf{k}\tabularnewline
\hline
\hline
1 & Explain the integrated circuit technology and basic engineering
process steps of CMOS &  &  & 2 &  &  &  &  &  &  &  & \tabularnewline
\hline
2 & Design combinational/sequential logic gates using CMOS
transistor & 2 & 1 &  &  &  &  &  & 1 &  &  & \tabularnewline
\hline
3 & Describe MOS circuit design processes, including CMOS design rules,
symbolic diagrams, and stick diagrams  &  &  & 3 &  &  &  &  &  &  &  & \tabularnewline
\hline
4 & Illustrate the details of layout design and verification, concepts of standard
gate design &  &  &  &  & 2 &  &  &  & 1 &  & \tabularnewline
\hline
5 & Describe the VLSI circuit characterization and perfrommace estimation & 3 & 2 &  &  &  &  &  &  &  &  & 2\tabularnewline
\hline
6 & Analyze and design complex logic gates in standard CMOS technology  & 3 & 3 & 2 &  &  &  &  &  &  &  & \tabularnewline
\hline
\end{tabular}
\end{table*}



\section{Course Level Performance Evaluation}
Direct assessment of an academic program is performed by evaluation of courses in the study plan. If not all, at least a selected subset of the courses is required to find out program's success level. Previous section presented formal formulations of the performance metrics that can be used in course evaluation. This section discusses an implementation of these metrics in evaluation of a sample course. The section starts with setup required for evaluation followed by evaluation results and concludes by discussing issues and concerns.
\subsection{Course Setup for Evaluation}
Course evaluation is computation of performance metrics from basic variable of the course and perform analysis. To compute metrics defined in section II from collected data, each course must have well defined CLOs, CLO to SO mapping as in \eqref{CLOSOmatrix}, CLO to question mapping in each assessment as in \eqref{CLOquestionMatrix}, and passing threshold as defined in \eqref{passingThreshold}.
Table \ref{tab:basicVariables} shows basic variables of a sample course. First two rows show mapping between CLOs and questions for all assessments conducted in the course. Table \ref{courseAssessmentMatrix} shows mapping between CLOs and SOs defined by the course designer. A numeric value in a cell represents a relationship between a CLO and an SO. A value of 1, 2, or 3 indicates that a CLO addresses an SO \emph{slightly, moderately,} or \emph{substantively}. Passing threshold is set to $\min(avg,70\%)$, which is used in computation of student achievement per CLO \eqref{SA_CLO} and student achievement per SO \eqref{weightedAvgSASO}.
% Table generated by Excel2LaTeX from sheet 'Sheet3'
\begin{table*}[t]
\small
  \centering
  \caption{Computation of SO attainment from CLO attainment using table \ref{courseAssessmentMatrix} CLO-SO mapping }\label{tab:courseAssessmentMatrixSO}
  \begin{tabular}{|c|c|c|c|c|c|c|c|c|c|c|c|}
\hline
CLO &  a & b & c & d & e & f & g & h & i & j & k\tabularnewline
\hline
\hline
1 &   &  & 79.08 &  &  &  &  &  &  &  & \tabularnewline
\hline
2 &  82.91 & 82.91 &  &  &  &  &  & 82.91 &  &  & \tabularnewline
\hline
3 &  &  & 78.78 &  &  &  &  &  &  &  & \tabularnewline
\hline
4 &   &  &  &  & 87.62 &  &  &  & 87.62 &  & \tabularnewline
\hline
5 &  74.18 & 74.18 &  &  &  &  &  &  &  &  & 74.18\tabularnewline
\hline
6 &  81.94 & 81.94 & 81.94 &  &  &  &  &  &  &  & \tabularnewline
\hline
\hline
  SO attainment  & 79.68 & 79.68 & 79.94 &  & 87.62 &  &  & 82.91 & 87.62 &  & 74.18 \tabularnewline
\hline
  Weighted SO attainment  & 79.27 & 79.52 & 79.77 &  & 87.62 &  &  & 82.91 & 87.62 &  & 74.18 \tabularnewline
\hline
 Relevance  & 3 & 2 & 3 &  & 2 &  &  & 1 & 1 &  & 2\tabularnewline
\hline
\end{tabular}
\end{table*}
\subsection{Performance Evaluation}
This section presents computed values of metrics defined in section II for the sample course. Table \ref{tab:directAssessment} presents a summary of CLO related metrics and table \ref{tab:courseAssessmentMatrixSO} shows how to compute SO attainments from CLO attainments.
\subsubsection{CLO Attainment}
CLO attainment for the sample single section course is shown in Fig. \ref{CLOs}. CLO attainment quantifies the student attainment level of particular CLO through the percentage marks allocated to that CLO. Since this is a percentage value of average marks obtained in the questions maps to a particular CLO, therefore, it is necessary to either distribute the marks uniformly over the CLOs or give an explicit evidence of CLO to marks ratio.
\begin{figure}[!htbp]
  \centering
  % Requires \usepackage{graphicx}
  \includegraphics[width=3.5in]{CLOs}\\
  \caption{CLO Performance Evaluation}\label{CLOs}
\end{figure}
\subsubsection{CLO Weightage}
The CLO weightage for a sample single section course is shown in Fig. \ref{CLOweightage}. The CLO attainment and student achievement of CLO are based on these weightages.
\begin{figure}[H]
  \centering
  % Requires \usepackage{graphicx}
  \includegraphics[width=2.5in]{CLOweightage}\\
  \caption{CLOs Weightage}\label{CLOweightage}
\end{figure}
\subsubsection{Student Achievement per CLO}
Student achievement per CLO for the sample single section course is shown in Fig. \ref{CLOs}. It is the percentage number of students that meet or exceed the target or expectations. There is an upper limit for the target ($70\%$) but there is no lower limit and it depends upon the average marks. We can get absolute student achievement by fixing the target, for example, with target value of $60\%$.

\subsubsection{\emph{50-th} Percentile per CLO}
The \emph{50-th } percentile for the sample course is shown in Fig. \ref{CLOs}. It shows the percentage median marks for each CLO.

\subsubsection{Student Perception of CLOs Attainment}
 For each CLO, student perception of CLO attainment can be on the scale of \textbf{1} to \textbf{5}, 1 mean \emph{strongly disagree} to 5 for \emph{strongly agree}. Student perception of CLO attainment for the sample course is shown in Fig. \ref{CLOs}. There were 10 students but 8 participated in the course survey.

\subsubsection{SO Attainment}
Bar graphs for SO attainment are shown in Fig. \ref{SOs}. These levels are averages of CLO attainments that map to particular SO, therefore, the health of CLO attainments and CLO-SO mapping is critical.

\begin{figure}[!htbp]
  \centering
  % Requires \usepackage{graphicx}
  \includegraphics[width=3.5in]{SOs}\\
  \caption{SO Performance Evaluation}\label{SOs}
\end{figure}

\subsubsection{Student Achievement per SO}
Student achievement for the sample course is given %in below Table \ref{tab:studentAchievementSO} and bar graphs are shown
 in Fig. \ref{SOs}. This is a derived value from CLO achievements and depicts the percentage number of students achieved the set target averaged over the CLOs mapped to that SO.
%\begin{table}[!htbp]
%\small
%  \centering
%  \caption{Student Achievement per SO}\label{tab:studentAchievementSO}
%  \begin{tabular}{|c|p {5cm}|c|c|c|c|c|c|c|c|c|c|c|}
%\hline
%S.No & Course Learning Outcome & a & b & c & d & e & f & g & h & i & j & k\tabularnewline
%\hline
%\hline
%1 & Understand the integrated circuit technology and basic engineering
%process steps of CMOS &  &  & 85 &  &  &  &  &  &  &  & \tabularnewline
%\hline
%2 & Know how to design combinational/sequential logic gates using CMOS
%transistor & 91.67 & 91.67 &  &  &  &  &  & 91.67 &  &  & \tabularnewline
%\hline
%3 & Understand MOS circuit design processes, including CMOS design rules,
%symbolic diagrams, and stick diagrams  &  &  & 78.33 &  &  &  &  &  &  &  & \tabularnewline
%\hline
%4 & Learn the details of layout design and verification, concepts of standard
%gate design &  &  &  &  & 95 &  &  &  & 95 &  & \tabularnewline
%\hline
%5 & Learn the VLSI circuit characterization and perfrommace estimation & 66 & 66 &  &  &  &  &  &  &  &  & 74.18\tabularnewline
%\hline
%6 & Analyze and design complex logic gates in standard CMOS technology  & 85 & 85 & 85 &  &  &  &  &  &  &  & \tabularnewline
%\hline
%\hline
% & Average \% SO attainment  & 80.89 & 80.89 & 82.78 &  & 95 &  &  & 91.67 & 95 &  & 66 \tabularnewline
%\hline
%& Weighted Average \% SO attainment  & 79.54 & 79.78 & 82.14 &  & 95 &  &  & 91.67 & 95 &  & 66 \tabularnewline
%\hline
% & Relevance (1-3)[1=Low, 2=Moderate, 3=High]  & 3 & 2 & 3 &  & 2 &  &  & 1 & 1 &  & 2\tabularnewline
%\hline
%\end{tabular}
%\end{table}


%The SO attainment in direct assessment is compared with the student perception of SO attainment through indirect assessment as shown in Fig. \ref{SOattainmentComparison}.
%\begin{table}[!htbp]
%\small
%  \centering
%  \caption{Student Perception of SOs Attainment}\label{tab:studentEvaluationSO}
%  \begin{tabular}{|c|p {5cm}|c|c|c|c|c|c|c|c|c|c|c|}
%\hline
%S.No & Course Learning Outcome & a & b & c & d & e & f & g & h & i & j & k\tabularnewline
%\hline
%\hline
%1 & Understand the integrated circuit technology and basic engineering
%process steps of CMOS &  &  & 85 &  &  &  &  &  &  &  & \tabularnewline
%\hline
%2 & Know how to design combinational/sequential logic gates using CMOS
%transistor & 82.86 & 82.86 &  &  &  &  &  & 82.86 &  &  & \tabularnewline
%\hline
%3 & Understand MOS circuit design processes, including CMOS design rules,
%symbolic diagrams, and stick diagrams  &  &  & 83.33 &  &  &  &  &  &  &  & \tabularnewline
%\hline
%4 & Learn the details of layout design and verification, concepts of standard
%gate design &  &  &  &  & 84 &  &  &  & 84 &  & \tabularnewline
%\hline
%5 & Learn the VLSI circuit characterization and perfrommace estimation & 80 & 80 &  &  &  &  &  &  &  &  & 74.18\tabularnewline
%\hline
%6 & Analyze and design complex logic gates in standard CMOS technology  & 86.67 & 86.67 & 86.67 &  &  &  &  &  &  &  & \tabularnewline
%\hline
%\hline
% & Average \% SO attainment  & 83.18 & 83.18 & 85 &  & 84 &  &  & 82.86 & 84 &  & 80 \tabularnewline
%\hline
%& Weighted Average \% SO attainment  & 83.22 & 83.81 & 84.76 &  & 84 &  &  & 82.86 & 84 &  & 80 \tabularnewline
%\hline
% & Relevance (1-3)[1=Low, 2=Moderate, 3=High]  & 3 & 2 & 3 &  & 2 &  &  & 1 & 1 &  & 2\tabularnewline
%\hline
%\end{tabular}
%\end{table}

%\begin{figure}[!htbp]
%  \centering
%  % Requires \usepackage{graphicx}
%  \includegraphics[width=3in]{SOattainmentComparison}\\
%  \caption{Comparison of SO attainment with students' perception of SO attainment}\label{SOattainmentComparison}
%\end{figure}
\subsubsection{\emph{50-th} Percentile per SO}
The \emph{50-th} percentile per SO values are derived from \emph{50-th} percentile per CLO. Fig. \ref{SOs} depicts the \emph{50-th} percentile or median marks for each mapped SO.

\subsubsection{Student Perception of SOs Attainment}
The student perception of SOs attainment is shown in Fig. \ref{SOs}. It is an indirect measurement obtained from the course exit survey where students provide their feedback about the CLOs attainment.
%\begin{table}[!htbp]
%\small
%  \centering
%  \caption{\emph{50-th} Percentile per SO}\label{tab:percentileSO}
%  \begin{tabular}{|c|p {5cm}|c|c|c|c|c|c|c|c|c|c|c|}
%\hline
%S.No & Course Learning Outcome & a & b & c & d & e & f & g & h & i & j & k\tabularnewline
%\hline
%\hline
%1 & Understand the integrated circuit technology and basic engineering
%process steps of CMOS &  &  & 82.76 &  &  &  &  &  &  &  & \tabularnewline
%\hline
%2 & Know how to design combinational/sequential logic gates using CMOS
%transistor & 87.71 & 87.71 &  &  &  &  &  & 87.71 &  &  & \tabularnewline
%\hline
%3 & Understand MOS circuit design processes, including CMOS design rules,
%symbolic diagrams, and stick diagrams  &  &  & 82.97 &  &  &  &  &  &  &  & \tabularnewline
%\hline
%4 & Learn the details of layout design and verification, concepts of standard
%gate design &  &  &  &  & 89.92 &  &  &  & 89.92 &  & \tabularnewline
%\hline
%5 & Learn the VLSI circuit characterization and perfrommace estimation & 76.44 & 76.44 &  &  &  &  &  &  &  &  & 74.18\tabularnewline
%\hline
%6 & Analyze and design complex logic gates in standard CMOS technology  & 79.48 & 79.48 & 79.48 &  &  &  &  &  &  &  & \tabularnewline
%\hline
%\hline
% & Average \% SO attainment  & 81.21 & 81.21 & 81.74 &  & 89.92 &  &  & 87.71 & 89.92 &  & 76.44 \tabularnewline
%\hline
%& Weighted Average \% SO attainment  & 80.4 & 79.84 & 81.91 &  & 89.92 &  &  & 87.71 & 89.92 &  & 76.44 \tabularnewline
%\hline
% & Relevance (1-3)[1=Low, 2=Moderate, 3=High]  & 3 & 2 & 3 &  & 2 &  &  & 1 & 1 &  & 2\tabularnewline
%\hline
%\end{tabular}
%\end{table}

%\subsection{Course with Multiple Sections}
%In a single section course, computation of performance metrics as it is done in previous sections, leads to analysis and decision making. For a course with multiple sections, additional computation is required to derive course level performance metrics from section level metrics as shown in Fig. \ref{p1CloAttainment}. Comparison of different sections is required to understand issues related to coordination, consistency, student quality and teaching methodologies.
%
%Course performance metrics derived from weighted average of performance metrics with number of students in each section. \emph{Computer Programming I} course offered in fall 2013 that has six sections has been taken as a sample course. Course assessment matrix for this course is given in Table \ref{tab:CScourseAssessmentMatrix}. The inter-section student retention ratios are compared in Table \ref{tab:studedentsRetentionComparison}. This table gives a snapshot of the number of students enrolled in each section along with the number of withdraw, incomplete, and prohibited students.
%\begin{table}[!htbp]
%  \centering
%  \caption{Course Assessment Matrix}\label{tab:CScourseAssessmentMatrix}
%  \begin{tabular}{|c|p {3cm}|c|c|c|c|c|c|c|c|c|c|c|}
%     \hline
%     % after \\: \hline or \cline{col1-col2} \cline{col3-col4} ...
%     S.NO. & Course Learning Outcome & a & b & c & d & e & f & g & h & i & j & k \\
%     \hline
%     1 & Ability to understand  the basic terminology used in computer programming &  &  &  &  &  &  &  &  & 3 &  &  \\
%     \hline
%     2 & Ability to write, compile and debug programs written in C++ language. & 3 &  &  &  &  &  &  &  &  &  &  \\
%     \hline
%     3 & Ability to use different data types in a computer program. &  &  &  &  &  &  &  &  & 2 &  &  \\
%     \hline
%     4 & Ability to design programs involving decision structures and loops. &  &  & 3 &  &  &  &  &  &  &  &  \\
%     \hline
%   \end{tabular}
%\end{table}
%
%% Table generated by Excel2LaTeX from sheet 'Course Specification'
%\begin{table}[!htbp]
%  \centering
%  \caption{Students Retention Comparison}
%    \begin{tabular}{ccccccccccccc}
%    \toprule
%    \textbf{Section} & \multicolumn{4}{c}{\textbf{Instructor}} & \multicolumn{2}{p {2cm}}{\textbf{Enrolled Students}} & \multicolumn{2}{p{2cm}}{\textbf{Withdraw (W)}} & \multicolumn{2}{p {2cm}}{\textbf{Incomplete (I)}} & \multicolumn{2}{c}{\textbf{Prohibited (M)}} \\
%    \midrule
%    220   & \multicolumn{4}{c}{Instructor A} & \multicolumn{2}{c}{57} & \multicolumn{2}{c}{20} & \multicolumn{2}{c}{0} & \multicolumn{2}{c}{3} \\
%    253   & \multicolumn{4}{c}{Instructor A} & \multicolumn{2}{c}{64} & \multicolumn{2}{c}{19} & \multicolumn{2}{c}{0} & \multicolumn{2}{c}{6} \\
%    286   & \multicolumn{4}{c}{Instructor B} & \multicolumn{2}{c}{44} & \multicolumn{2}{c}{3} & \multicolumn{2}{c}{2} & \multicolumn{2}{c}{3} \\
%    305   & \multicolumn{4}{c}{Instructor B} & \multicolumn{2}{c}{36} & \multicolumn{2}{c}{8} & \multicolumn{2}{c}{1} & \multicolumn{2}{c}{6} \\
%    321   & \multicolumn{4}{c}{Instructor C} & \multicolumn{2}{c}{43} & \multicolumn{2}{c}{0} & \multicolumn{2}{c}{0} & \multicolumn{2}{c}{12} \\
%    332   & \multicolumn{4}{c}{Instructor D} & \multicolumn{2}{c}{41} & \multicolumn{2}{c}{6} & \multicolumn{2}{c}{0} & \multicolumn{2}{c}{0} \\
%    \bottomrule
%    \end{tabular}%
%  \label{tab:studedentsRetentionComparison}%
%\end{table}%
%The percentage grade distribution table is given in table \ref{tab:percentageGradeDistr} and the respective bar graphs are shown in Fig. \ref{gradesDistribution}.
%% Table generated by Excel2LaTeX from sheet 'Course Specification'
%\begin{table}[htbp]
%  \centering
%  \caption{Percentage Grade Distribution}
%    \begin{tabular}{ccccccccccccc}
%    \toprule
%    \textbf{Section} & \textbf{F} & \textbf{D} & \textbf{D+} & \textbf{C} & \textbf{C+} & \textbf{B} & \textbf{B+} & \textbf{A} & \textbf{A+} & \textbf{W} & \textbf{I} & \textbf{M} \\
%    \midrule
%    \textbf{220} & 7.5   & 18.8  & 0.0   & 7.5   & 0.0   & 3.8   & 1.3   & 1.3   & 2.5   & 25.0  & 0.0   & 3.8 \\
%    \textbf{253} & 13.5  & 20.2  & 4.5   & 1.1   & 1.1   & 0.0   & 1.1   & 1.1   & 1.1   & 21.3  & 0.0   & 6.7 \\
%    \textbf{286} & 13.5  & 26.9  & 11.5  & 5.8   & 3.8   & 1.9   & 5.8   & 0.0   & 0.0   & 5.8   & 3.8   & 5.8 \\
%    \textbf{305} & 15.7  & 15.7  & 0.0   & 7.8   & 2.0   & 0.0   & 0.0   & 0.0   & 0.0   & 15.7  & 2.0   & 11.8 \\
%    \textbf{321} & 7.3   & 12.7  & 12.7  & 1.8   & 9.1   & 9.1   & 0.0   & 3.6   & 0.0   & 0.0   & 0.0   & 21.8 \\
%    \textbf{332} & 0.0   & 25.5  & 19.1  & 6.4   & 12.8  & 4.3   & 2.1   & 2.1   & 2.1   & 12.8  & 0.0   & 0.0 \\
%    \hline
%    \textbf{Course} & \textbf{9.9} & \textbf{19.8} & \textbf{7.0} & \textbf{4.8} & \textbf{4.0} & \textbf{2.9} & \textbf{1.6} & \textbf{1.3} & \textbf{1.1} & \textbf{15.0} & \textbf{0.8} & \textbf{8.0} \\
%    \bottomrule
%    \end{tabular}%
%  \label{tab:percentageGradeDistr}%
%\end{table}%
%
%\begin{figure}[!htbp]
%  \centering
%  % Requires \usepackage{graphicx}
%  \includegraphics[width=5in]{gradesDistribution}\\
%  \caption{Grades Distribution in a Multi-Section Course}\label{gradesDistribution}
%\end{figure}
%
%\begin{figure}[!htbp]
%  \centering
%  % Requires \usepackage{graphicx}
%  \includegraphics[width=5in]{p1CloAttainment}\\
%  \caption{Comparison of CLO attainment among course sections and the course itself}\label{p1CloAttainment}
%\end{figure}

%\subsubsection{CLO Attainment}
%\subsubsection{Student Achievement per CLO}
%\subsubsection{Student Evaluation of CLOs}
%\subsubsection{\emph{50-th} Percentile per CLO}
%
%Now, by using CLO-SO mapping in table \ref{courseAssessmentMatrix} we can have course level SO assessment.
%\subsubsection{SO Attainment}
%\subsubsection{Student Achievement per SO}
%\subsubsection{Student Evaluation of SOs}
%\subsubsection{\emph{50-th} Percentile per SO}
\subsection{Issues and Guidelines}
Course designer is responsible to establish quality mapping between CLOs and SOs. Course instructor is responsible for CLOs to questions mapping for all assessments. Quality of these mappings have direct impact on the evaluation results as discussed in rest of this section.
\subsubsection{Relationship of Questions, Marks distribution and CLOs}
It has been observed that questions to CLO mapping requires uniform marks distribution over the CLOs. The quantitative measurement of CLOs provides the baseline data for direct assessment, therefore, questions to CLOs mapping is critical in direct assessment. CLOs should be designed in such a way that they cover all the core topics (qualitative equality) and course assessments should cover all CLOs with uniform marks distribution over the CLOs (quantitative equality). Similar measures are required in capstone project rubrics' design. Capstone project is an important entity of program in which students apply the knowledge gained during the course of the program to solve the engineering problems. The capstone project rubrics map to CLOs and these CLOs usually cover all the SOs. Since the sample size in this assessment is not as large as of direct assessment therefore results may differ in these assessments.

\subsubsection{Questions to CLOs Mapping Approaches}
Due to the many-to-many mapping between questions and CLOs, a common question arises about the weights of a question that maps to multiple CLOs. There are three possibilities:
\begin{itemize}
  \item One-to-many mapping with equal weights
  \item One-to-many mapping with proportional weights
  \item One-to-one mapping between questions and CLOs.
 \end{itemize}
  In this manuscript, equal weights have been used in questions to CLOs mapping. The proportional weights add one more level of complexity for the faculty and hence more chances of errors. One-to-one mapping is another attractive solution which eliminates the weight problem because in this case one question can be mapped to one CLO at most. In this technique many questions can be mapped to one CLO but converse is not possible. Proportional weights and one-to-one schemes require a proper design of CLOs and the mapping table between questions and CLOs.
% Proper weightage among CLOs that contribute to same SO within a course

\subsubsection{CLOs to SOs Mapping within a Course}
There are three choices:
\begin{itemize}
  \item One CLO can be mapped to any number of SOs without weights (one-to-many mapping without weights)
  \item One CLO can be mapped to any number of SOs with weights (one-to-many mapping with weights)
  \item One CLO can be mapped to one SO only (one-to-one mapping) \cite{dargham_linking_2012}
\end{itemize}
In this manuscript, one-to-many mapping with weights has been used as shown in Table \ref{courseAssessmentMatrix}. One-to-many mapping without weights assumes equal weights across all SOs mapped to a particular CLO. A straight forward way of mapping is one-to-one mapping which does not require weights but again the design of CLOs is important in this case.
% Table generated by Excel2LaTeX from sheet 'PEO'


\section{Program Level Performance Evaluation}
Achievement of student outcomes (SO) and program educational objectives (PEO) are the basis for program assessment. PEOs are computed from prgram level SOs and indirect methods of assessment such as graduate exit surveys, alumni surveys, and  employer surveys \cite{pierrakos_comprehensive_2013}.

To measure the achievement level of SOs, direct as well as indirect methods are used as shown in \ref{AssessmentProcess}.
Previous section presents direct assessment approaches and results of SO achievements derived from students' performance in courses. These course level SOs achievements are used to build the program level direct assessment of SOs. Direct method uses a subset of courses in the study plan to compute program level SOs.

\begin{table}[!htbp]
  \centering
  \caption{Relationship between the Program Educational Objectives and Student Outcomes}
    \begin{tabular}{cccccccccccc}
    \toprule
    \multirow{2}[4]{*}{\textbf{PEOs}} & \multicolumn{11}{c}{\textbf{Student Outcomes}} \\
 \cline{2-12}%   \midrule
          & a     & b     & c     & d     & e     & f     & g     & h     & i     & j     & k \\
          \cline{2-12}
    I     & x     & x     & x     & x     & x     &       &       &       &       &       & x \\
    II    & x     & x     & x     &       & x     &       &       & x     &       &       & x \\
    III   &       &       &       & x     &       &       & x     &       &       &       &  \\
    IV    &       &       &       &       &       & x     &       & x     & x     & x     &  \\
    \bottomrule
    \end{tabular}%
  \label{tab:PEOSOmapping}%
\end{table}%


\begin{table*}[htpb]
\centering
\caption{Program Assessment Matrix based on Courses (Substantial: 3, Moderate: 2, Slight: 1, None: blank)}\label{programAssessmentMatrixCourses}
\begin{tabular}{|c|p {5cm}|c|c|c|c|c|c|c|c|c|c|c|}
\hline
\multicolumn{2}{|c|}{ Assessment} & a & b & c & d & e & f & g & h & i & j & k\tabularnewline
\hline
\hline
\multirow{17}{*}{\begin{turn}{90}DIRECT and INDIRECT\end{turn}}
& English &  &  &  &  &  &  & 3 &  & 2 &  & \tabularnewline
\cline{2-13}
 & Calculus I,II,III & 3 &  &  &  &  &  &  &  &  &  &2 \tabularnewline
 \cline{2-13}
 & Linear Algebra & 3 &  &  &  &  &  &  &  &  &  & 2\tabularnewline
 \cline{2-13}
 & Differential Equations & 3 &  &  &  &  &  &  &  &  &  & 2\tabularnewline
\cline{2-13}
 & Probability and Stats & 3 &  &  &  &  &  &  &  &  &  & \tabularnewline
\cline{2-13}
 & Discrete Math & 3 &  &  &  &  &  &  &  &  &  & \tabularnewline
\cline{2-13}
 & Physics I,II & 3 & 3 &  &  &  &  &  &  &  &  & \tabularnewline
 \cline{2-13}
 & Computer Programming I, II & 3 & 3 & 2 &  &  &  &  &  & 2 & 2 & \tabularnewline
 \cline{2-13}
 & Data Structure &  & 2 &  &  &  &  &  &  &  & 3 & \tabularnewline
 \cline{2-13}
 & Operating Systems & 3 & 2 &  &  &  &  &  &  & 2 & 1 & \tabularnewline
 \cline{2-13}
 & Fundamental of Database &  & 3 & 2 &  &  &  &  &  & 2 &  & \tabularnewline
 \cline{2-13}
 & Software Engineering &  & 3 & 2 &  &  &  &  &  & 2 & 2 & \tabularnewline
\cline{2-13}
 & Digital Logic Design & 3 & 3 & 1 &  & 1 &  &  &  &  &  & \tabularnewline
\cline{2-13}
 & Electric Circuits & 3 & 3 & 2 &  & 1 &  &  &  &  &  &3 \tabularnewline
\cline{2-13}
 & Electronics & 3 & 3 &  &  &  &  &  &  &  &  & \tabularnewline
\cline{2-13}
 & Computer Architecture & 2 & 3 & 1 &  &  &  &  &  & 1 & 1 & \tabularnewline
 \cline{2-13}
 & Computer Organization and Design & 1 & 2 & 1 &  &  &  &  & 1 &  & 1 & \tabularnewline
\cline{2-13}
 & Microprocessor & 3 & 2 & 3 &  &  &  &  &  &  &  & 2 \tabularnewline
 \cline{2-13}
 & Professional Ethics &  &  & 2 &  &  & 3 &  &  &  &  & \tabularnewline
\cline{2-13}
 & Embedded Systems & 2 & 2 &  &  & 1 &  &  &  &  &  &1 \tabularnewline
\cline{2-13}
 & Signals and Systems & 3 & 3 & 3 &  & 1 &  &  &  &  &  & \tabularnewline
\cline{2-13}
 & Communication Systems & 3 & 1 & 2 &  &  &  &  &  &  &  & \tabularnewline
\cline{2-13}
 & Digital Signal Processing & 2 & 3 & 2 &  &  &  &  &  &  &  & 1 \tabularnewline
\cline{2-13}
 & Control Engineering & 3 & 3 & 1 &  & 1 &  &  &  &  &  & \tabularnewline
\cline{2-13}
 & VLSI Design & 3 & 2 & 3 &  & 2 &  &  & 1 & 1 &  &2 \tabularnewline
\cline{2-13}
 & Capstone Design Project & 2 & 3 & 3 & 3 & 3 & 3 & 3 & 3 & 3 & 3 & 3\tabularnewline
\hline
\end{tabular}
\end{table*}

\begin{table*}[htpb]
\centering
\caption{Program Assessment Matrix based on Surveys (Substantial: 3, Moderate: 2, Slight: 1, None: blank)}\label{programAssessmentMatrixSurveys}
\begin{tabular}{|c|p {5cm}|c|c|c|c|c|c|c|c|c|c|c|}
\hline
\multicolumn{2}{|c|}{ Assessment} & a & b & c & d & e & f & g & h & i & j & k\tabularnewline
\hline
\hline
\multirow{8}{*}{\begin{turn}{90}INDIRECT\end{turn}}
& Graduate Exit Survey & 2 & 2 & 2 & 2 & 2 & 2 & 2 & 2 & 2 & 2 & 2\tabularnewline
\cline{2-13}
 & Student Course Exit Survey & 2 & 2 & 2 & 2 & 2 & 2 & 2 & 2 & 2 & 2 &2 \tabularnewline
 \cline{2-13}
& Capstone Project Rubrics  & 2 & 2 & 2 & 2 & 2 & 2 & 2 & 2 & 2 & 2 &2 \tabularnewline
\cline{2-13}
 & Course Assessment by Instructor(s)& 2 & 2 & 2 & 2 & 2 & 2 & 2 & 2 & 2 & 2 & 2\tabularnewline
\cline{2-13}
 & Course Assessment by Reviewer(s) & 3 & 3 & 3 & 2 & 2 & 2 & 2 & 2 & 2 & 3 & 3\tabularnewline
\cline{2-13}\\
\cline{2-13}
& Alumni Survey & 1 & 1 & 1 & 1 & 2 & 2 & 2 & 2 & 2 & 3 & 3\tabularnewline
\cline{2-13}
& Employer Survey & 1 & 1 & 1 & 1 & 2 & 2 & 2 & 2 & 2 & 3 & 3\tabularnewline
\cline{2-13}
& Advisory Board Feedback & 1 & 1 & 1 & 1 & 2 & 2 & 2 & 2 & 2 & 3 & 3\tabularnewline
\hline
\end{tabular}
\end{table*}


\subsection{Program Setup}

The program educational objectives are well-defined and published objectives of an institution.
Department of Computer Engineering at Taif University has published the following program educational objectives.\\
Within two to three years of graduation, our Computer Engineering Graduates will:
\begin{description}
  \item[(I)] Be employed as computer engineering professionals in public or private sector and/or perform at a satisfactory level in graduate programs.
  \item[(II)] Possess the necessary knowledge in fundamental theories, techniques and tools to solve computer engineering problems, design hardware/software systems and improve their performance.
  \item[(III)] Demonstrate communication and inter-disciplinary collaboration skills, and leadership qualities.
  \item[(IV)] Engage in lifelong learning activities within a professional, legal and ethical framework.
  \end{description}

A mapping between PEOs and SOs is required to derive PEO achievment values from program level SO values. This mapping should be designed by stakeholders responsible for program assessment. Table \ref{tab:PEOSOmapping} present a sample mapping between PEOs and SOs, which is used in this paper.


 Student outcomes are the defined skills that a student should have; after successfully completing the educational program. An educational program consists of \emph{general knowledge}, \emph{foundation core}, \emph{advanced core}, and \emph{elective} courses. All courses in a program can be used to derive SO achievement levels or a set of witness courses can be selected to ensure that all SO are properly covered. Required advanced courses in the major field of study are obvious candidates for this set. Courses such as mathematics, physics, chemistry, and English might be included as long as they consistently address outcomes. Elective courses or courses whose content varies from one offering to another (so that the outcomes might not be addressed in a particular offering) should not be included \cite{felder_designing_2003}.

Core courses of a program include courses related to the specific program along with math and basic sciences courses. All core courses must have mapping between CLOs and SOs \cite{felder_designing_2003}. Electives courses for which CLOs are not consistent in subsequent offering, should not be considered in program assessment.

A program assessment matrix (PAM) can be constructed as shown in Table \ref{programAssessmentMatrixCourses} for direct assessment based on students' performance in the courses. For courses to SOs mapping, each course gives a relevance number ($1-3$) for each mapped SO in the Table \ref{programAssessmentMatrixCourses}. This is calculated from the weights along each SO column in Table \ref{courseAssessmentMatrix}. It has been observed that program assessment Table \ref{programAssessmentMatrixCourses} should contain the weights for credit hours ($1-4$) along the SOs columns, such that weight is one for one credit hour, two for two credit hours and so on. It will provide more fair values of attainments, achievements, and percentiles.

An identical matrix should also be used for indirect assessment based on course exit survey that shows students' perception of SO achievement. Surveys are indirect assessment methods and a PAM can be created as shown in Table \ref{programAssessmentMatrixSurveys}. First five rows of the table are related to the course offerings in a program while last three row are about perception from the stakeholders in the marketplace about the program itself. Entries of 1, 2, and 3 for each SO in the matrix denote \emph{slight, moderate}, and \emph{substantial} relevance to the program outcomes \cite{ezeldin_international_2013}.
These matrices provide a brief summary of how SOs are assessed when attempting to raise the attainment level of a particular SO.

Program level direct assessments are obtained by putting the weighted average of SO attainment values from all course level SO attainment tables in program assessment Table \ref{programAssessmentMatrixCourses}. For example, the weighted average of SO attainment (second last row) from Table \ref{tab:courseAssessmentMatrixSO} in the corresponding course row of Table \ref{programAssessmentMatrixCourses}. Similar matrix can be used to compute student perception of SOs for each course as indirect assessment. Since capstone project contains both direct and indirect assessment of CLOs and SOs, SO attainment can be directly used here.
%It is critically important to validate the evaluation results to get a level of confidence. This paper uses triangulation method \cite{ghrayeb_art_2011} to validate the convergence to a meaningful conclusion. The data collected for PEO and SO attainment from different sources have been analysed and validated using triangulation method. More specifically, direct assessment, indirect assessment, and hybrid assessment data is used to validate the attainment level of each SO and PEO.

%The entries for a course should be based on a review of
%course materials (syllabi, learning objectives, tests and other assessment
%measures, and the course assessment matrix) conducted
%by a committee that includes all faculty members who teach the course.






\subsection{Performance Evaluation}
There are multiple sources of data to compute the attainment of SOs at program level. Bulk of the data and evidences come from courses offered during the program and students' performance in the courses. For every course students provide their perception on attainment of SOs using course survey. Assessment of capstone graduation project is done directly based on students' performance as well as done indirectly based on rubrics.

Table \ref{tab:programSOattaiment} shows attainment trend of SO \textbf{''a''} from different direct and indirect assessment methods and same data is shown in Fig. \ref{programSOattainment}. The intra-assessment trend for direct assessment shows marginal improvement in three years so as in the case of capstone project rubrics. But there is significant increase in students' perception of SO attainment. This all due to the increasing standards of education quality during the ABET preparation and after the ABET accreditation. The inter-assessments comparison shows that direct assessment levels are constant despite the monotonic increase in students' perception. It exhibits the gradual increase of quality standards, realized by students. The capstone project rubrics assessment have highest values among all assessments which are always anticipated because of small sample sizes and for the graduating students. Attainment of other SOs can be computed and presented similarly.


% Table generated by Excel2LaTeX from sheet 'program assessment'
\begin{table}[!htbp]
  \centering
  \caption{Comparison of SO \textbf{''a''} attainment }
    \begin{tabular}{rrrr}
    \toprule
    \textbf{Assessment type} & \textbf{2011-12} & \textbf{2012-13} & \textbf{2013-14}\\
    \midrule
    Direct assessment & 66.75    & 68.9 & 68.89 \\
    Course exit survey (indirect) & 62.98    & 75.67 & 83.77 \\
    Capstone rubrics (hybrid) & 85.4    & 82.58 & 83.36 \\
    \bottomrule
    \end{tabular}%
  \label{tab:programSOattaiment}%
\end{table}%

\begin{figure}[!htbp]
  \centering
  % Requires \usepackage{graphicx}
  \includegraphics[width=3in]{programSOattainmentR1}\\
  \caption{Comparison of SO \textbf{''a''} attainment}\label{programSOattainment}
\end{figure}


% Table generated by Excel2LaTeX from sheet 'program assessment'
\begin{table}[!htbp]
  \centering
  \caption{Attainment of Program Educational Objective I }
    \begin{tabular}{rrrr}
    \toprule
    \textbf{Assessment type} & \textbf{2011-12} & \textbf{2012-13} & \textbf{2013-14} \\
    \midrule
    Direct assessment & 70    & 68.5 & 67 \\
    Graduate exit survey & 76    & 80 & 83 \\
    Alumni survey & 89    & 86 & 87 \\
    Employer survey & 79    & 85 & 84 \\
    \bottomrule
    \end{tabular}%
  \label{tab:PEOattainment}%
\end{table}%

\begin{figure}[!htbp]
  \centering
  % Requires \usepackage{graphicx}
  \includegraphics[width=3in]{PEOattainmentR1}\\
  \caption{Attainment of Program Educational Objective I }\label{PEOattainment}
\end{figure}
The trend of PEOs attainment can be represented for direct assessment, graduate exit survey, alumni survey, and employer survey. As an example Table \ref{tab:PEOattainment} shows attainment of \textbf{PEO I} and a graph representation is shown in Fig. \ref{PEOattainment}. Similar process can be used to compute and present attainment of PEO II, III, and IV for the program.

\section{Analysis and Interpretations}
This section provides a detail analysis of course and program level evaluations based on the formulated evaluation metrics. The implementation of these metrics have revealed several new directions and interpretations.

\subsection{Analysis and Interpretations: Course level}
Course evaluation produces quantitative values of attainment, achievement, and $x-th$ percentile metrics for CLOs and SOs. For a course, relative values of these metrics provide insight into what happened in the course and zero value for a metric indicates that topics related to the corresponding CLO or SO are either not covered in the course or data was not collected for evaluation. For a multi-section course, these metrics can point to lack of coordination among course instructors, and difference in teaching and evaluation standards,

If a course instructor does not cover some CLOs then corresponding metrics values will be zero as shown in figure \ref{CourseMetrics04} and \ref{CourseMetrics05}. For the sample course, CLO 4 and 5 are not covered and since these two CLOs maps to SO ''a'', so the SO is also not covered in the course. For multi-section courses, a zero value for any of the defined metrics in some of the section indicates lack of coordination among course instructors. Figures \ref{CourseMetrics01} and \ref{CourseMetrics02} points to different teaching and evaluation criteria among different instructors of the same course.

In order to explain the relationship, CLO attainment, student achievement, and \emph{50-th} have been plotted against the students' average marks in Fig. \ref{CLOattenAchPer_trunc}. These graphs show the three evaluation metrics' values for a range of average marks associated with a particular CLO. In this figure, the average marks are obtained from normal distribution  mean (average) with standard deviation $5$. Number of students is $30$ and the results are averaged over $1000$ iterations. From this figure, it can be seen that CLO attainment is a linear function of questions' average marks mapped to that CLO. The attainment is equal to the \emph{50-th} percentile because the mean and median of normal distribution are equal. The composite student achievement remains almost constant up to $70\%$ average marks due to the passing threshold $\min(averageMarks,70\%TotalMarks)$. For normally distributed marks, there are always $50\%$ students below the average value and $50\%$ students are above the average value, hence, student achievement remains constant at $~50\%$ value. When the average marks go above $70\%$, the passing threshold shifts from average value to $70\%$ and all the students with marks greater than $70\%$ contribute to the student achievement. At about $80\%$ average marks, the student achievement reaches to $100\%$ value because all the students even with the standard deviation of $5$ now lie above $70\%$ threshold. The relative achievement always remains around $50\%$. The absolute achievement for $\alpha =0.7$ crosses the $50\%$ value at average marks equal to $70$.


\begin{figure}
  \centering
  % Requires \usepackage{graphicx}
  \includegraphics[width=3in]{CLOattenAch_variousThresholds}\\
  \caption{Comparison of CLO attainment, student achievement, and \emph{50-th} percentile}\label{CLOattenAchPer_trunc}
\end{figure}

The designed evaluation metrics give comprehensive results when considered collectively as shown in figures \ref{CourseMetrics01} to \ref{CourseMetrics05}.
%We define metric level greater than $70\%$ as high, between $40-70\%$ as medium, and below $40\%$ as low.
\begin{enumerate}
  \item \emph{Attainment and Achievement}:
   The relationship between attainment and student achievement (absolute, composite) is linear for average marks greater than set target, i.e., for sufficiently large population size and normal distribution of obtained marks, high values of attainment corresponds to high values of achievements and the low attainment expects low achievement. Attainment and achievement are independent for achievement level below the threshold. If the distribution is not normal then linearity is not guaranteed. For example, if there are $10$ students and $9$ students secure $50$ marks (out of $100$) and one student get $10$ marks then the composite achievement is $90\%$ but the attainment is $46\%$
  \item \emph{Attainment and Percentile}:
   The \emph{50-th} percentile (or median) gives an additional information about the health of attainment. It is also called location parameter. Median close to attainment indicates normal distribution of marks.
  \item \emph{Achievement and Percentile}:
   If \emph{50-th} percentile (median) is equal to the target value of achievement then the achievement is equal to $50\%$. Median values above the the achievement target shows that more students have met the expectation and value of achievement will be high. Median values less than the achievement target results in the achievement level less than $50\%$.
  \item \emph{Attainment, Achievement, and Percentile}:
   Attainment and \emph{50-th} percentile (median) have the same units, i.e., average and median marks in a particular CLO, whereas, student achievement gives the number of students. If attainment and median are similar (normal distribution) and have high values then the absolute and composite achievements will also be high because more number of students would have marks greater than the set target, whereas, the relative achievement will remain flat at $50\%$ because of normally distributed marks. Conversely, if attainment and median have low levels then the composite achievement becomes $~50\%$. Note that the absolute achievement is proportional to the attainment and median near the target and becomes independent for the average marks sufficiently less or greater than the target value.
\end{enumerate}

\begin{figure}[!htbp]
  \centering
  % Requires \usepackage{graphicx}
  \includegraphics[width=3in]{AssessmentDataG1_501123F13}\\
  \caption{Comparison of computed metrics for three sections of the same course for a CLO-SO mapping}\label{CourseMetrics01}
\end{figure}

\begin{figure}[!htbp]
  \centering
  % Requires \usepackage{graphicx}
  \includegraphics[width=3in]{AssessmentDataG2_501220F14}\\
  \caption{Comparison of computed metrics for four sections of the same course for a CLO-SO mapping}\label{CourseMetrics02}
\end{figure}

\begin{figure}[!htbp]
  \centering
  % Requires \usepackage{graphicx}
  \includegraphics[width=3in]{AssessmentDataG3_501220F14_1016_SO}\\
  \caption{Relationship of SO attainment and achievment}\label{CourseMetrics03}
\end{figure}

\begin{figure}[!htbp]
  \centering
  % Requires \usepackage{graphicx}
  \includegraphics[width=3in]{AssessmentDataG4_501243S14_SO}\\
  \caption{Relationship of SO attainment and achievment and coverage of SO in the course}\label{CourseMetrics04}
\end{figure}

\begin{figure}[!htbp]
  \centering
  % Requires \usepackage{graphicx}
  \includegraphics[width=3in]{AssessmentDataG5_501243S14_CLO}\\
  \caption{CLO metrics and coverage of CLO in the course}\label{CourseMetrics05}
\end{figure}




\begin{table}[!htbp]
%\tiny
  \centering
  \caption{Interpretations of attainment, student achievement, and median values. Other combinations are not common}
    \begin{tabular}{p{1cm} p{1cm} p {1cm} p {3cm}}
    \toprule
      Attainment & Student Achievement & 50th-Percentile (Median) & Interpretations \\
    \midrule
    High & High & High & If values are greater than 80, then course and assessments should be reviewed \\
    Low  & High & Low  & Majority have low grades and few have worse; special case of one student with low grade\\
    Low  & Low  & Low  & Majority have low grades and few have good\\
    \bottomrule
    \end{tabular}%
  \label{tab:interpretation}%
\end{table}%

\subsection{Analysis and Interpretations: Program level}
Program assessment depends mainly on the course level direct assessments. Other sources of program assessments are indirect assessments through surveys which complement the results obtained from direct assessments.



SOs and PEOs level obtained from various sources are finally analyzed through statistical tool. Results from two datasets are compared with t-test whereas, more than two datasets are compared with analysis of variance (ANOVA) test and graphical plots \cite{montgomery2009engineering}.
% \subsubsection{Paired t-test}
% t-test is a statistical hypothesis test that asks whether the null hypothesis is rejected or not. The paired t-test takes two samples data as input and compares the mean of the samples difference:
%\begin{equation}\label{ttest}
%  [p,ci,t_{stat}] = ttest(x,y)
%\end{equation}
%The returned value $t_{stat}$ is the test value for the null hypothesis that the data in $x-y$ comes from a normal distribution with mean equal to zero and unknown variance, using the paired-sample t-test. The returned value of $t_{stat}$ is compared with the critical value of $t$ with certain significance level to determine that t-test does or does not reject the null hypothesis at that significance level. Let the null hypothesis is given by $H_{0}:\mu_{x}= \mu_{y}$, and the alternate hypothesis is $H_{a}:\mu_{x}\neq \mu_{y}$,  where $\mu_{x}$ and $\mu_{y}$ are the means of samples $x$ and $y$, respectively.
%Parameter $p$ is a scalar value in the range $[0,1]$, it is the probability of observing a test statistic as extreme as, or more extreme than, the observed value under the null hypothesis. Small values of $p$ increase doubt on the validity of the null hypothesis. %Confidence interval ($ci$) for the mean, returned as a two-element vector containing the lower and upper boundaries of the $100 \times (1 - \alpha)\%$ confidence interval, where $\alpha$ is significance level.
%%Test statistics can returned a structure containing the following:\\
%%df - Degrees of freedom of the test ($n-1$) where $n$ is the total numbers of samples in $x$ and $y$.\\
%%sd - Estimated population standard deviation. For a paired t-test, this is the standard deviation of $x-y$.\\
%For example, direct and indirect assessment results are compared for SO (a) attainment level. In this case, \textbf{x} will be the vector that contains SO (a) attainment levels in direct assessments and the vector \textbf{y} will contain the SO (a) attainment levels in indirect assessments (course exit surveys). Substitution of \textbf{x} and \textbf{y} in \eqref{ttest} gives the following result:
%\begin{eqnarray*}
%% \nonumber to remove numbering (before each equation)
%\nonumber  p &=& 0.9556\\
%%\nonumber  ci &=& -1.0109 \\
%%\nonumber   & & 0.9563 \\
%%\nonumber  stat & &  \\
%\nonumber  t_{stat} &=& -0.0560 \\
%\nonumber  df &=& 36 \\
%%\nonumber  sd &=& 3.1564
%\end{eqnarray*}
%Using the t-table for $5\%$ significance level with $36$ degree of freedom, the critical value $|t_{critical}|$ is $2.028$. Since the obtained $t_{stat}$ value is less than the critical value, therefore this test does not reject the null hypothesis at the $5\%$ significance level, and both measurements have the same means. Large value of $p=0.9556$ indicates strong validity of null hypothesis.
%
%\subsubsection{Analysis of Variance (ANOVA) and F-test}
%The t-test is generally carried out to compare the means of two populations, whereas, analysis of variance (ANOVA) uses variance to cast inference on the means for two or more than two populations \cite{gerstman_analysis_????}. The null and alternate hypotheses for $k$ populations are:
%\begin{eqnarray}\label{anovaHypotheses}
%\nonumber  H_{0}&:&\mu_{1}=\mu_{2}= \cdots =\mu_{k} \\
%\nonumber  H_{a}&:&H_{0} \quad \text{is false}
%\end{eqnarray}
%The difference in means between groups (or populations) depends on the variance within the groups. Widely different means can more likely arise if individual readings within groups vary greatly. Therefore, while assessing differences between groups, we should take into account the variance within group.
%As an example, we compare the direct assessment ($\mathbf{x}$), indirect assessments ($\mathbf{y}$, student course exit survey), and indirect assessment ($\mathbf{z}$, capstone project rubrics) for SO (a). Capstone design project is an important entity of the program because it covers most of the SOs \cite{pierrakos_comprehensive_2013} through Capstone Design Process Rubric, Capstone Report
%Writing Rubric, and Capstone Presentation Rubrics.  Let $n_{x}$, $n_{y}$, and $n_{z}$ denote the size of group $x$, $y$, and $z$, respectively.
%$x (n_{x}=37)$:49,54,49,66,81,71,65,63,59,52,46,69,76,81,66,80,
%61,86,79,71,58,67,43,66,63,66,78,80,74,68,61,68,89,71,72,80,73.
%$y (n_{y}=37)$:51,60,47,68,78,70,62,65,63,54,48,67,72,77,69,78,
%70,80,80,74,60,67,45,66,64,67,75,78,75,70,62,65,80,74,73,78,70.
%$z (n_{z}=12)$:94,85,85,79,84,91,83,85,80,81,83,80.
%We see that $n_{x}=n_{y}\neq n_{z}$ which cannot be handled by multiple paired t-tests because paired t-test requires equal sized populations.
%
%It is recommended to use the non-pooled procedure, however, if the population variances are equal, the pooled test has a slightly better chance of detecting differences.
%%Estimate of the variance based on the combined or “pooled” data is called the pooled variance.
%%If the population sizes are equal and the sample standard deviations are similar, then the pooled variance could be used. But
%%if population sizes are very different, pooled test can be quite misleading unless sample standard deviations are similar.
%When to use pool or un-pooled analysis, depends on the standard deviations of populations. There is a rule of thumb which states that if the larger sample standard deviation is MORE THAN twice the smaller sample standard deviation then perform the analysis using unpooled methods. For the above datasets, the standard deviations are $std_{x}=11.26$, $std_{y}=9.5$, and $std_{z}=4.53$. Since largest to smallest standard deviation ratio $\frac{11.26}{4.53}=2.48$ is $>2$ therefore we use unpooled variance.\\
%\emph{Variance between groups}:\\
%Let $\sigma_{B}^{2}$ represents the variance between groups and is given by ratio of Sum of Squares Between ($SSB$) and degree of freedom ($df_{B}$) as
%\begin{equation}\label{varianceBetween}
%  \sigma_{B}^{2}=\frac{SSB}{df_{B}}=\frac{\sum_{i}n_{i}(\bar{x}_{i}-\bar{x})}{df_{B}}
%\end{equation}
%where $\bar{x}_{i}$ is the mean of group $i,i\in {x,y,z}$ and $\bar{x}$ is overall mean. Using this expression we get, $\sigma_{B}^{2}=\frac{2847}{3-1}=1423$.\\
%\emph{Variance within groups}:\\
%Variance within groups $\sigma_{W}^{2}$ is given by
%\begin{equation}\label{varianceWithin}
%  \sigma_{W}^{2}=\frac{SSW}{df_{W}}=\frac{\sum_{i}(n_{i}-1)\sigma_{i}^{2}}{df_{W}}
%\end{equation}
%where $df=N-k$. The above equation gives $ \sigma_{W}^{2}=96.96$. The ANOVA F statistic is the ratio of $\sigma_{B}^{2}$ and $\sigma_{W}^{2}$, hence, $F=\frac{1423}{96.96}=14.68$. Formally, F statistics is written as $F(df_{B},df_{W},p)=F(2,83,0.95)=14.48$. The $p$ value for the test is represented as the area under $F(2,83,0.95)$ to the right tail of the $F$ density function. Using the F-table, the critical value of $F(2,83)$ at the $p=0.05$ level of significance is $3.11$ with $2$ and $83$ degree of freedom. Since the obtained F-ratio is larger than this value, therefore, it can be concluded that the obtained F-ratio is likely to occur by chance with a $p<.05$. This F-test rejects the null hypothesis, such that there is at least one population with different mean and this is obviously the indirect assessment by capstone project rubrics.
%The critical value of $F(2,83)$ at the $p=0.01$ level of significance is $4.87$ which is again smaller than the obtained F-ratio. Thus, the ANOVA and F-test reject the null hypothesis with $99\%$ level of significance, in other words, the probability of not rejecting the null hypothesis is less than $1\%$.
%\subsubsection{Graphical Analysis}
%A boxplot is a common statistical graphical tool to analyze the locations, spreads, and the medians. A side-by-side boxplot for the above datasets is shown in Fig. \ref{boxPlot}. On each box, the central mark is the median, the edges of the box are the 25th and 75th percentiles, and the whiskers correspond to the most extreme data points. This boxplot shows that the direct assessment has highest dispersion of data due to the fact that direct assessment of SO involves large number of questions in various courses solved by students. The 75th, 50th (median), and 25th percentiles of direct and course exit survey (CES) assessments are more or less same, showing a high degree of confidence on the achieved level of SO (a). The capstone project rubrics (CPR) plot shows a different description of SO (a) attainment. These rubrics are completed by faculty at the end of the capstone project I and II during last year of undergraduate program. This assessment is taken from limited number of students (graduating students) and the faculty generally assign relative grades within the group, therefore, one cannot see wide variation in this assessment. Another notable point is the higher level of attainment in CPR as compare to the direct and CES assessments. One of the major reason of this phenomenon is the small size of students' group who have been passed through almost all the learning phases of their undergraduate studies.
%
%\begin{figure}[!htbp]
%  \centering
%  % Requires \usepackage{graphicx}
%  \includegraphics[width=3in]{boxPlot}\\
%  \caption{Graphical comparison of direct and indirect (course exit survey and capstone project rubrics) assessments}\label{boxPlot}
%\end{figure}

The indirect assessments of SOs and PEOs are very subjective. One cannot compare the SO's and PEO's levels in direct and indirect assessment using the same weights. The indirect assessments, especially, the surveys, require extra efforts in determining the the weights relative to the direct assessment. These weights depend upon the type of employer, quality of advisory board, and the interaction level with alumni. In general, these weights may differ institute by institute and region by region.

\section{Conclusions}
Established universities have assessment and evaluation processes in-place but for less established and newer institutions learning, understanding, and establishing these processes are big challenges. This paper describes a complete evaluation process of assessment data for an educational degree program with the help of various evaluation metrics and quantification approaches of PEOs and SOs attainment levels. The main focus of this paper is the formulation and implementation of performance metrics for CLOs, SOs, and PEOs evaluation. Analysis and interpretation of processed data along with the relationship among questions, CLOs and SOs is presented to give an insight into evaluation process.


%\newpage
\bibliographystyle{ieeetr}
\bibliography{assessbib}
\end{document} 
dataprocessing_journal_150813.bbl
\begin{thebibliography}{10}

\bibitem{ABET_abet:_2014}
``{ABET} vision and mission.'' Online: \url{http://www.abet.org}, Accessed:
  Feb. 3, 2015.

\bibitem{al-yahya_successful_2013}
S.~A. Al-Yahya and M.~A. Abdel-halim, ``A successful experience of {ABET}
  accreditation of an electrical engineering program,'' {\em {IEEE}
  Transactions on Education}, vol.~56, pp.~165--173, May 2013.

\bibitem{pierrakos_comprehensive_2013}
O.~Pierrakos and H.~Watson, ``A comprehensive {ABET}-focused assessment plan
  designed to involve all program faculty,'' in {\em Frontiers in Education
  Conference, 2013 {IEEE}}, pp.~1716--1722, {IEEE}, 2013.

\bibitem{reed_hierarchical_2013}
J.~Reed and H.~Zhang, ``A hierarchical framework for mapping and quantitatively
  assessing program and learning outcomes,'' in {\em Proceedings of the 18th
  {ACM} conference on Innovation and technology in computer science education},
  pp.~52--57, {ACM}, 2013.

\bibitem{el-ariss_civil_2009}
B.~El-Ariss, A.~Sweedan, and K.~El-Sawy, ``Civil engineering course outcome
  assessment,'' in {\em Proceedings of the {ICEE} and {ICEER}}, (Korea),
  pp.~1--9, 2009.

\bibitem{kalaani_continuous_2014}
Y.~Kalaani and R.~J. Haddad, ``Continuous improvement in the assessment process
  of engineering programs,'' in {\em Proceedings of {ASEE} Southeast Section
  Conference}, (Macon, {GA}), Apr. 2014.

\bibitem{sriraman_lessons_2013}
V.~Sriraman and W.~Stapleton, ``Lessons learned in first time accreditation of
  engineering programmes,'' {\em Global Journal of Engineering Education},
  vol.~15, no.~2, 2013.

\bibitem{ezeldin_international_2013}
A.~S. Ezeldin, ``International accreditation for engineering programs: Mission,
  learning objectives and outcomes,'' {\em Procedia - Social and Behavioral
  Sciences}, vol.~102, pp.~267--275, Nov. 2013.

\bibitem{nasir_towards_2011}
K.~A.~M. Nasir and A.~Alghamdi, ``Towards {ABET} accreditation for a {SWE}
  program: Alternative student assessment techniques,'' {\em Science
  International Journal}, vol.~23, no.~1, pp.~7--12, 2011.

\bibitem{Hughes_case_2013}
C.~Hughes, ``A case study of assessment of graduate learning outcomes at the
  programme, course and task level,'' {\em Assessment and Evaluation in Higher
  Education}, vol.~38, no.~4, pp.~492--506, 2013.

\bibitem{Harris_Implementing_2010}
L.~Harris, P.~Driscoll, M.~Lewis, L.~Matthews, C.~Russell, and S.~Cumming,
  ``Implementing curriculum evaluation: case study of a generic undergraduate
  degree in health sciences,'' {\em Assessment and Evaluation in Higher
  Education}, vol.~35, no.~4, pp.~477--490, 2010.

\bibitem{dargham_linking_2012}
J.~A. Dargham, A.~Chekima, S.~Omatu, Y.~Chin, and R.~Ka, ``Linking program
  outcomes to the courses outcomes: A top-down approach,'' in {\em Engineering
  Education ({ICEED}), 2012 4th International Congress on}, pp.~1--3, {IEEE},
  2012.

\bibitem{felder_designing_2003}
R.~M. Felder and R.~Brent, ``Designing and teaching courses to satisfy the
  {ABET} engineering criteria,'' {\em Journal of Engineering Education},
  vol.~92, no.~1, pp.~7--25, 2003.

\bibitem{montgomery2009engineering}
D.~C. Montgomery, G.~C. Runger, and N.~F. Hubele, {\em Engineering statistics}.
\newblock John Wiley \& Sons, 2009.

\end{thebibliography}
