%\renewcommand{\listalgorithmname}{List of \ALG@name s}
%\makeatother
\usepackage{subcaption}
\usepackage{subcaption}
\graphicspath{{../figures/}}
\graphicspath{{../figures/}}
...
@@ -85,6 +98,8 @@
...
@@ -85,6 +98,8 @@
\tableofcontents
\tableofcontents
%\listofalgorithms
\begin{abstract}
\begin{abstract}
This document presents the implementations of RL in pseudocode level. First, I present most of the nomenclature used in these notes. Then I proceed to give my personal views and comments on the motivation behind Selective labels paper. In chapter 2, I define the framework for this problem and give the required definitions. In the following sections, I present the data generating algorithms and algorithms for obtaining failure rates using different methods. Finally in the last section, I present results using multiple different settings.
This document presents the implementations of RL in pseudocode level. First, I present most of the nomenclature used in these notes. Then I proceed to give my personal views and comments on the motivation behind Selective labels paper. In chapter 2, I define the framework for this problem and give the required definitions. In the following sections, I present the data generating algorithms and algorithms for obtaining failure rates using different methods. Finally in the last section, I present results using multiple different settings.
\end{abstract}
\end{abstract}
...
@@ -230,6 +245,7 @@ Given the above framework, the goal is to create an evaluation algorithm that ca
...
@@ -230,6 +245,7 @@ Given the above framework, the goal is to create an evaluation algorithm that ca
& Y &
& Y &
\end{tikzcd}
\end{tikzcd}
\caption{$\M$}
\caption{$\M$}
\label{fig:dgm}
\end{wrapfigure}
\end{wrapfigure}
\emph{Below is the framework as was written on the whiteboard, then RL presents his own remarks on how he understood this.}
\emph{Below is the framework as was written on the whiteboard, then RL presents his own remarks on how he understood this.}
...
@@ -622,6 +638,181 @@ Given our framework defined in section \ref{sec:framework}, the results presente
...
@@ -622,6 +638,181 @@ Given our framework defined in section \ref{sec:framework}, the results presente
\label{fig:random_predictions}
\label{fig:random_predictions}
\end{figure}
\end{figure}
\section{Modules}
Different types of modules are presented in this section. Summary table is presented last.
\subsection{Data generation modules}
Data generation modules usually take only some generative parameters as input.
\begin{algorithm}[H] % enter the algorithm environment
\caption{Data generation module: "results by threshold" with unobservables}% give the algorithm a caption
%\label{alg:} % and a label for \ref{} commands later in the document
\begin{algorithmic}[1] % enter the algorithmic environment
\REQUIRE Total number of subjects $N_{total},~\beta_X=1,~\beta_Z=1$ and $\beta_W=0.2$.
\ENSURE
\FORALL{$i$ in $1, \ldots, N_{total}$}
\STATE Draw $x_i, z_i$ and $w_i$ from from standard Gaussians independently.
\STATE Set Y to 0 if $P(Y =0| X, Z, W)=\sigma(\beta_XX+\beta_ZZ+\beta_WW)\geq0.5$ and \\to 1 otherwise.
\STATE Attach to data.
\ENDFOR
\RETURN data
\end{algorithmic}
\end{algorithm}
\begin{algorithm}[H] % enter the algorithm environment
\caption{Data generation module: "coin-flip results" with unobservables}% give the algorithm a caption
%\label{alg:} % and a label for \ref{} commands later in the document
\begin{algorithmic}[1] % enter the algorithmic environment
\REQUIRE Total number of subjects $N_{total},~\beta_X=1,~\beta_Z=1$ and $\beta_W=0.2$.
\ENSURE
\FORALL{$i$ in $1, \ldots, N_{total}$}
\STATE Draw $x_i, z_i$ and $w_i$ from from standard Gaussians independently.
\STATE Draw $y_i$ from Bernoulli$(\sigma(\beta_XX+\beta_ZZ+\beta_WW))$.
\STATE Attach to data.
\ENDFOR
\RETURN data
\end{algorithmic}
\end{algorithm}
\subsection{Decider modules}
%For decider modules, input as terms of knowledge and parameters should be as explicitly specified as possible.
\begin{algorithm}[H] % enter the algorithm environment
\caption{Decider module: human judge as specified by Lakkaraju et al.}% give the algorithm a caption
%\label{alg:} % and a label for \ref{} commands later in the document
\begin{algorithmic}[1] % enter the algorithmic environment
\REQUIRE Data with features $X, Z$ of size $N_{total}$, knowledge that both of them affect the outcome Y and that they are independent, $\beta_X=1, \beta_Z=1$.
\ENSURE
\STATE Sample acceptance rates for each M judges from $U(0.1; 0.9)$ and round to tenth decimal place.
\STATE Assign each observation to a judge at random.
\STATE Calculate $P(T=0|X, Z)=\sigma(\beta_XX+\beta_ZZ)$ for each observation and attach to data.
\STATE Sort the data by (1) the judges' and (2) by probabilities $P(T=0|X, Z)$ in descending order.
\STATE\hskip3.0em $\rhd$ Now the most dangerous subjects for each of the judges are at the top.
\STATE If subject belongs to the top $(1-r)\cdot100\%$ of observations assigned to that judge, set $T=0$ else set $T=1$.
\RETURN data with decisions
\end{algorithmic}
\end{algorithm}
\begin{algorithm}[H] % enter the algorithm environment
\caption{Decider module: "coin-flip decisions"}% give the algorithm a caption
%\label{alg:} % and a label for \ref{} commands later in the document
\begin{algorithmic}[1] % enter the algorithmic environment
\REQUIRE Data with features $X, Z$ of size $N_{total}$, knowledge that both of them affect the outcome Y and that they are independent, $\beta_X=1, \beta_Z=1$.
\ENSURE
\FORALL{$i$ in $1, \ldots, N_{total}$}
\STATE Draw $t_i$ from Bernoulli$(\sigma(\beta_XX+\beta_ZZ)))$.
\STATE Attach to data.
\ENDFOR
\RETURN data with decisions
\end{algorithmic}
\end{algorithm}
\subsection{Evaluator modules}
\begin{algorithm}[H] % enter the algorithm environment
\caption{Evaluator module: Contraction algorithm \cite{lakkaraju17}}% give the algorithm a caption
%\label{alg:} % and a label for \ref{} commands later in the document
\begin{algorithmic}[1] % enter the algorithmic environment
\REQUIRE Data $\D$ with properties $\{x_i, t_i, y_i\}$, acceptance rate r, knowledge that X affects Y
\ENSURE
\STATE Split data to a test set and training set.
\STATE Train a predictive model $\B$ on training data.
\STATE Estimate probability scores $\s$ using $\B$ for all observations in test data and attach to test data.
\STATE Let $q$ be the decision-maker with highest acceptance rate in $\D$.
\STATE$\D_q =\{(x, j, t, y)\in\D|j=q\}$
\STATE\hskip3.0em $\rhd$$\D_q$ is the set of all observations judged by $q$
\STATE
\STATE$\RR_q =\{(x, j, t, y)\in\D_q|t=1\}$
\STATE\hskip3.0em $\rhd$$\RR_q$ is the set of observations in $\D_q$ with observed outcome labels
\STATE
\STATE Sort observations in $\RR_q$ in descending order of confidence scores $\s$ and assign to $\RR_q^{sort}$.
\STATE\hskip3.0em $\rhd$ Observations deemed as high risk by the black-box model $\mathcal{B}$ are at the top of this list
\STATE
\STATE Remove the top $[(1.0-r)|\D_q |]-[|\D_q |-|\RR_q |]$ observations of $\RR_q^{sort}$ and call this list $\mathcal{R_B}$
\STATE\hskip3.0em $\rhd$$\mathcal{R_B}$ is the list of observations assigned to $t =1$ by $\mathcal{B}$
