\documentclass[11pt,a4paper]{amsart}
\usepackage[margin=1in]{geometry}                % See geometry.pdf to learn the layout options. There are lots.
%\geometry{a4paper}                   % ... or letterpaper or a5paper or ... 
%\geometry{landscape}                % Activate for for rotated page geometry 
%\usepackage[parfill]{parskip}    % Activate to begin paragraphs with an empty line rather than an indent
\usepackage{graphicx}
\usepackage{amssymb}
\usepackage{epstopdf}
\usepackage[hidelinks, colorlinks=true]{hyperref}
%\DeclareGraphicsRule{.tif}{png}{.png}{`convert #1 `dirname #1`/`basename #1 .tif`.png}

\usepackage[normalem]{ulem} %tables
\useunder{\uline}{\ul}{}
\usepackage{multirow}

%\usepackage[toc,page]{appendix}
%
%\renewcommand\appendixtocname{Appendices -- Mostly old}
%\renewcommand\appendixpagename{Appendices -- Mostly old}

\usepackage{wrapfig} % wrap figures

\usepackage{booktabs}% http://ctan.org/pkg/booktabs
\newcommand{\tabitem}{~~\llap{\textbullet}~~}

\usepackage{pgf}
\usepackage{tikz}
\usepackage{tikz-cd}
\usetikzlibrary{arrows,automata, positioning}

\usepackage{algorithm}% http://ctan.org/pkg/algorithms
\usepackage{algorithmic}% http://ctan.org/pkg/algorithms
\renewcommand{\algorithmicrequire}{\textbf{Input:}}
\renewcommand{\algorithmicensure}{\textbf{Procedure:}}
\renewcommand{\algorithmicreturn}{\textbf{Return}}

\newcommand{\pr}{\mathbb{P}} % tn merkki
\newcommand{\D}{\mathcal{D}} % aineisto
\newcommand{\s}{\mathcal{S}} % "fancy S"
\newcommand{\M}{\mathcal{M}} % "fancy M"
\newcommand{\B}{\mathcal{B}} % "fancy B"
\newcommand{\RR}{\mathcal{R}} % supistusalgon R
\newcommand{\invlogit}{\text{logit}^{-1}} % supistusalgon R

\renewcommand{\descriptionlabel}[1]{\hspace{\labelsep}\textnormal{#1}}

\makeatletter
%Table of Contents
\setcounter{tocdepth}{3}

% Add bold to \section titles in ToC and remove . after numbers
\renewcommand{\tocsection}[3]{%
  \indentlabel{\@ifnotempty{#2}{\bfseries\ignorespaces#1 #2\quad}}\bfseries#3}
% Remove . after numbers in \subsection
\renewcommand{\tocsubsection}[3]{%
  \indentlabel{\@ifnotempty{#2}{\ignorespaces#1 #2\quad}}#3}
%\let\tocsubsubsection\tocsubsection% Update for \subsubsection
%...

\newcommand\@dotsep{4.5}
\def\@tocline#1#2#3#4#5#6#7{\relax
  \ifnum #1>\c@tocdepth % then omit
  \else
    \par \addpenalty\@secpenalty\addvspace{#2}%
    \begingroup \hyphenpenalty\@M
    \@ifempty{#4}{%
      \@tempdima\csname r@tocindent\number#1\endcsname\relax
    }{%
      \@tempdima#4\relax
    }%
    \parindent\z@ \leftskip#3\relax \advance\leftskip\@tempdima\relax
    \rightskip\@pnumwidth plus1em \parfillskip-\@pnumwidth
    #5\leavevmode\hskip-\@tempdima{#6}\nobreak
    \leaders\hbox{$\m@th\mkern \@dotsep mu\hbox{.}\mkern \@dotsep mu$}\hfill
    \nobreak
    \hbox to\@pnumwidth{\@tocpagenum{\ifnum#1=1\bfseries\fi#7}}\par% <-- \bfseries for \section page
    \nobreak
    \endgroup
  \fi}
\AtBeginDocument{%
\expandafter\renewcommand\csname r@tocindent0\endcsname{0pt}
}
\def\l@subsection{\@tocline{2}{0pt}{2.5pc}{5pc}{}}
\makeatother

%\makeatletter
%\def\listofalgorithms{\@starttoc{loa}\listalgorithmname}
%\def\l@algorithm{\@tocline{0}{3pt plus2pt}{0pt}{1.9em}{}}
%\renewcommand{\ALG@name}{AlGoRiThM}
%\renewcommand{\listalgorithmname}{List of \ALG@name s}
%\makeatother

\usepackage{subcaption}
\graphicspath{ {../figures/} }

\title[Potential outcomes in model evaluation]{Notes, working title: \\ From would-have-beens to should-have-beens: Potential outcomes in model evaluation}

\author{RL}
%\date{}                                           % Activate to display a given date or no date

\begin{document}

\maketitle

\tableofcontents

\begin{abstract}
%This document presents the implementations of RL in pseudocode level. First, I present most of the nomenclature used in these notes. Then I proceed to give my personal views and comments on the motivation behind Selective labels paper. In sections \ref{sec:framework} and \ref{sec:modular_framework}, I present two frameworks for the selective labels problems. The latter of these has subsequently been established the current framework, but former is included for documentation purposes. In the following sections, I present the data generating algorithms and algorithms for obtaining failure rates using different methods in the old framework. Results are presented in section \ref{sec:results}. All the algorithms used in the modular framework are presented with motivations in section \ref{sec:modules}. 

For notes of the meetings please refer to the \href{https://helsinkifi-my.sharepoint.com/personal/rikulain_ad_helsinki_fi/Documents/Meeting_Notes.docx?web=1}{{\ul word document}}.

Old discussions have been moved to the appendices.

\end{abstract}

\section*{Terms and abbreviations}

\begin{description}
\item[R :] acceptance rate, leniency of decision maker, $r \in [0, 1]$
\item[X :] personal features, observable to a predictive model
\item[Z :] some features of a subject, unobservable to a predictive model, latent variable
\item[W :] noise added to result variable Y
\item[T :] decision variable, bail/positive decision equal to 1, jail/negative decision equal to 0
\item[Y :] result variable, no crime/positive result equal to 1, crime/negative result equal to 0
\item[MAE :] Mean absolute error
\item[SL :] Selective labels, for more information see Lakkaraju's paper  \cite{lakkaraju17}
\item[Labeled data :] data that has been censored, i.e. if negative decision is given (T=0), then Y is set to NA.
\item[Full data :] data that has all labels available, i.e. \emph{even if} negative decision is given (T=0), Y will still be available.
\item[Unobservables :] unmeasured confounders, latent variables, Z
\end{description}

Mnemonic rule for the binary coding: zero bad (crime or jail), one good!

\section{Introduction}

\setcounter{figure}{-1}
\begin{wrapfigure}{r}{0.3\textwidth} %this figure will be at the right
    \centering
    \begin{tikzpicture}[->,>=stealth',node distance=1.5cm, semithick]

  \tikzstyle{every state}=[fill=none,draw=black,text=black]

  \node[state] (R)                    {$R$};
  \node[state] (X) [right of=R] {$X$};
  \node[state] (T) [below of=X] {$T$};
  \node[state] (Z) [rectangle, right of=X] {$Z$};
  \node[state] (Y) [below of=Z] {$Y$};

  \path (R) edge (T)
        (X) edge (T)
	     edge (Y)
        (Z) edge (T)
	     edge (Y)
        (T) edge (Y);
\end{tikzpicture}
\caption{Initial model.}
\label{fig:initial_model}
\end{wrapfigure}

Complex computational algorithms have already been deployed in numerous fields such as healthcare and justice to make influential decisions. These machine decisions have a great impact on human lives and therefore they should be audited to show that they improve on human decision-making. In the conventional settings, outcome labels are readily available and performance evaluation is trivial and hence numerous metrics have been proposed in the literature (ROC, AUC etc.). In settings as judicial bail decisions, some outcomes can not be observed due to the selective labeling of the data. This results in a complicated missing data problem where the missingness of an item is connected with its outcome and where the available labels are a non-random sample of the underlying population. Approaches addressing this problem can be separated into two major classes: imputation approaches and other approaches.

%In this paper we propose a novel framework for presenting these missing data problems and present an approach for inferring the missing labels to evaluate the performance of predictive models. We use a flexible Bayesian approach and etc. Additionally we show that our method is robust against violations and modifications in the data generating mechanisms.

%\begin{itemize}
%\item Evaluating predictive models is important. Models are even now deployed in the fields such as X, Y and Z to make influential decisions.
%\item Missing data problems ? Multiple kinds, MAR, MNAR, MCAR, what not. In the current setting data missingness is correlated with the outcome. Labeling has been performed by expert decision-makers wanting to avoid negative outcomes. This causes non-random missingness which has been recently been called selective labels/-ing.
%\item Approaches to this problem can roughly be divided into three categories: ignore, impute and other methods. Imputation methods X, Y, Z. Other is contraction presented by Lakkaraju et al which utilises...
%\item Several nuances to this setting a more relaxed setting in Jung et al preprint where counterfactuals were used to construct optimal policies. There wasn't selective labels.
%\item in addition selective labels.
%\item Interest in presenting a method for robust evaluation of predictive algorithms using potential outcomes.
%\end{itemize}

\section{Framework}

$$DG+DEC = DATA \rightarrow MODEL \rightarrow EVALUATION$$

In our framework, the flow of information is divided into three steps: data generation, modelling and evaluation steps. Our framework definition relies on precise definitions on the properties of each step. A step is fully defined when its input and output are unambiguously described.

The data generation step generates all the data, including the features and the selectively labeled outcomes. In our setting we discuss mainly five different variables: R for leniency, X for observed features, Z for unobserved features (or \emph{unobservables} in short), T for the decisions and Y for the outcome. The central variants for data generation are presented in section \ref{sec:modules}. (we have separated the labeling process from the feature generation) The effects of variables on each other is presented as a directed graph in figure \ref{fig:initial_model}.

After the feature generation, the data is moved to a predictive model in the modelling step. The model, e.g. a regression model, is trained using one portion of the original data. Using the trained model we assign predictions for negative outcomes for the observations in the other part of the data.

Finally, an evaluation algorithm tries to output a reliable estimate of the model's failure rate. A good evaluation algorithm gives a precise, unbiased and low variance estimate of the failure rate of the model with a given leniency.

%seuraavaksi treeni data annetaan prediktiiviselle malli lle joka oppii mallin jota soveltaa sitten testidataan antaen jokaiselle instanssille tn y nollalle. Malleja kuten neuroverkot, log.reg., lin.reg etc. me käytetään logreg jotka on upotettu evaluointi moduuleihin

%Viimeisenä data scoreineen annetaan evaluointialgoritmille. Evalualgon tavoitteena on antaa mahdollisimman tarkka ja konsistentti arvo mallin FRstä tietyllä leniencyllä. Kappaleessa moduulit evaluointi algot etc. Hyyvä evaluaatio on tarkka (pieni bias ja varianssi) ja robusti erilaisille dg metodeille.


\section{Potential outcomes in model evaluation}

Potential outcomes evaluator uses Stan to infer the latent variable and the path coefficients and to estimate expectation of Y for the missing outcomes. Full hierarchical model is presented in eq. \ref{eq:po} below. Priors for the $\beta$ coefficients were chosen to be sufficiently non-informative without restricting the density estimation. Coefficients for the latent Z were restricted to be positive for posterior estimation. The  resulting distribution is the half-normal distribution ($X\sim N(0, 1) \Rightarrow Y=|X| \sim$ Half-Normal). The alphas are only for the decision variable to emulate the differences in leniency. Then subjects with equal x and z will have a different probability for bail given the judges leniency. 

\begin{align} \label{eq:po}
 Y~|~t,~x,~z,~\beta_{xy},~\beta_{zy} & \sim \text{Bernoulli}(\invlogit(\beta_{xy}x + \beta_{zy}z)) \\ \nonumber
 T~|~x,~z,~\beta_{xt},~\beta_{zt} & \sim \text{Bernoulli}(\invlogit(\alpha_j + \beta_{xt}x + \beta_{zt}z)) \\ \nonumber
 Z &\sim N(0, 1) \\ \nonumber
  \beta_{xt}, \beta_{xy} & \sim N(0, 10^2) \\ \nonumber
  \beta_{zt}, \beta_{zy} & \sim N_+(0, 10^2) \\ \nonumber
  p(\alpha_j) & \propto 1 \hskip1.0em \text{for } j \in \{1, 2, \ldots, M\}
\end{align}

Model is fitted on the test set and the expectations of potential outcomes $Y_{T=0}$ are used in place of missing outcomes.

\section{Results}

Here we present the results obtained from four core settings and four settings created for robustness check. Our core settings include situations where data is generated with the unobservables Z. Settings are characterized by the outcome creating mechanism and decision assignment.

The four main result figures are presented in figure set \ref{fig:results_bayes} with captions indicating the outcome and decision mechanisms. Figures showing the variance of the failure rate estimates are in appendix section \ref{sec:diagnostic}.

\begin{figure}[]
    \centering
    \begin{subfigure}[b]{0.475\textwidth}
        \includegraphics[width=\textwidth]{sl_result_bernoulli_independent}
        \caption{Outcome Y from Bernoulli, independent decisions using the quantiles.}
        %\label{fig:}
    \end{subfigure}
    \quad %add desired spacing between images, e. g. ~, \quad, \qquad, \hfill etc. 
      %(or a blank line to force the subfigure onto a new line)
    \begin{subfigure}[b]{0.475\textwidth}
        \includegraphics[width=\textwidth]{sl_result_threshold_independent}
        \caption{Outcome Y from threshold rule, independent decisions using the quantiles.}
        %\label{fig:}
    \end{subfigure}
    \begin{subfigure}[b]{0.475\textwidth}
        \includegraphics[width=\textwidth]{sl_result_bernoulli_batch}
        \caption{Outcome Y from Bernoulli, non-independent decisions.}
        %\label{fig:}
    \end{subfigure}
    \quad %add desired spacing between images, e. g. ~, \quad, \qquad, \hfill etc. 
      %(or a blank line to force the subfigure onto a new line)
    \begin{subfigure}[b]{0.475\textwidth}
        \includegraphics[width=\textwidth]{sl_result_threshold_batch}
        \caption{Outcome Y from threshold rule, non-independent decisions.}
        %\label{fig:}
    \end{subfigure}
    \caption{Core results: Failure rate vs. acceptance rate with varying levels of leniency. Different combinations of deciders and data generation mechanisms.}
    \label{fig:results_bayes}
\end{figure}

The robustness checks are presented in figure set \ref{fig:results_robustness} depicting deciders who
\begin{itemize} 
\item assign bail randomly with probability $r$, 
\item who favour and dislike some defendants with certain values of X
\item bad judge with a wrong $\beta_x$ 
\end{itemize}
Last figure is to illustrate the situation where the data generation mechanism is exactly the same as in model specified in eq. \ref{eq:po} with $\beta$ coefficients equal to 1 and $\alpha$ coefficients equal to 0.

The figures show that the contraction method is fairly robust to changes in data and decision generation although it has some variance. The potential outcomes approach consistently underestimates the true failure rate resulting in mediocre performance (compared to the imputation algorithms presented in the SL paper). The mean absolute errors of contraction with regard to true evaluation were in the order of 0.009...0.005 compared to our MAEs of approximately 0.015...0.035. Worrying aspect of the analysis is that even if the data generating process follows exactly the specified hierarchical model, the failure rate is still approximately 0.015.

Notable is that the diagnostic figures in section \ref{sec:diagnostic} show that few failure rate estimates are too high showing as "flyers/outliers" implying problems in model identifiability, probably as in bimodal posterior. (In the output when Stan tries to find the minimum of the negative log posterior (max of posterior) it occasionally converges to log probability of approximately -200...-400 when most of the times the minimum is at -7000...-9000.)

Imposing constraints to model in equation \ref{eq:po} did not yield significantly better results. The model was constrained so that there was only one intercept $\alpha$ and so that $\beta_{xt}=\beta_{xy}$ and $\beta_{zt}=\beta_{zy}$. Other caveats of the current approach is its scalability to big data sets.

\begin{figure}[H]
    \centering
    \begin{subfigure}[b]{0.475\textwidth}
        \includegraphics[width=\textwidth]{sl_result_random}
        \caption{Random decisions.}
        %\label{fig:}
    \end{subfigure}
    \quad %add desired spacing between images, e. g. ~, \quad, \qquad, \hfill etc. 
      %(or a blank line to force the subfigure onto a new line)
    \begin{subfigure}[b]{0.475\textwidth}
        \includegraphics[width=\textwidth]{sl_result_biased}
        \caption{Biased decisions.}
        %\label{fig:}
    \end{subfigure}
    \begin{subfigure}[b]{0.475\textwidth}
        \includegraphics[width=\textwidth]{sl_result_bad}
        \caption{Bad judge with $\beta_x=0.2$.}
        %\label{fig:}
    \end{subfigure}
    \quad %add desired spacing between images, e. g. ~, \quad, \qquad, \hfill etc. 
      %(or a blank line to force the subfigure onto a new line)
    \begin{subfigure}[b]{0.475\textwidth}
        \includegraphics[width=\textwidth]{sl_result_bernoulli_bernoulli}
        \caption{Data generation as corresponding to model.}
        %\label{fig:}
    \end{subfigure}
    \caption{Robustness check figures: Failure rate vs. acceptance rate with varying levels of leniency. Different combinations of deciders and data generation mechanisms.}
    \label{fig:results_robustness}
\end{figure}

\begin{thebibliography}{9} % Might have been apa

\bibitem{lakkaraju17} 
   Lakkaraju, Himabindu. The Selective Labels Problem: Evaluating Algorithmic Predictions in the Presence of Unobservables. 2017. 

\end{thebibliography}

%\end{document}

\newpage

%\begin{appendices}
\appendix

\section{RL's comments on the selective labels paper} \label{sec:comments}

\emph{This chapter is to present my comments and insight regarding the topic.}

The motivating idea behind the SL paper of Lakkaraju et al. \cite{lakkaraju17} is to evaluate if machines could improve on human performance. In general case, comparing the performance of human and machine evaluations is simple. In the domains addressed by Lakkaraju et al. simple comparisons would be unethical and therefore algorithms are required.

The general idea of the SL paper is to train some predictive model with selectively labeled data. The question is then "how would this predictive model perform if it was to make independent bail-or-jail decisions?" That quantity can not be calculated from real-life data sets due to the ethical reasons and hidden labels. We can however use more selectively labeled data to estimate it's performance. But, because the available data is biased, the performance estimates are too good or "overly optimistic" if they are calculated in the conventional way ("labeled outcomes only"). This is why they are proposing the contraction algorithm.

One of the concepts to denote when reading the Lakkaraju paper is the difference between the global goal of prediction and the goal in this specific setting. The global goal is to have a low failure rate with high acceptance rate, but at the moment we are not interested in it. The goal in this setting is to estimate the true failure rate of the model with unseen biased data. That is, given only selectively labeled data and an arbitrary black-box model $\B$ we are interested in predicting the performance of model $\B$ in the whole data set with all ground truth labels.

On acceptance rate R: We discussed how Lakkaraju's paper treats variable R in a seemingly non-sensical way, it is as if a judge would have to let someone go today in order to detain some other defendant tomorrow to keep their acceptance rate at some $r$. A more intuitive way of thinking $r$ would be the "threshold perspective". That is, if a judge sees that a defendant has probability $p_x$ of committing a crime if let out, the judge would detain the defendant if $p_x > r$, the defendant would be too dangerous to let out. Now, it can be claimed that judges do have a threshold, even in Lakkaraju's setting. Let $c$ be a judge's acceptance threshold. Now decision $T$ can be defined with function $f_T(x, z)$ as follows:

\begin{equation}
  \pr(T=0|x, z)=\begin{cases}
    p_T, & \text{if $f_T(x, z) < c$}\\
    1-p_T, & \text{otherwise}.
  \end{cases}
\end{equation}

If $c$ is defined so that the ratio of positive decisions to all decisions will be equal to $r$, we will arrive at a similar data generation process as Lakkaraju  and as is presented in algorithm \ref{alg:data_with_Z}.

Finally, chapters from Lakkaraju \cite{lakkaraju17} about counterfactual inference, see references from their paper [sic]:

\begin{quote}
There has also been some work on inferring labels using counterfactual inference techniques [9, 12, 16, 36, 38] and leveraging these estimates when computing standard evaluation metrics. However, counterfactual inference techniques explicitly assume that there are no unmeasured confounders (that is, no unobservable variables Z) that could affect the outcome Y. This assumption does not typically hold in cases where human decisions are providing data labels [7, 21]. Thus, the combination of two ingredients -- selective labels and non-trivial unobservables -- poses problems for these existing techniques.

Counterfactual inference. Counterfactual inference techniques have been used extensively to estimate treatment effects in observational studies. These techniques have found applications in a variety of fields such as machine learning, epidemiology, and sociology [3, 8--10, 30, 34]. Along the lines of Johansson et al. [16], counterfactual inference techniques can be broadly categorized as: (1) parametric methods which model the relationship between observed features, treatments, and outcomes. Examples include any type of regression model such as linear and logistic regression, random forests and regression trees [12, 33, 42]. (2) non-parametric methods such as propensity score matching, nearest-neighbor matching, which do not explicitly model the relationship between observed features, treatments, and outcomes [4, 15, 35, 36, 41]. (3) doubly robust methods which combine the two aforementioned classes of techniques typically via a propensity score weighted regression [5, 10]. The effectiveness of parametric and non-parametric methods depends on the postulated regression model and the postulated propensity score model respectively. If the postulated models are not identical to the true models, then these techniques result in biased estimates of outcomes. Doubly robust methods require only one of the postulated models to be identical to the true model in order to generate unbiased estimates. However, due to the presence of unobservables, we cannot guarantee that either of the postulated models will be identical to the true models.
\end{quote}

\section{Framework definition -- 13 June discussion} \label{sec:framework}

\emph{In this section we define some key terms and concepts and derive a more unambiguous framework for the selective labels problem. The framework is presented in writing and as a picture in figures \ref{fig:framework} and \ref{fig:framework_data_flow}.}

First, data is generated through a \textbf{data generating process (DGP)}. DGP comprises of generating the private features for the subjects, generating the acceptance rates for the judges and assigning the subjects to the judges. \textbf{Acceptance rate (AR)} is defined as the ratio of positive decisions to all decisions that a judge will give. As a formula \[ AR = \dfrac{\#\{Positive~decisions\}}{\#\{Decisions\}}. \]   Data generating process is depicted in the first box of Figure \ref{fig:framework}.

Next, the all of the generated data goes to the \textbf{labeling process}. In the labeling process, it is determined which instances of the data will have an outcome label available. This is done by humans and is presented in lines 5--7 of algorithm \ref{alg:data_without_Z} and 5--8 of algorithm \ref{alg:data_with_Z}. The data is then split randomly into training and test datasets, $\D_{train}$ and $\D_{test}$ respectively.

After labeling, the labeled training data is used to train a machine that will either make decisions or predictions using some features of the data. Then, the test data will be given to the machine and it will output either binary decisions (yes/no), probabilities (a real number in interval $[0, 1]$) or a metric for ordering for all the instances in the test data set. The machine will be denoted with $\M$.

Finally the decisions and/or predictions made by the machine $\M$ and human judges (see dashed arrow in figure \ref{fig:framework}) will be evaluated using an \textbf{evaluation algorithm}. Evaluation algorithms will take the decisions, probabilities or ordering generated in the previous steps as input and then output an estimate of the machine's failure rate in the test data. \textbf{Failure rate (FR)} is defined as the ratio of undesired outcomes to given decisions. One special characteristic of FR in this setting is that a failure can only occur with a positive decision. More explicitly \[ FR = \dfrac{\#\{Failures\}}{\#\{Decisions\}}. \] Second characteristic of FR is that the number of positive decisions and therefore FR itself can be controlled through acceptance rate defined above.

Given the above framework, the goal is to create an evaluation algorithm that can accurately estimate the failure rate of any model $\M$ if it were to replace human decision makers in the labeling process. The estimations have to be made using only data that human decision-makers have labeled. The failure rate has to be accurately estimated for various levels of acceptance rate. The accuracy of the estimates can be compared by computing e.g. mean absolute error w.r.t the estimates given by \nameref{alg:true_eval} algorithm.

\begin{figure} [H]
\centering
\begin{tikzpicture}[->, >=stealth, shorten >=1pt, auto, node distance=1.5cm, semithick]

  \tikzstyle{every state}=[fill=none, draw=black, text=black, rectangle, minimum width=6.0cm]

  \node[state] (DG) {Data generation};
  \node[state] (LP) [below of=DG] {Labeling process (human)};
  \node[state] (MP) [below of=LP] {Machine predictions};
  \node[state] (EA)  [below of=MP] {Evaluation algorithm};

  \path (DG) edge (LP)
                    edge [bend right=90, dashed] node [left] {(1)} (MP)
        (LP) edge (MP)
             edge [bend left=90, dashed] node {(2)} (EA)
        (MP) edge (EA);
\end{tikzpicture}
\caption{The selective labels framework. Dashed arrow (1) marks the flow of unlabeled data to \nameref{alg:true_eval} algorithm and (2) indicates how human evaluations are evaluated without machine intervention using \nameref{alg:human_eval} algorithm.}
\label{fig:framework}
\end{figure}

\begin{figure} [H]
\centering
\begin{tikzpicture}[->,>=stealth',shorten >=1pt,auto,node distance=1.5cm,
                            semithick]
        
          \tikzstyle{every state}=[fill=none,draw=black,text=black, rectangle, minimum width=6.0cm]
        
          \node[state] (DG) {Data generation};
          \node[state] (LP) [below of = DG] {Labeling process (human)};
          \node[state] (MT) [below left=1.0cm and -4cm of LP] {Model training};
          \node[state] (MP) [below=1.0cm of MT] {Machine predictions};
          \node[state] (EA)  [below right=0.75cm and -4cm of MP] {Evaluation algorithm};
        
          \path (DG) edge (LP)
                            edge [out=180, in=180, dashed] node [left]  {$\D_{test,~unlabeled}$ (1)}   (MP)
                (LP) edge [bend right=19] node [left] {$\D_{train}$} (MT)
                      edge [bend left=60] node [right]  {$\D_{test}$} (MP)
                     edge [bend left=75, dashed] node [right]  {$\D_{test}$ (2)}  (EA)
                (MT) edge node {$\M$} (MP)
                (MP) edge (EA);
        \end{tikzpicture}
\caption{The selective labels framework with explicit data flow. Dashed arrow (1) marks the flow of unlabeled data to \nameref{alg:true_eval} algorithm and (2) indicates how human evaluations are evaluated without machine intervention using \nameref{alg:human_eval} algorithm.}
\label{fig:framework_data_flow}
\end{figure}


\section{Modular framework -- 19 June discussion} \label{sec:modular_framework}

\emph{Below is the framework as was written on the whiteboard, then RL presents his own remarks on how he understood this.}

\begin{wrapfigure}{r}{0.25\textwidth} %this figure will be at the right
    \centering
    \begin{tikzcd}
       \arrow[d]   &  \arrow[d]  &  \arrow[d]   \\
    X \arrow[rd] & Z \arrow[d] & W \arrow[ld] \\
                       & Y                 &             
    \end{tikzcd}
    \caption{$\M$}
    \label{fig:dgm}
\end{wrapfigure}

~ \\
\begin{description}

\item[Data generation:] 
	~ \\
	\hskip 3em \textbf{Input:} [none] \\ 
			 \textbf{Output:} $X, Z, W, Y$ as specified by $\M$~ \\ 

\item[Decider:] single vs. batch 
	~ \\
	\hskip 3em \textbf{Input:}  
		\begin{itemize}  
			\item one defendant 
			\item $\M$ 
		\end{itemize} 
	\textbf{Output:} 
		\begin{itemize}  
			\item argmax likelihood  $y$ 
			\item $\pr(Y=0~|~input)$
			\item order \\
		\end{itemize} 
\item[Evaluator:] 
	~ \\
	\hskip 3em \textbf{Input:} 
		\begin{itemize}  
			\item Data sample $(X, T, Y)$ 
			\item something about $\M$ and something about Decider(r)
		\end{itemize}
	\textbf{Output:}
		\begin{itemize}  
			\item $\mathbb{E}[FR~|~input]$
			\item curve
		\end{itemize} 

\end{description}

The above framework is now separated into three different modules: data generation, decider and evaluator. In the first module, all the data points $\{x_i, z_i, w_i, y_i\}$ for all $i=1, \ldots, n$ are created. Outcome $y_i$ is available for all observations.

The next module, namely the decider, assigns decisions for each observation with a given/defined way. This 'decision' can be either the most likely value for y (argmax likelihood y, usually binary 0 or 1), probability of an outcome or an ordering of the defendants.

The evaluator module takes a data sample, some information about the data generation and some information about the decider as input. The data sample includes features $X,~T$ and $Y$ where $Y \in \{0, 1, NA\}$ as specified before. The "some information about $\M$" might be knowledge on the distribution of some of the variables or their interdependencies. In our example, we know that the $X$ is a standard Gaussian and independent from the other variables. From the decider it is known that its decisions are affected by leniency and private properties X. Our objective is to simulate the decision-maker's process within the data sample. But to do this we need to learn the predictive model $\B$ with the restriction that Z can't be observed. With this setting, we now need to define an algorithm which outputs $\mathbb{E}[FR~|~input]$ for a given model $\B$ and leniency $r$. 

\begin{quote}
\emph{MM:} For example, consider an evaluation process that knows (i.e., is given as input) the decision process and what decisions it took for a few data points. The same evaluation process knows only some of the attributes of those data points -- and therefore it has only partial information about the data generation process. To make the example more specific, consider the case of decision process $\s$ mentioned above, which does not know W -- and consider an evaluation process that knows exactly how $\s$ works and what decisions it took for a few data points, but does not know either W or Z of those data points. This evaluation process outputs the expected value of FR according to the information that's given to it.
\end{quote}

\textbf{Example} For illustration, let's consider the contraction algorithm in this framework. Data is generated from three Gaussians and the outcome Y is decided as in algorithm \ref{alg:data_with_Z}. Now the deciders are humans assigning the decisions as in algorithm \ref{alg:data_with_Z}. Next, the evaluator module
\begin{itemize}
\item takes the whole of the observable data ($\{x_i, t_i, y_i\}$) as input
\item knows that X affects the results and the decisions
\item knows that the variables are Gaussians.
\end{itemize}
Now the evaluator module splits the data to training and test sets and trains a predictive model $\B$ on the training set. Finally the contraction algorithm outputs an estimate of the failure rate following its definition (see algorithm \ref{alg:contraction}) using predictions from $\B$.


\section{Data generation}

Both of the data generating algorithms are presented in this chapter. 

\subsection{Without unobservables (see also algorithm \ref{alg:data_without_Z})}

In the setting without unobservables Z, we first sample an acceptance rate $r$ for all $M=100$ judges uniformly from a half-open interval $[0.1; 0.9)$. Then we assign 500 unique subjects for each of the judges randomly (50000 in total) and simulate their features X as i.i.d standard Gaussian random variables with zero mean and unit (1) variance. Then, probability for negative outcome is calculated as 
\begin{equation} \label{eq:inv_logit}
	P(Y=0|X=x) = \dfrac{1}{1+\exp(-x)}=\invlogit(x).
\end{equation}
Because $P(Y=1|X=x) = 1-P(Y=0|X=x) = 1-\invlogit(x)$ the outcome variable Y can be sampled from Bernoulli distribution with parameter $1-\invlogit(x)$. The data is then sorted for each judge by the probabilities $P(Y=0|X=x)$ in descending order. If the subject is in the top $(1-r) \cdot 100 \%$ of observations assigned to a judge, the decision variable T is set to zero and otherwise to one.

\begin{algorithm}[] 			% enter the algorithm environment
\caption{Create data without unobservables} 		% give the algorithm a caption
\label{alg:data_without_Z} 			% and a label for \ref{} commands later in the document
\begin{algorithmic}[1] 		% enter the algorithmic environment
\REQUIRE Number of judges $M=100$ and number of subjects distributed to each of them $N=500$ s.t. $N_{total} = N \cdot M$
\ENSURE
\STATE Sample acceptance rates for each M judges from $U(0.1; 0.9)$ and round to tenth decimal place.
\STATE Sample features X for each $N_{total}$ observations from standard Gaussian.
\STATE Calculate $P(Y=0|X=x)=\invlogit(x)$ for each observation
\STATE Sample Y from Bernoulli distribution with parameter $1-\invlogit(x)$.
\STATE Sort the data by (1) the judges and (2) by probabilities $P(Y=0|X=x)$ in descending order. 
\STATE \hskip3.0em $\rhd$ Now the most dangerous subjects for each of the judges are at the top.
\STATE If subject belongs to the top $(1-r) \cdot 100 \%$ of observations assigned to a judge, set $T=0$ else set $T=1$.
\STATE Halve the data to training and test sets at random.
\STATE For both halves, set $Y=$ NA if decision is negative ($T=0$).
\RETURN labeled training data, full training data, labeled test data, full test data
\end{algorithmic}
\end{algorithm}

\subsection{With unobservables (see also algorithm \ref{alg:data_with_Z})}

In the setting with unobservables Z, we first sample an acceptance rate r for all $M=100$ judges uniformly from a half-open interval $[0.1; 0.9)$. Then we assign 500 unique subjects (50000 in total) for each of the judges randomly and simulate their features X, Z and W as i.i.d standard Gaussian random variables with zero mean and unit (1) variance. Then, probability for negative outcome is calculated as
\begin{equation}
	P(Y=0|X=x, Z=z, W=w)=\invlogit(\beta_Xx+\beta_Zz+\beta_Ww)~,
\end{equation}
where $\beta_X=\beta_Z =1$ and $\beta_W=0.2$. Next, value for result Y is set to 0 if $P(Y = 0| X, Z, W) \geq 0.5$ and 1 otherwise. The conditional probability for the negative decision (T=0) is defined as 
\begin{equation}
	P(T=0|X=x, Z=z)=\invlogit(\beta_Xx+\beta_Zz)+\epsilon~,
\end{equation}
where $\epsilon \sim N(0, 0.1)$. Next, the data is sorted for each judge by the probabilities $P(T=0|X, Z)$ in descending order. If the subject is in the top $(1-r) \cdot 100 \%$ of observations assigned to a judge, the decision variable T is set to zero and otherwise to one.

\begin{algorithm}[] 			% enter the algorithm environment
\caption{Create data with unobservables} 		% give the algorithm a caption
\label{alg:data_with_Z} 			% and a label for \ref{} commands later in the document
\begin{algorithmic}[1] 		% enter the algorithmic environment
\REQUIRE Number of judges $M=100$, number of subjects distributed to each of them $N=500$ s.t $N_{total} = N \cdot M$, $\beta_X=1, \beta_Z=1$ and $\beta_W=0.2$.
\ENSURE
\STATE Sample acceptance rates for each M judges from $U(0.1; 0.9)$ and round to tenth decimal place.
\STATE Sample features X, Z and W for each $N_{total}$ observations from standard Gaussian independently.
\STATE Calculate $P(Y=0|X, Z, W)$ for each observation.
\STATE Set Y to 0 if $P(Y = 0| X, Z, W) \geq 0.5$ and to 1 otherwise.
\STATE Calculate $P(T=0|X, Z)$ for each observation and attach to data.
\STATE Sort the data by (1) the judges and (2) by probabilities $P(T=0|X, Z)$ in descending order. 
\STATE \hskip3.0em $\rhd$ Now the most dangerous subjects for each judge are at the top.
\STATE If subject belongs to the top $(1-r) \cdot 100 \%$ of observations assigned to that judge, set $T=0$ else set $T=1$.
\STATE Halve the data to training and test sets at random.
\STATE For both halves, set $Y=$ NA if decision is negative ($T=0$).
\RETURN labeled training data, full training data, labeled test data, full test data
\end{algorithmic}
\end{algorithm}

\section{Model fitting} \label{sec:model_fitting}

The models that are being fitted are logistic regression models from scikit-learn package. The solver is set to lbfgs (as there is no closed-form solution) and intercept is estimated by default. The resulting LogisticRegression model object provides convenient functions for fitting the model and getting probabilities for class labels. Please see the documentation at \url{https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html} or ask me (RL) for more details. Similar analyses were conducted using random forest classifier, but the results (see section \ref{sec:random_forest}) were practically identical.

All of the algorithms 4--8 are model agnostic, i.e. do not depend on a specific predictive model. The model has to give probabilities for given output with some determined input. Lakkaraju says in their paper "We train logistic regression on this training set. We also experimented with other predictive models and observed similar behaviour."

NB: The sklearn's regression model can not be fitted if the data includes missing values. Therefore list-wise deletion is done in cases of missing data (whole record is discarded). 

\section{Plotting}

The following quantities are computed from the data:

\begin{itemize}
\item True evaluation: The true failure rate of the model. Can only be calculated for synthetic data sets. See algorithm \ref{alg:true_eval} and discussion in section \ref{sec:comments}.
\item Labeled outcomes: The "traditional"/vanilla estimate of model performance. See algorithm \ref{alg:labeled_outcomes}.
\item Human evaluation: The failure rate of human decision-makers who have access to the latent variable Z. Decision-makers with similar values of leniency are binned and treated as one hypothetical decision-maker. See algorithm \ref{alg:human_eval}.
\item Contraction: See algorithm \ref{alg:contraction} from \cite{lakkaraju17}.
\item Causal model: In essence, the empirical performance is calculated over the test set as 
\begin{equation}\label{eq:ep}
	\dfrac{1}{n}\sum_{(x, y)\in D}f(x)\delta(F(x) < r)
\end{equation}
where
\begin{equation}\label{eq:causal_prediction}
	f(x) = P(Y=0|T=1, X=x)
\end{equation}
is a logistic regression model (see section \ref{sec:model_fitting}, random forest used in section \ref{sec:random_forest}) trained on the labeled data predicting Y from X and 
\begin{equation} \label{eq:causal_cdf}
F(x_0) = \int_{x\in\mathcal{X}} P(x)\delta(f(x) < f(x_0)) ~ dx.
\end{equation}
All observations, even ones with missing outcome labels, can be used since empirical performance doesn't depend on them. $P(x)$ is Gaussian pdf from scipy.stats package and it is integrated over interval [-15, 15] with 40000 steps using si.simps function from scipy.integrate which uses Simpson's rule in estimating the value of the integral. (docs: \url{https://docs.scipy.org/doc/scipy/reference/generated/scipy.integrate.simps.html}) See also algorithm \ref{alg:causal_model} and motivation from section \ref{sec:motivation} . \label{causal_cdf}
\end{itemize}

The plotted curves are constructed using pseudo code presented in algorithm \ref{alg:perf_comp}.

\subsection{Short motivation of the causal model presented above:} \label{sec:motivation}

The causal model tries to predict the probability of adverse outcome $Y=0$ when an acceptance rate $r$ is imposed. Estimating such probability in the selective labels setting consists of two parts: predicting the probability for an individual to commit a crime if they are given bail and deciding whether to bail or jail them. 

In equations \ref{eq:ep} and \ref{eq:causal_prediction}, $f(x)$ gives the probability given private features $x$. In equation \ref{eq:ep} $\delta(F(x) < r)$ indicates the defendants bail decision. They will be let out if the proportion of people less dangerous than $x_0$ is under $r$. For example, if a defendant $x_0$ arrives in front of a judge with leniency 0.65 they will not be left out if the judge deems that $F(x_0) > 0.65$ that is if the judge thinks that more than 65\% of the defendants are more dangerous than them.

Now the equation \ref{eq:ep} simply calculates the mean of the probabilities forcing the probbility of crime to zero if they will not be given bail.

\begin{algorithm}[] 			% enter the algorithm environment
\caption{Performance comparison} 		% give the algorithm a caption
\label{alg:perf_comp} 			% and a label for \ref{} commands later in the document
\begin{algorithmic}[1] 		% enter the algorithmic environment
\REQUIRE Number of iterations $N_{iter}$
\ENSURE
\FORALL{$r$ in $0.1, 0.2, ..., 0.9$}
	\FOR{i = 1 \TO $N_{iter}$}
		\STATE Create data using either Algorithm \ref{alg:data_without_Z} or \ref{alg:data_with_Z}.
		\STATE Train a logistic regression model using observations in the training set with available outcome labels and assign to $f$.
		\STATE Using $f$, estimate probabilities $\s$ for Y=0 in both test sets (labeled and full) for all observations and attach them to the respective data sets.
        		\STATE Compute failure rate of true evaluation with leniency $r$ and full test data using algorithm \ref{alg:true_eval}.
        		\STATE Compute failure rate of labeled outcomes approach with leniency $r$ and labeled test data using algorithm \ref{alg:labeled_outcomes}.
        		\STATE Compute failure rate of human judges with leniency $r$ and labeled test data using algorithm \ref{alg:human_eval}.
        		\STATE Compute failure rate of contraction algorithm with leniency $r$ and labeled test data.
        		\STATE Compute the empirical performance of the causal model with leniency $r$, predictive model $f$ and labeled test data using algorithm \ref{alg:causal_model}.
	\ENDFOR
	\STATE Calculate means of the failure rates for each level of leniency and for each algorithm separately.
	\STATE Calculate standard error of the mean for each level of leniency and for each algorithm separately.
\ENDFOR 
\STATE Plot the failure rates with given levels of leniency $r$.
\STATE Calculate absolute mean errors of each algorithm compared to true evaluation.
\end{algorithmic}
\end{algorithm}

\begin{algorithm}[] 			% enter the algorithm environment
\caption{True evaluation} 		% give the algorithm a caption
\label{alg:true_eval} 			% and a label for \ref{} commands later in the document
\begin{algorithmic}[1] 		% enter the algorithmic environment
\REQUIRE Full test data $\D$ with probabilities $\s$ and \emph{all outcome labels}, acceptance rate r
\ENSURE
\STATE Sort the data by the probabilities $\s$ to ascending order.
\STATE \hskip3.0em $\rhd$ Now the most dangerous subjects are last.
\STATE Calculate the number to release $N_{free} = |\D| \cdot r$.
\RETURN $\frac{1}{|\D|}\sum_{i=1}^{N_{free}}\delta\{y_i=0\}$
\end{algorithmic}
\end{algorithm}

\begin{algorithm}[] 			% enter the algorithm environment
\caption{Labeled outcomes} 		% give the algorithm a caption
\label{alg:labeled_outcomes} 			% and a label for \ref{} commands later in the document
\begin{algorithmic}[1] 		% enter the algorithmic environment
\REQUIRE Labeled test data $\D$ with probabilities $\s$ and \emph{missing outcome labels} for observations with $T=0$, acceptance rate r
\ENSURE
\STATE Assign observations with observed outcomes to $\D_{observed}$.
\STATE Sort $\D_{observed}$ by the probabilities $\s$ to ascending order.
\STATE \hskip3.0em $\rhd$ Now the most dangerous subjects are last.
\STATE Calculate the number to release $N_{free} = |\D_{observed}| \cdot r$.
\RETURN $\frac{1}{|\D_{observed}|}\sum_{i=1}^{N_{free}}\delta\{y_i=0\}$
\end{algorithmic}
\end{algorithm}

\begin{algorithm}[] 			% enter the algorithm environment
\caption{Human evaluation} 		% give the algorithm a caption
\label{alg:human_eval} 			% and a label for \ref{} commands later in the document
\begin{algorithmic}[1] 		% enter the algorithmic environment
\REQUIRE Labeled test data $\D$ with probabilities $\s$ and \emph{missing outcome labels} for observations with $T=0$, acceptance rate r
\ENSURE
\STATE Assign judges with leniency in $[r-0.05, r+0.05]$ to $\mathcal{J}$
\STATE $\D_{released} = \{(x, j, t, y) \in \D~|~t=1 \wedge j \in  \mathcal{J}\}$
\STATE \hskip3.0em $\rhd$ Subjects judged \emph{and} released by judges with correct leniency.
\RETURN $\frac{1}{|\mathcal{J}|}\sum_{i=1}^{\D_{released}}\delta\{y_i=0\}$
\end{algorithmic}
\end{algorithm}

\begin{algorithm}[] 			% enter the algorithm environment
\caption{Contraction algorithm \cite{lakkaraju17}} 		% give the algorithm a caption
\label{alg:contraction} 			% and a label for \ref{} commands later in the document
\begin{algorithmic}[1] 		% enter the algorithmic environment
\REQUIRE Labeled test data $\D$ with probabilities $\s$ and \emph{missing outcome labels} for observations with $T=0$, acceptance rate r
\ENSURE
\STATE Let $q$ be the decision-maker with highest acceptance rate in $\D$.
\STATE $\D_q = \{(x, j, t, y) \in \D|j=q\}$
\STATE \hskip3.0em $\rhd$ $\D_q$ is the set of all observations judged by $q$
\STATE
\STATE $\RR_q = \{(x, j, t, y) \in \D_q|t=1\}$
\STATE \hskip3.0em $\rhd$ $\RR_q$ is the set of observations in $\D_q$ with observed outcome labels
\STATE
\STATE Sort observations in $\RR_q$ in descending order of confidence scores $\s$ and assign to $\RR_q^{sort}$.
\STATE \hskip3.0em $\rhd$ Observations deemed as high risk by the black-box model $\mathcal{B}$ are at the top of this list
\STATE
\STATE Remove the top $[(1.0-r)|\D_q |]-[|\D_q |-|\RR_q |]$ observations of $\RR_q^{sort}$ and call this list $\mathcal{R_B}$
\STATE \hskip3.0em $\rhd$ $\mathcal{R_B}$ is the list of observations assigned to $t = 1$ by $\mathcal{B}$
\STATE
\STATE Compute $\mathbf{u}=\sum_{i=1}^{|\mathcal{R_B}|} \dfrac{\delta\{y_i=0\}}{| \D_q |}$.
\RETURN $\mathbf{u}$
\end{algorithmic}
\end{algorithm}

\begin{algorithm}[] 			% enter the algorithm environment
\caption{Causal model, empirical performance (ep, see also section \ref{causal_cdf})} 		% give the algorithm a caption
\label{alg:causal_model} 			% and a label for \ref{} commands later in the document
\begin{algorithmic}[1] 		% enter the algorithmic environment
\REQUIRE Labeled test data $\D$ with probabilities $\s$ and \emph{missing outcome labels} for observations with $T=0$, predictive model $f$, pdf $P_X(x)$ for features $x$, acceptance rate r
\ENSURE
\STATE For all $x_0 \in \D$ evaluate $F(x_0) = \int_{x\in\mathcal{X}} P_X(x)\delta(f(x)<f(x_0)) ~dx$ and assign to $\mathcal{F}_{predictions}$ 
\STATE Create boolean array $T_{causal} = \mathcal{F}_{predictions} < r$.
\RETURN $\frac{1}{|\D|}\sum_{i=1}^{\D} \s \cdot T_{causal}$ which is equal to $\frac{1}{|\D|}\sum_{x\in\D} f(x)\delta(F(x) < r)$
\end{algorithmic}
\end{algorithm}

\section{Modules} \label{sec:modules}

Different types of modules (data generation, decider and evaluator) are presented in this section. Summary table of the modules is presented last. See section \ref{sec:modular_framework} for some discussion on the properties of each module.

\subsection{Data generation modules} 

The purpose of a data generation module is to define a process which generates all of the data consisting of all the features and outcomes. The data generation module should include information on what are the distributions of the features and how each of them affect the generation of the outcome.

In our synthetic setting, we have three different kinds of data generating modules (DG modules). The DG modules can be separated by two features: inclusion of unobservables and the outcome generating mechanism. Private features are mainly sampled from standard Gaussians. See summary of DG modules from table \ref{tab:dg_modules}.

\begin{table}[h]
\centering
\caption{Summary of data generation modules}
\label{tab:dg_modules}
\begin{tabular}{@{}llcc@{}}
\toprule
                                             &                       & \multicolumn{2}{c}{Feature generation}                                             \\ \cmidrule(l){3-4} 
                                             &                       & \multicolumn{1}{l}{With unobservables} & \multicolumn{1}{l}{Without unobservables} \\
\multicolumn{1}{c}{\multirow{2}{*}{Outcome}} & Drawn from Bernoulli  & \ref{alg:dg:coinflip_with_z}           & \ref{alg:dg:coinflip_without_z}           \\
\multicolumn{1}{c}{}                         & Assigned by threshold & \ref{alg:dg:threshold_with_Z}          &                                           \\ \cmidrule(l){2-4} \bottomrule
\end{tabular}
\end{table}

\begin{algorithm}[H] 			% enter the algorithm environment
\caption{Data generation module: outcome from Bernoulli without unobservables} 		% give the algorithm a caption
\label{alg:dg:coinflip_without_z} 			% and a label for \ref{} commands later in the document
\begin{algorithmic}[1] 		% enter the algorithmic environment
\REQUIRE Parameters: Total number of subjects $N_{total}$
\ENSURE
\FORALL{observations}
	\STATE Draw $x$ from from a standard Gaussian.
	\STATE Draw $y$ from Bernoulli$(1-\invlogit(x))$.
\ENDFOR 
\RETURN data
\end{algorithmic}
\end{algorithm}

\begin{algorithm}[H] 			% enter the algorithm environment
\caption{Data generation module: outcome by threshold with unobservables} 		% give the algorithm a caption
\label{alg:dg:threshold_with_Z} 			% and a label for \ref{} commands later in the document
\begin{algorithmic}[1] 		% enter the algorithmic environment
\REQUIRE Parameters: Total number of subjects $N_{total},~\beta_X=1,~\beta_Z=1$ and $\beta_W=0.2$.
\ENSURE
\FORALL{observations}
	\STATE Draw $x, z$ and $w$ from from standard Gaussians independently.
	\IF{$\invlogit(\beta_Xx+\beta_Zz+\beta_Ww) \geq 0.5$}
		\STATE {Set $y$ to 0.}
	\ELSE
		\STATE {Set $y$ to 1.}
	\ENDIF
\ENDFOR 
\RETURN data
\end{algorithmic}
\end{algorithm}

\begin{algorithm}[H] 			% enter the algorithm environment
\caption{Data generation module: outcome from Bernoulli with unobservables} 		% give the algorithm a caption
\label{alg:dg:coinflip_with_z} 			% and a label for \ref{} commands later in the document
\begin{algorithmic}[1] 		% enter the algorithmic environment
\REQUIRE Parameters: Total number of subjects $N_{total},~\beta_X=1,~\beta_Z=1$ and $\beta_W=0.2$.
\ENSURE
\FORALL{observations}
	\STATE Draw $x, z$ and $w$ from from standard Gaussians independently.
	\STATE Draw $y$ from Bernoulli$(1-\invlogit(\beta_Xx+\beta_Zz+\beta_Ww))$.
\ENDFOR 
\RETURN data
\end{algorithmic}
\end{algorithm}

%For decider modules, input as terms of knowledge and parameters should be as explicitly specified as possible.

\subsection{Decider modules} 

We have three different kinds of decider modules. Their distinctive feature is the decisions' independence, for example in algorithm \ref{alg:decider:lakkaraju} the decisions of a decision-maker are dependent on the other subjects assigned to that decision-maker.

Below is presented the decision-maker by Lakkaraju \cite{lakkaraju17}. The decision-maker (1) takes all the subjects as a batch, (2) makes an approximation of the subjects' tendencies for negative outcomes and (3) assigns the decisions by giving $r\cdot 100\%$ of the least likely to fail a positive decision. The resulting decisions are not independent as they depend on the presence of other observations.

\begin{algorithm}[H] 			% enter the algorithm environment
\caption{Decider module: decision-maker by Lakkaraju et al. \cite{lakkaraju17}} 		% give the algorithm a caption
\label{alg:decider:lakkaraju} 			% and a label for \ref{} commands later in the document
\begin{algorithmic}[1] 		% enter the algorithmic environment
\REQUIRE Data with features $X, Z$, knowledge that both of them affect the outcome Y and that they are independent / Parameters: $M=100, \beta_X=1, \beta_Z=1$.
\ENSURE
\STATE Sample acceptance rates for each M judges from Uniform$(0.1; 0.9)$ and round to tenth decimal place.
\STATE Assign each observation to a judge at random.
\STATE Calculate $\pr(T=0|X, Z) = \invlogit(\beta_XX+\beta_ZZ) + \epsilon$ for each observation and attach to data.
\STATE Sort the data by (1) the judges and (2) by the probabilities in descending order. 
\STATE If subject belongs to the top $(1-r) \cdot 100 \%$ of observations assigned to that judge, set $T=0$ else set $T=1$.
\STATE Set $Y=$ NA if decision is negative ($T=0$).
\RETURN data with decisions.
\end{algorithmic}
\end{algorithm}

One discussed way of making the decisions independent was to "flip a coin at some probability". An implementation of that idea is presented below in algorithm \ref{alg:decider:coinflip}. As $\pr(T=0|X, Z) = \invlogit(\beta_XX+\beta_ZZ)$ the parameter for the Bernoulli distribution is set to $1-\invlogit(\beta_XX+\beta_ZZ)$. In the practical implementation, as some algorithms need to know the leniency of the decision, acceptance rate is then calculated then from the decisions.

\begin{algorithm}[H] 			% enter the algorithm environment
\caption{Decider module: decisions from Bernoulli} 		% give the algorithm a caption
\label{alg:decider:coinflip} 			% and a label for \ref{} commands later in the document
\begin{algorithmic}[1] 		% enter the algorithmic environment
\REQUIRE Data with features $X, Z$, knowledge that both of them affect the outcome Y and that they are independent / Parameters: $\beta_X=1, \beta_Z=1$.
\ENSURE
\STATE Draw $t$ from Bernoulli$(1-\invlogit(\beta_Xx+\beta_Zz))$ for all observations.
\STATE Compute the acceptance rate.
\STATE Set $Y=$ NA if decision is negative ($T=0$).
\RETURN data with decisions.
\end{algorithmic}
\end{algorithm}

A quantile-based decider module is presented in algorithm \ref{alg:decider:quantile}. The algorithm tries to emulate Lakkaraju's decision-maker while giving out independent decisions. The independence is achieved by comparing the values of $\invlogit(\beta_Xx+\beta_Zz)$ to the corresponding value of the inverse cumulative distribution function $F^{-1}_{\invlogit(\beta_XX+\beta_ZZ)}$ or $F^{-1}$ in short. The derivation of $F^{-1}$ is deferred to the next section. %The decisions have a guarantee that the fraction of positive decisions will converge to $r$ based on the law of large numbers. 

\textbf{Example} Consider a decision-maker with leniency 0.60 who gets a new subject $\{x, z\}$ with a predicted probability $\invlogit(\beta_Xx+\beta_Zz)\approx 0.7$ for a negative outcome with some coefficients $\beta$. Now, as the judge has leniency 0.6, their cut-point $F^{-1}(0.60)\approx0.65$. That is, the judge will not give a positive decision to anyone with failure probability greater than 0.65, so our example subject will receive a negative decision.

\begin{algorithm}[H] 			% enter the algorithm environment
\caption{Decider module: "quantile decisions"} 		% give the algorithm a caption
\label{alg:decider:quantile} 			% and a label for \ref{} commands later in the document
\begin{algorithmic}[1] 		% enter the algorithmic environment
\REQUIRE Data with features $X, Z$, knowledge that both of them affect the outcome Y and that they are independent / Parameters: $\beta_X=1, \beta_Z=1$.
\ENSURE
\STATE Sample acceptance rates for each M judges from Uniform$(0.1; 0.9)$ and round to tenth decimal place.
\STATE Assign each observation to a judge at random.
\STATE Calculate $\pr(T=0|X, Z) = \invlogit(\beta_XX+\beta_ZZ)$ for all observations.
\STATE If $\invlogit(\beta_Xx+\beta_Zz) \geq F^{-1}(r)$ set $t=0$, otherwise set $t=1$.
\STATE Set $Y=$ NA if decision is negative ($T=0$).
\RETURN data with decisions.
\end{algorithmic}
\end{algorithm}

\subsection{Evaluator modules} \label{sec:evaluators}

Evaluator modules take some version of data as input and output an estimate of the failure rate given the input. More discussion on the evaluator module is in section \ref{sec:modular_framework}.

Motivation for the contraction algorithm is presented in Lakkaraju's paper \cite{lakkaraju17}. The original algorithm has been copied to algorithm \ref{alg:contraction}. The algorithm below has slight modifications to incorporate model $\B$ which tries to predict Y from x. About $\B$ from Lakkaraju:

\begin{quote}
Black box predictive model. Another input to our framework is a black box predictive model $\B$ which assigns risk scores to observations in $\D$. More specifically, $\B$ is a function which maps the characteristics (or feature values) $\mathbf{x}$ of an observation in $\D$ to a probability score $s \in [0, 1]$. This score indicates how confident the model is in assigning the observation to $t = 0$ (e.g., denying bail).
\end{quote}

\begin{algorithm}[H] 			% enter the algorithm environment
\caption{Evaluator module: Contraction algorithm \cite{lakkaraju17}} 		% give the algorithm a caption
\label{alg:eval:contraction} 			% and a label for \ref{} commands later in the document
\begin{algorithmic}[1] 		% enter the algorithmic environment
\REQUIRE Data $\D$ with properties $\{x_i, j_i, t_i, y_i\}$, acceptance rate r, knowledge that X affects Y
\ENSURE
\STATE Split data to test set and training set.
\STATE Train a predictive model $\B$ on training data.
\STATE Estimate and assign probability scores $\s$ using $\B$ for all observations in test data.
\STATE Let $q$ be the decision-maker with highest acceptance rate in $\D$.
\STATE $\D_q = \{(x, j, t, y) \in \D|j=q\}$
\STATE $\RR_q = \{(x, j, t, y) \in \D_q|t=1\}$
\STATE Sort observations in $\RR_q$ in descending order of confidence scores $\s$ and assign to $\RR_q^{sort}$.
\STATE Remove the top $[(1.0-r)|\D_q |]-[|\D_q |-|\RR_q |]$ observations of $\RR_q^{sort}$ and call this list $\mathcal{R_B}$
\STATE Compute $\mathbf{u}=\sum_{i=1}^{|\mathcal{R_B}|} \dfrac{\delta\{y_i=0\}}{| \D_q |}$.
\RETURN $\mathbf{u}$
\end{algorithmic}
\end{algorithm}

True evaluation module computes the "true failure rate" of a predictive model \emph{had it been deployed to make independent decisions}. For computing the true failure rate "had the model been deployed" we need all outcome labels which is why the true failure rate can only be computed on synthetic data.

In practice, the module first trains a model $\B$ and assigns each observation with a probability score $\s$ using it as described above. Then the observations are sorted in ascending order by the scores so that most risky subjects are last (subjects with the highest predicted probability for a negative outcome). Taking the first $r \cdot 100\%$ of observations, the true failure rate can be computed from the ground truth directly.

\begin{algorithm}[H] 			% enter the algorithm environment
\caption{Evaluator module: True evaluation} 		% give the algorithm a caption
\label{alg:eval:true_eval} 			% and a label for \ref{} commands later in the document
\begin{algorithmic}[1] 		% enter the algorithmic environment
\REQUIRE Data $\D$ with properties $\{x_i, t_i, y_i\}$ and \emph{all outcome labels}, acceptance rate r, knowledge that X affects Y
\ENSURE
\STATE Split data to test set and training set.
\STATE Train a predictive model $\B$ on training data.
\STATE Estimate and assign probability scores $\s$ using $\B$ for all observations in test data.
\STATE Sort the data by the probabilities $\s$ to ascending order.
\STATE Calculate the number to release $N_{free} = |\D| \cdot r$.
\RETURN $\frac{1}{|\D|}\sum_{i=1}^{N_{free}}\delta\{y_i=0\}$
\end{algorithmic}
\end{algorithm}

The labeled outcomes evaluator can be described to be the "vanilla estimate of the failure rate". Using probability scores $\s$ from $\B$, the estimate of failure rate is computed straight from the available labels. From Lakkaraju:

\begin{quote}
Labeled Outcomes Only: To plot this curve, we first obtain all the subjects whose outcome labels are available in the evaluation set and rank them in ascending order based on the probability scores assigned by the predictive model. We then simulate the model at various values of acceptance rates r by assigning the observations corresponding to the top r fraction of the sorted list to yes decisions (t = 1). We then compute the failure rate on the observations assigned to yes decisions directly from their corresponding ground truth labels.
\end{quote}

\begin{algorithm}[H] 			% enter the algorithm environment
\caption{Evaluator module: Labeled outcomes} 		% give the algorithm a caption
\label{alg:eval:labeled_outcomes} 			% and a label for \ref{} commands later in the document
\begin{algorithmic}[1] 		% enter the algorithmic environment
\REQUIRE Data $\D$ with properties $\{x_i, t_i, y_i\}$, acceptance rate r, knowledge that X affects Y
\ENSURE
\STATE Split data to test set and training set.
\STATE Train a predictive model $\B$ on training data.
\STATE Estimate and assign probability scores $\s$ using $\B$ for all observations in test data.
\STATE Assign observations in test data with observed outcomes (T=1) to $\D_{observed}$.
\STATE Sort $\D_{observed}$ by the probabilities $\s$ to ascending order.
\STATE \hskip3.0em $\rhd$ Now the most dangerous subjects are last.
\STATE Calculate the number to release $N_{free} = |\D_{observed}| \cdot r$.
\RETURN $\frac{1}{|\D_{test}|}\sum_{i=1}^{N_{free}}\delta\{y_i=0\}$
\end{algorithmic}
\end{algorithm}

The performance of human decision-makers (this is the decider in the modular framework) is evaluated with algorithm \ref{alg:eval:human_eval}. From Lakkaraju:

\begin{quote}
This [failure rate estimation for human decision-makers] can be done by grouping decision-makers with similar values of acceptance rate into bins and treating each bin as a single hypothetical decision-maker. We can then compute the failure rate and acceptance rate values for each such bin and plot them as a curve. We refer to this curve as the human evaluation curve.
\end{quote}

\begin{algorithm}[H] 			% enter the algorithm environment
\caption{Evaluator module: Human evaluation} 		% give the algorithm a caption
\label{alg:eval:human_eval} 			% and a label for \ref{} commands later in the document
\begin{algorithmic}[1] 		% enter the algorithmic environment
\REQUIRE Data $\D$ with properties $\{j_i, t_i, y_i\}$, acceptance rate r
\ENSURE
\STATE Assign judges with similar acceptance rate to $\mathcal{J}$
\STATE Assign subjects judged and released by judges in $\mathcal{J}$ to $\D_{released}$.
\RETURN $\frac{1}{|\mathcal{J}|}\sum_{i=1}^{\D_{released}}\delta\{y_i=0\}$
\end{algorithmic}
\end{algorithm}

The initial approach to the selective labels problem is in algorithm \ref{alg:eval:causal_eval} below. In the algorithm, in addition to computing the probability scores $\s$ we also try to evaluate the quantity $F(x_0)$. We can describe $F$ to answer to the question "what fraction of observations have a lower risk than $x_0$?". Now if it is deemed that the fraction given by $F$ is lower than $r$, the subject is released and these decisions are stored in $T_{causal}$. Now the estimated failure rate is the sum of the predicted probability scores of those released (according to $F$) divided by the number of observations. 

\begin{algorithm}[H] 			% enter the algorithm environment
\caption{Evaluator module: Causal evaluator} 		% give the algorithm a caption
\label{alg:eval:causal_eval} 			% and a label for \ref{} commands later in the document
\begin{algorithmic}[1] 		% enter the algorithmic environment
\REQUIRE Data $\D$ with properties $\{x_i, t_i, y_i\}$, acceptance rate r
\ENSURE
\STATE Split data to test set and training set.
\STATE Train a predictive model $\B$ on training data.
\STATE Estimate and assign probability scores $\s$ using $\B$ for all observations in test data.
\FORALL{observations in $\D_{test}$}
	\STATE Evaluate $F(x_i) = \int_{x\in\mathcal{X}} P_x(x)\delta(\B(x)<\B(x_i)) ~dx$ and assign to $\mathcal{F}_{predictions}$ 
\ENDFOR
\STATE Create boolean array $T_{causal} = \mathcal{F}_{predictions} < r$.
\RETURN $\frac{1}{|\D_{test}|}\sum_{i=1}^{|\D_{test}|} \s[i] \cdot T_{causal}[i]$
\end{algorithmic}
\end{algorithm}

The latest approach to the problem is presented below in algorithm \ref{alg:eval:mc}. The high-level idea is to use the counterfactual outcomes Y(1) to impute the missing outcomes and then compute the failure rate. To infer the probability  $\pr(Y(1)=0)$, we need to make some inference about the latent variable Z. We can always infer some information about Z. If features X and the decision T are contradictory, we can infer more about Z than when X and T are aligned.

The algorithm is based on the following equation which expresses the posterior probability of Z after observing T, X and R:
\begin{equation} \label{eq:posterior_Z}
\pr(Z|T, X, R) \propto \pr(T|X, Z, R) \cdot \pr(Z).
\end{equation}
Here $\pr(Z)$ is a prior on $Z$, in this case a standard Gaussian. The second term can formally be written as
\begin{equation} \label{eq:Tprob}
  \pr(T=t|x, z, r)=\begin{cases}
    1-t, & \text{if $\pr(Y=0|x, z, DG) \geq F^{-1}(r)$}\\
    t, & \text{otherwise}
  \end{cases}
\end{equation}
in probabilities or just deterministically
\begin{equation} \label{eq:Tdet}
  T=\begin{cases}
    0, & \text{if $\pr(Y=0|x, z, DG) \geq F^{-1}(r)$}\\
    1, & \text{otherwise}.
  \end{cases}
\end{equation}
In equations \ref{eq:Tprob} and \ref{eq:Tdet}, $\pr(Y=0|x, z, DG)$ is the predicted probability of a negative outcome given x and z. The probability $\pr(Y=0|x, z, DG)$ is predicted by the judge and here we used an approximation that 
\begin{equation}
\pr(Y=0|x, z, DG) = \invlogit(\beta_Xx+\beta_Zz)
\end{equation}
which is an increasing function of $z$ when $x$ is given. Now we do not know the $\beta$ coefficients so here we used the information that they are one. (In the future, they should be inferred.) 

The inverse cumulative function $F^{-1}(r)$ in equations \ref{eq:Tprob} and \ref{eq:Tdet} is the inverse cumulative distribution of \emph{logit-normal distribution} with parameters $\mu=0$ and $\sigma^2=2$, i.e. $F^{-1}$ is the inverse cumulative distribution function of the sum of two standard Gaussians after logistic transformation. If $\beta_X \neq 1$ and/or $\beta_Z \neq 1$ then from the basic properties of variance $\sigma^2=Var(\beta_XX+\beta_ZZ)=\beta_X^2Var(X)+\beta_Z^2Var(Z)$. Finally the inverse cumulative function
\begin{equation} \label{eq:cum_inv}
F^{-1}(r) = \invlogit\left(\text{erf}^{-1}(2r-1)\sqrt{2\sigma^2}-\mu\right)
\end{equation}
where the parameters are as discussed and erf is the error function.

With this knowledge, it can be stated that if we observed $T=0$ with some $x$ and $r$ it must have been that $\invlogit(\beta_Xx+\beta_Zz) \geq F^{-1}(r)$. Using basic algebra we obtain that
\begin{equation} \label{eq:bounds}
\invlogit(x + z) \geq F^{-1}(r) \Leftrightarrow x+z \geq logit(F^{-1}(r)) \Leftrightarrow z \geq logit(F^{-1}(r)) - x
\end{equation}
as the logit and its inverse are strictly increasing functions and hence preserve the order of magnitude for all pairs of values in their domains. From equations \ref{eq:posterior_Z}, \ref{eq:Tprob} and \ref{eq:bounds} we can conclude that $\pr(Z < \invlogit(F^{-1}(r)) - x | T=0, X=x, R=r) = 0$ and that elsewhere the distribution of Z follows a standard Gaussian. The expectation of Z can be computed analytically from the formulas for truncated Gaussians. All this follows analogically for cases with $T=1$ with the changes of some inequalities.

In practise, in lines 1--3 and 10--13 of algorithm \ref{alg:eval:mc} we do as in the True evaluation evaluator algorithm with the distinction that some of the values of Y are imputed with the corresponding counterfactual probabilities. In line 4 we compute the bounds as motivated above. In the for-loop (lines 5--8) we merely compute the expectation of Z given the knowledge of the decision and that the distribution of Z follows a truncated Gaussian. The equation
\begin{equation}
\hat{z} = (1-t) \cdot E(Z | Z > Q_r) + t \cdot E(Z | Z < Q_r)
\end{equation}
computes the correct expectation automatically. Using the expectation, we then compute the probability for the counterfactual $\pr(Y(1) = 0)$ (probability of a negative outcome had a positive decision been given). In line 9 the imputation can be performed in couple of ways: either by taking a random guess with probability $\pr(Y(1) = 0)$ or by assigning the most likely value for Y.

\begin{algorithm}[H] 			% enter the algorithm environment
\caption{Evaluator module: Analytic solution} 		% give the algorithm a caption
\label{alg:eval:mc} 			% and a label for \ref{} commands later in the document
\begin{algorithmic}[1] 		% enter the algorithmic environment
\REQUIRE Data $\D$ with properties $\{x_i, j_i, t_i, y_i\}$, acceptance rate r
\ENSURE
\STATE Split data to test set and training set.
\STATE Train a predictive model $\B$ on training data.
\STATE Estimate and assign probability scores $\s$ using $\B$ for all observations in test data.
\STATE Compute bounds $Q_r = logit(F^{-1}(r)) - x$ for all judges.
\FORALL{observations in test set}
	\STATE Compute expectation $\hat{z} = (1-t) \cdot E(Z | Z > Q_r) + t \cdot E(Z | Z < Q_r)$. % 
	\STATE Compute $\pr(Y(1) = 0) = \invlogit(x + \hat{z})$.
\ENDFOR
\STATE Impute missing observations using the estimates $\pr(Y(1) = 0)$.
\STATE Sort the data by the probabilities $\s$ to ascending order.
\STATE \hskip3.0em $\rhd$ Now the most dangerous subjects are last.
\STATE Calculate the number to release $N_{free} = |\D_{test}| \cdot r$.
\RETURN Compute $\frac{1}{|\D_{test}|}\sum_{i=1}^{N_{free}}\delta\{y_i=0\}$ using the observed and imputed observations.
\end{algorithmic}
\end{algorithm}

\subsection{Summary table}

Summary table of different modules.

\begin{table}[H]
  \centering
  \caption{Summary of modules (under construction)}
  \begin{tabular}{lll}
    \toprule
    \multicolumn{3}{c}{Module type} \\[.5\normalbaselineskip]
    \textbf{Data generator} & \textbf{Decider} & \textbf{Evaluator}  \\
    \midrule
    {\ul Without unobservables}	& {\ul Independent decisions}	& {\ul Labeled outcomes} \\
     					 	& 1. draw T from a Bernoulli	& \tabitem Data $\D$ with properties $\{x_i, t_i, y_i\}$ \\
    {\ul With unobservables}       	& with $P(T=0|X, Z)$			& \tabitem acceptance rate r \\
    \tabitem $P(Y=0|X, Z, W)$ 	& 						& \tabitem knowledge that X affects Y \\[.5\normalbaselineskip]
    
     {\ul With unobservables}	& 2. determine with $F^{-1}(r)$		& {\ul True evaluation} \\
     \tabitem assign $Y=1$		& 							& \tabitem Data $\D$ with properties $\{x_i, t_i, y_i\}$ \\
     if $P(Y=0|X, Z, W) \geq 0.5$	& {\ul Non-independent decisions}	& and \emph{all outcome labels} \\
     						& 3. sort by $P(T=0|X, Z)$			& \tabitem acceptance rate r \\
     						& and assign $t$ by $r$			& \tabitem knowledge that X affects Y \\[.5\normalbaselineskip]
     
     &  & {\ul Human evaluation} \\
     &  & \tabitem Data $\D$ with properties $\{x_i, j_i, t_i, y_i\}$ \\
     &  & \tabitem acceptance rate r \\[.5\normalbaselineskip]
     
     &  & {\ul Contraction algorithm} \\
     &  & \tabitem Data $\D$ with properties $\{x_i, j_i, t_i, y_i\}$ \\
     &  & \tabitem acceptance rate r \\
     &  & \tabitem knowledge that X affects Y \\[.5\normalbaselineskip]
     
%     &  & {\ul Causal model} \\
%     &  & \tabitem Data $\D$ with properties $\{x_i, t_i, y_i\}$ \\
%     &  & \tabitem acceptance rate r \\
%     &  & \tabitem knowledge that X affects Y \\[.5\normalbaselineskip]
     
     &  & {\ul Analytic solution} \\
     &  & \tabitem Data $\D$ with properties $\{x_i, j_i, t_i, y_i\}$ \\
     &  & \tabitem acceptance rate r \\
     &  & \tabitem knowledge that X affects Y \\
     &  & \tabitem more intricate knowledge about $\M$ ? \\[.5\normalbaselineskip]
     
     &  & {\ul Potential outcomes evaluator} \\
     &  & \tabitem Data $\D$ with properties $\{x_i, j_i, t_i, y_i\}$ \\
     &  & \tabitem acceptance rate r \\
     &  & \tabitem knowledge that X affects Y \\[.5\normalbaselineskip]
    \bottomrule
  \end{tabular}
  \label{tab:modules}
\end{table}

\section{Old results} \label{sec:results}

Results obtained from running algorithm \ref{alg:perf_comp} are presented in table \ref{tab:results} and figure \ref{fig:results}. All parameters are in their default values and a logistic regression model is trained.

\begin{table}[H]
\centering
\caption{Mean absolute error (MAE) w.r.t true evaluation. \\ \emph{RL: Updated 26 June.}}
\begin{tabular}{l | c c}
Method & MAE without Z & MAE with Z \\ \hline
Labeled outcomes 	& 0.107249375 	& 0.0827844\\
Human evaluation 	& 0.002383729 	& 0.0042517\\
Contraction 		& 0.004633164		& 0.0075497\\
Causal model, ep 	& 0.000598624 	& 0.0411532\\
\end{tabular}
\label{tab:results}
\end{table}


\begin{figure}[]
    \centering
    \begin{subfigure}[b]{0.5\textwidth}
        \includegraphics[width=\textwidth]{sl_without_Z_8iter}
        \caption{Results without unobservables}
        \label{fig:results_without_Z}
    \end{subfigure}
    ~ %add desired spacing between images, e. g. ~, \quad, \qquad, \hfill etc. 
      %(or a blank line to force the subfigure onto a new line)
    \begin{subfigure}[b]{0.5\textwidth}
        \includegraphics[width=\textwidth]{sl_with_Z_8iter_betaZ_1_0}
        \caption{Results with unobservables, $\beta_Z=1$.}
        \label{fig:results_with_Z}
    \end{subfigure}
    \caption{Failure rate vs. acceptance rate with varying levels of leniency. Logistic regression was trained on labeled training data. \emph{RL: Updated 26 June.}}
    \label{fig:results}
\end{figure}

\subsection{$\beta_Z=0$ and data generated with unobservables.}

If we assign $\beta_Z=0$, almost all failure rates drop to zero in the interval 0.1, ..., 0.3 but the human evaluation failure rate. Results are presented in figures \ref{fig:betaZ_1_5} and \ref{fig:betaZ_0}. 

The disparities between figures \ref{fig:results_without_Z} and \ref{fig:betaZ_0} (result without unobservables and with $\beta_Z=0$) can be explained in the slight difference in the data generating process, namely the effect of $\epsilon$. The effect of adding $\epsilon$ (noise to the decisions) is further explored in section \ref{sec:epsilon}.

\begin{figure}[]
    \centering
    \begin{subfigure}[b]{0.475\textwidth}
        \includegraphics[width=\textwidth]{sl_with_Z_4iter_betaZ_1_5}
        \caption{Results with unobservables, $\beta_Z$ set to 1.5 in algorithm \ref{alg:data_with_Z}.}
        \label{fig:betaZ_1_5}
    \end{subfigure}
    \quad %add desired spacing between images, e. g. ~, \quad, \qquad, \hfill etc. 
      %(or a blank line to force the subfigure onto a new line)
    \begin{subfigure}[b]{0.475\textwidth}
        \includegraphics[width=\textwidth]{sl_with_Z_4iter_beta0}
        \caption{Results with unobservables, $\beta_Z$ set to 0 in algorithm \ref{alg:data_with_Z}.}
        \label{fig:betaZ_0}
    \end{subfigure}
    \caption{Effect of $\beta_z$. Failure rate vs. acceptance rate with unobservables in the data (see algorithm \ref{alg:data_with_Z}). Logistic regression was trained on labeled training data. Results from algorithm \ref{alg:perf_comp}.}
    \label{fig:betaZ_comp}
\end{figure}

\subsection{Noise added to the decision and data generated without unobservables} \label{sec:epsilon}

In this part, Gaussian noise with zero mean and 0.1 variance was added to the probabilities $P(Y=0|X=x)$ after sampling Y but before ordering the observations in line 5 of algorithm \ref{alg:data_without_Z}. Results are presented in Figure \ref{fig:sigma_figure}.

\begin{figure}[]
    \centering
    \includegraphics[width=0.5\textwidth]{sl_without_Z_3iter_sigma_sqrt_01}
    \caption{Failure rate with varying levels of leniency without unobservables. Noise has been added to the decision probabilities. Logistic regression was trained on labeled training data.}
    \label{fig:sigma_figure}
\end{figure}

\subsection{Predictions with random forest classifier} \label{sec:random_forest}

In this section the predictive model was switched to random forest classifier to examine the effect of changing the predictive model. Results are practically identical to those presented in figure \ref{fig:results} previously and are presented in figure \ref{fig:random_forest}.

\begin{figure}[]
    \centering
    \begin{subfigure}[b]{0.475\textwidth}
        \includegraphics[width=\textwidth]{sl_withoutZ_4iter_randomforest}
        \caption{Results without unobservables.}
        \label{fig:results_without_Z_rf}
    \end{subfigure}
    \quad %add desired spacing between images, e. g. ~, \quad, \qquad, \hfill etc. 
      %(or a blank line to force the subfigure onto a new line)
    \begin{subfigure}[b]{0.475\textwidth}
        \includegraphics[width=\textwidth]{sl_withZ_6iter_betaZ_1_0_randomforest}
        \caption{Results with unobservables, $\beta_Z=1$.}
        \label{fig:results_with_Z_rf}
    \end{subfigure}
    \caption{Failure rate vs. acceptance rate with varying levels of leniency. Random forest classifier was trained on labeled training data}
    \label{fig:random_forest}
\end{figure}

\subsection{Sanity check for predictions}

Predictions were checked by drawing a graph of predicted Y versus X, results are presented in figure \ref{fig:sanity_check}. The figure indicates that the predicted class labels and the probabilities for them are consistent with the ground truth.

\begin{figure}[]
    \centering
    \includegraphics[width=0.5\textwidth]{sanity_check}
    \caption{Predicted class label and probability of $Y=1$ versus X. Prediction was done with a logistic regression model. Colors of the points denote ground truth (yellow = 1, purple = 0). Data set was created with the unobservables.}
    \label{fig:sanity_check}
\end{figure}

\subsection{Fully random model $\M$}

Given our framework defined in section \ref{sec:framework}, the results presented next are with model $\M$ that outputs probabilities 0.5 for every instance of $x$. Labeling process is still as presented in algorithm \ref{alg:data_with_Z}.  

\begin{figure}[]
    \centering
    \begin{subfigure}[b]{0.475\textwidth}
        \includegraphics[width=\textwidth]{sl_without_Z_15iter_random_model}
        \caption{Failure rate vs. acceptance rate. Data without unobservables. Machine predictions with random model.}
        \label{fig:random_predictions_without_Z}
    \end{subfigure}
    \quad %add desired spacing between images, e. g. ~, \quad, \qquad, \hfill etc. 
      %(or a blank line to force the subfigure onto a new line)
    \begin{subfigure}[b]{0.475\textwidth}
        \includegraphics[width=\textwidth]{sl_with_Z_15iter_fully_random_model}
        \caption{Failure rate vs. acceptance rate. Data with unobservables. Machine predictions with random model.}
        \label{fig:random_predictions_with_Z}
    \end{subfigure}
    \caption{Failure rate vs. acceptance rate with varying levels of leniency. Machine predictions were done with completely random model, that is prediction $P(Y=0|X=x)=0.5$ for all $x$.}
    \label{fig:random_predictions}
\end{figure}

\subsection{Modular framework -- Monte Carlo evaluator} \label{sec:modules_mc}

For these results, data was generated either with module in algorithm \ref{alg:dg:coinflip_with_z} (drawing Y from Bernoulli distribution with parameter $\pr(Y=0|X, Z, W)$ as previously) or with module in algorithm \ref{alg:dg:threshold_with_Z} (assign Y based on the value of $\invlogit(\beta_XX+\beta_ZZ)$). Decisions were determined using one of the two modules: module in algorithm \ref{alg:decider:quantile} (decision based on quantiles) or \ref{alg:decider:lakkaraju} ("human" decision-maker as in \cite{lakkaraju17}). Curves were computed with True evaluation (algorithm \ref{alg:eval:true_eval}), Labeled outcomes (\ref{alg:eval:labeled_outcomes}), Human evaluation (\ref{alg:eval:human_eval}), Contraction (\ref{alg:eval:contraction}) and Monte Carlo evaluators (\ref{alg:eval:mc}). Results are presented in figure \ref{fig:modules_mc}. The corresponding MAEs are presented in table \ref{tab:modules_mc}.

From the result table we can see that the MAE is at the lowest when the data generating process corresponds closely to the Monte Carlo algorithm.

\begin{table}[]
\centering
\caption{Mean absolute error w.r.t true evaluation. See modules used in section \ref{sec:modules_mc}. Bern = Bernoulli,  indep. = independent, TH = threshold}
\begin{tabular}{l | c c c c}
Method & Bern + indep. & Bern + non-indep. & TH + indep. & TH + non-indep.\\ \hline
Labeled outcomes 	& 0.111075	& 0.103235	& 0.108506 & 0.0970325\\
Human evaluation 	& 0.027298	& NaN (TBA)	& 0.049582 & 0.0033916\\
Contraction 		& 0.004206	& 0.004656	& 0.005557 & 0.0034591\\
Monte Carlo	 	& 0.001292	& 0.016629	& 0.009429 & 0.0179825\\
\end{tabular}
\label{tab:modules_mc}
\end{table}


\begin{figure}[]
    \centering
    \begin{subfigure}[b]{0.475\textwidth}
        \includegraphics[width=\textwidth]{sl_with_Z_10iter_coinflip_quantile_defaults_mc}
        \caption{Outcome Y from Bernoulli, independent decisions using the quantiles.}
        %\label{fig:modules_mc_without_Z}
    \end{subfigure}
    \quad %add desired spacing between images, e. g. ~, \quad, \qquad, \hfill etc. 
      %(or a blank line to force the subfigure onto a new line)
    \begin{subfigure}[b]{0.475\textwidth}
        \includegraphics[width=\textwidth]{sl_with_Z_20iter_threshold_quantile_defaults_mc}
        \caption{Outcome Y from threshold rule, independent decisions using the quantiles.}
        %\label{fig:modules_mc_with_Z}
    \end{subfigure}
    \begin{subfigure}[b]{0.475\textwidth}
        \includegraphics[width=\textwidth]{sl_with_Z_10iter_coinflip_lakkarajudecider_defaults_mc}
        \caption{Outcome Y from Bernoulli, non-independent decisions.}
        %\label{fig:modules_mc_without_Z}
    \end{subfigure}
    \quad %add desired spacing between images, e. g. ~, \quad, \qquad, \hfill etc. 
      %(or a blank line to force the subfigure onto a new line)
    \begin{subfigure}[b]{0.475\textwidth}
        \includegraphics[width=\textwidth]{sl_with_Z_10iter_threshold_lakkarajudecider_defaults_mc}
        \caption{Outcome Y from threshold rule, non-independent decisions.}
        %\label{fig:modules_mc_with_Z}
    \end{subfigure}
    \caption{Failure rate vs. acceptance rate with varying levels of leniency. Different combinations of deciders and data generation modules. See other modules used in section \ref{sec:modules_mc}}
    \label{fig:modules_mc}
\end{figure}

\section{Diagnostic figures} \label{sec:diagnostic}

Here we present supplementary figures of all the settings in the main result section.

\begin{figure}[]
    \centering
    \includegraphics[width=\textwidth]{sl_diagnostic_bernoulli_independent_without_Z}
    \caption{Results from estimating failure rate with different levels of leniency using different methods.}
    %\label{fig:}
\end{figure}

\begin{figure}[]
    \centering
    \includegraphics[width=\textwidth]{sl_diagnostic_bernoulli_independent_with_Z}
    \caption{Results from estimating failure rate with different levels of leniency using different methods.}
    %\label{fig:}
\end{figure}

\begin{figure}[]
    \centering
    \includegraphics[width=\textwidth]{sl_diagnostic_threshold_independent_with_Z}
    \caption{Results from estimating failure rate with different levels of leniency using different methods.}
    %\label{fig:}
\end{figure}

\begin{figure}[]
    \centering
    \includegraphics[width=\textwidth]{sl_diagnostic_bernoulli_batch_with_Z}
    \caption{Results from estimating failure rate with different levels of leniency using different methods.}
    %\label{fig:}
\end{figure}

\begin{figure}[]
    \centering
    \includegraphics[width=\textwidth]{sl_diagnostic_threshold_batch_with_Z}
    \caption{Results from estimating failure rate with different levels of leniency using different methods.}
    %\label{fig:}
\end{figure}

\begin{figure}[]
    \centering
    \includegraphics[width=\textwidth]{sl_diagnostic_random_decider_with_Z}
    \caption{Results from estimating failure rate with different levels of leniency using different methods.}
    %\label{fig:}
\end{figure}

\begin{figure}[]
    \centering
    \includegraphics[width=\textwidth]{sl_diagnostic_biased_decider_with_Z}
    \caption{Results from estimating failure rate with different levels of leniency using different methods.}
    %\label{fig:}
\end{figure}

\begin{figure}[]
    \centering
    \includegraphics[width=\textwidth]{sl_diagnostic_bad_decider_with_Z}
    \caption{Results from estimating failure rate with different levels of leniency using different methods.}
    %\label{fig:}
\end{figure}

%\end{appendices}

\end{document}