diff --git a/analysis_and_scripts/notes.tex b/analysis_and_scripts/notes.tex
index 2992b2f53ba996c253f58dd3be5d6b3749232635..a29e8ce77486c2e55ad070e01dae15873195f2c4 100644
--- a/analysis_and_scripts/notes.tex
+++ b/analysis_and_scripts/notes.tex
@@ -7,40 +7,42 @@
\usepackage{amssymb}
\usepackage{epstopdf}
\usepackage{hyperref}
-\DeclareGraphicsRule{.tif}{png}{.png}{`convert #1 `dirname #1`/`basename #1 .tif`.png}
+%\DeclareGraphicsRule{.tif}{png}{.png}{`convert #1 `dirname #1`/`basename #1 .tif`.png}
\usepackage{algorithm}% http://ctan.org/pkg/algorithms
\usepackage{algorithmic}% http://ctan.org/pkg/algorithms
\renewcommand{\algorithmicrequire}{\textbf{Input:}}
\renewcommand{\algorithmicensure}{\textbf{Procedure:}}
+\usepackage{subcaption}
+\graphicspath{ {../figures/} }
+
\title{Notes}
-\author{RL, 10 June 2019}
+\author{RL, 11 June 2019}
%\date{} % Activate to display a given date or no date
\begin{document}
\maketitle
-
\begin{abstract}
-This document presents the implementations of RL in pseudocode level.
+This document presents the implementations of RL in pseudocode level. First, I present the nomenclature used in these notes. Then I proceed to give my personal views and comments on the motivation behind Selective labels paper. In the following sections, I present the data generating algorithms and algorithms for obtaining failure rates using different methods. In the end I present some some results that I was asked to present in the meeting Friday $7^{th}$.
\end{abstract}
\section*{Terms and abbreviations}
\begin{description}
-\item[R] acceptance, leniency of decision maker, $r \in [0, 1]$
+\item[R] acceptance rate, leniency of decision maker, $r \in [0, 1]$
\item[X] personal features, observable to a predictive model
\item[Z] some features of a subject, unobservable to a predictive model, latent variable
\item[W] noise added to result variable Y
\item[T] decision variable, bail/positive decision equal to 1, jail/negative decision equal to 0
\item[Y] result variable, no crime/positive result equal to 1, crime/negative result equal to 0
\item[SL] Selective labels
-\item[Labeled data] data that has been censored, i.e. if negative decision is given (T=0), then Y is set to NA.
-\item[Unobservables] unmeasured confounders, latent variables, Z
+\item[Labeled data] data that has been censored, i.e. if negative decision is given (T=0), then Y is set to NA.
+\item[Unobservables] unmeasured confounders, latent variables, Z
\end{description}
-Rule of thumb for the binary coding: zero bad (crime or jail), one good!
+Mnemonic rule for the binary coding: zero bad (crime or jail), one good!
\section{RL's notes about the selective labels paper (optional reading)}
@@ -104,7 +106,7 @@ In the setting with unobservables Z, we first sample an acceptance rate r for al
\subsection{Model fitting}
-The models that are being fitted are logistic regression models from scikit-learn package. The solver is set to lbfgs and intercept is estimated by default. The resulting LogisticRegression model object provides convenient functions for fitting the model and predicting probabilities for class labels. Please see the documentation at \url{https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html} or ask me (RL) for more details.
+The models that are being fitted are logistic regression models from scikit-learn package. The solver is set to lbfgs (as there is no closed-form solution) and intercept is estimated by default. The resulting LogisticRegression model object provides convenient functions for fitting the model and getting probabilities for class labels. Please see the documentation at \url{https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html} or ask me (RL) for more details.
NB: These models can not be fitted if the data includes missing values. Therefore listwise deletion is done in cases of missing data (whole record is discarded).
@@ -117,7 +119,7 @@ The following quantities are estimated from the data:
\item Labeled outcomes: The "traditional"/vanilla estimate of model performance. See algorithm \ref{alg:labeled_outcomes}.
\item Human evaluation: The failure rate of human decision-makers who have acces to the latent variable Z. Decision-makers with similar values of leniency are binned and trated as one hypothetical decision-maker. See algorithm \ref{alg:human_eval}.
\item Contraction: See algorithm 1 of \cite{lakkaraju17}
-\item Causal model: In essence, the empirical performance is calculated over the test set as $$\dfrac{1}{n}\sum_{(x, y)\in D}f(x)\delta(F(x) < r)$$ where $$f(x) = P(Y=0|T=1, X=x)$$ is a predictive model trained on the labeled data and $$ F(x_0) = \int P(x)\delta(f(x) > f(x_0)) ~ dx.$$ $P(x)$ is Gaussian pdf from scipy.stats package and it is integrated over interval [-15, 15] with 40000 steps using si.simps function from scipy.integrate which uses Simpson's rule in estimating the value of the integral. (docs: \url{https://docs.scipy.org/doc/scipy/reference/generated/scipy.integrate.simps.html})
+\item Causal model: In essence, the empirical performance is calculated over the test set as $$\dfrac{1}{n}\sum_{(x, y)\in D}f(x)\delta(F(x) < r)$$ where $$f(x) = P(Y=0|T=1, X=x)$$ is a predictive model trained on the labeled data and $$ F(x_0) = \int P(x)\delta(f(x) < f(x_0)) ~ dx.$$ All observations, even ones with missing outcome labels, can be used since empirical performance doesn't depend on them. $P(x)$ is Gaussian pdf from scipy.stats package and it is integrated over interval [-15, 15] with 40000 steps using si.simps function from scipy.integrate which uses Simpson's rule in estimating the value of the integral. (docs: \url{https://docs.scipy.org/doc/scipy/reference/generated/scipy.integrate.simps.html}) \label{causal_cdf}
\end{itemize}
The plotted curves are constructed using pseudo code presented in algorithm \ref{alg:perf_comp}.
@@ -193,11 +195,37 @@ The plotted curves are constructed using pseudo code presented in algorithm \ref
\begin{algorithmic}[1] % enter the algorithmic environment
\REQUIRE Labeled test data $\mathcal{D}$ with probabilities $\mathcal{S}$ and \emph{missing outcome labels} for observations with ($T=0$), predictive model f, acceptance rate r
\ENSURE
-\STATE $T_{causal} = cdf(\mathcal{D}, f) < r$
+\STATE Create boolean array $T_{causal} = cdf(\mathcal{D}, f) < r$. See "Causal model" in \ref{causal_cdf}.
\RETURN $\frac{1}{|\mathcal{D}|}\sum_{i=1}^{\mathcal{D}} \mathcal{S} \cdot T_{causal} = \frac{1}{|\mathcal{D}|}\sum_x f(x)\delta(F(x) < r)$
\end{algorithmic}
\end{algorithm}
+
+\section{What if...}
+
+\subsection{We assign $\beta_Z=0$?}
+
+If we assign $\beta_Z=0$, almost all failure rates drop to zero in the interval 0.1, ..., 0.3 but the human evaluation failure rate. Figures are drawn in Figures \ref{fig:betaZ_1_5} and \ref{fig:betaZ_0}.
+
+\begin{figure}
+ \centering
+ \begin{subfigure}[b]{0.5\textwidth}
+ \includegraphics[width=\textwidth]{sl_with_Z_4iter_betaZ_1_5}
+ \caption{$\beta_Z=1.5$}
+ \label{fig:betaZ_1_5}
+ \end{subfigure}
+ ~ %add desired spacing between images, e. g. ~, \quad, \qquad, \hfill etc.
+ %(or a blank line to force the subfigure onto a new line)
+ \begin{subfigure}[b]{0.5\textwidth}
+ \includegraphics[width=\textwidth]{sl_with_Z_4iter_beta0}
+ \caption{$\beta_Z=0$}
+ \label{fig:betaZ_0}
+ \end{subfigure}
+ \caption{Failure rate with varying levels of leniency and unobservables. Results from algorithm \ref{alg:perf_comp} with $N_{iter}=4$.}\label{fig:betaZ_comp}
+\end{figure}
+
+%\subsection{}
+
\begin{thebibliography}{9}
\bibitem{dearteaga18}
diff --git a/figures/sl_with_Z_4iter_betaZ_1,5.png b/figures/sl_with_Z_4iter_betaZ_1_5.png
similarity index 100%
rename from figures/sl_with_Z_4iter_betaZ_1,5.png
rename to figures/sl_with_Z_4iter_betaZ_1_5.png