notes.tex

The following hierarchical model was used as an initial approach to the problem. Data was generated with unobservables and both outcome Y and decision T were drawn from Bernoulli distributions. The $\beta$ coefficients were systematically overestimated as shown in figure \ref{fig:posteriors}.

\begin{align} \label{eq1}
 1-t~|~x,~z,~\beta_x,~\beta_z & \sim \text{Bernoulli}(\invlogit(\beta_xx + \beta_zz)) \\ \nonumber
 Z &\sim N(0, 1) \\ \nonumber
% \alpha_j & \sim N(0, 100), j \in \{1, \ldots, N_{judges} \} \\
  \beta_x & \sim N(0, 10^2) \\ \nonumber
  \beta_z & \sim N_+(0, 10^2) 
\end{align}


\begin{figure}[]
    \centering
    \begin{subfigure}[b]{0.475\textwidth}
        \includegraphics[width=\textwidth]{sl_posterior_betax}
        \caption{Posterior of $\beta_x$.}
        %\label{fig:random_predictions_without_Z}
    \end{subfigure}
    \quad %add desired spacing between images, e. g. ~, \quad, \qquad, \hfill etc. 
      %(or a blank line to force the subfigure onto a new line)
    \begin{subfigure}[b]{0.475\textwidth}
        \includegraphics[width=\textwidth]{sl_posterior_betaz}
        \caption{Posterior of $\beta_z$.}
        \label{fig:posteriors}
    \end{subfigure}
    \caption{Coefficient posteriors from model \ref{eq1}.}
    %\label{fig:random_predictions}
\end{figure}

\newpage

\subsection{Summary table}

Summary table of different modules.

\begin{table}[H]
  \centering
  \caption{Summary of modules (under construction)}
  \begin{tabular}{lll}
    \toprule
    \multicolumn{3}{c}{Module type} \\[.5\normalbaselineskip]
    \textbf{Data generator} & \textbf{Decider} & \textbf{Evaluator}  \\
    \midrule
    {\ul Without unobservables}	& Independent decisions		& {\ul Labeled outcomes} \\
     					 	& 1. draw T from a Bernoulli	& \tabitem Data $\D$ with properties $\{x_i, t_i, y_i\}$ \\
    {\ul With unobservables}       	& with $P(T=0|X, Z)$			& \tabitem acceptance rate r \\
    \tabitem $P(Y=0|X, Z, W)$ 	& 2. determine with $F^{-1}(r)$	& \tabitem knowledge that X affects Y \\[.5\normalbaselineskip]
    
     {\ul With unobservables}	& Non-independent decisions  	& {\ul True evaluation} \\
     \tabitem assign Y by		& 3. sort by $P(T=0|X, Z)$		& \tabitem Data $\D$ with properties $\{x_i, t_i, y_i\}$ \\
     "threshold rule"			& and assign $t$ by $r$  		& and \emph{all outcome labels} \\
     						&   						& \tabitem acceptance rate r \\
     						&   						& \tabitem knowledge that X affects Y \\[.5\normalbaselineskip]
     
     &  & {\ul Human evaluation} \\
     &  & \tabitem Data $\D$ with properties $\{x_i, j_i, t_i, y_i\}$ \\
     &  & \tabitem acceptance rate r \\[.5\normalbaselineskip]
     
     &  & {\ul Contraction algorithm} \\
     &  & \tabitem Data $\D$ with properties $\{x_i, j_i, t_i, y_i\}$ \\
     &  & \tabitem acceptance rate r \\
     &  & \tabitem knowledge that X affects Y \\[.5\normalbaselineskip]
     
     &  & {\ul Causal model} \\
     &  & \tabitem Data $\D$ with properties $\{x_i, t_i, y_i\}$ \\
     &  & \tabitem acceptance rate r \\
     &  & \tabitem knowledge that X affects Y \\[.5\normalbaselineskip]
     
     &  & {\ul Monte Carlo evaluator} \\
     &  & \tabitem Data $\D$ with properties $\{x_i, j_i, t_i, y_i\}$ \\
     &  & \tabitem acceptance rate r \\
     &  & \tabitem knowledge that X affects Y \\
     &  & \tabitem more intricate knowledge about $\M$ ? \\[.5\normalbaselineskip]
    \bottomrule
  \end{tabular}
  \label{tab:modules}
\end{table}

\begin{thebibliography}{9} % Might have been apa

\bibitem{dearteaga18}
   De-Arteaga, Maria. Learning Under Selective Labels in the Presence of Expert Consistency. 2018. 
\bibitem{lakkaraju17} 
   Lakkaraju, Himabindu. The Selective Labels Problem: Evaluating Algorithmic Predictions in the Presence of Unobservables. 2017. 

\end{thebibliography}

\end{document}