Appendix figures 2/3

paper/appendix.tex

-
-
 \documentclass{article}
 \usepackage{float}
 \usepackage[caption = false]{subfig}
 \usepackage[final]{graphicx}
 
+\usepackage{mathtools}
+\usepackage{xspace}     % fix space in macros
+\usepackage{units}     % nicer slanted fractions
+
+\input{macros}
+\usepackage{chato-notes}
+
+
 
 \usepackage[margin=0.5in]{geometry}
+
 \begin{document}
 
 \appendix
@@ -15,23 +22,144 @@
 
 
 \begin{figure}[H]
-\subfloat[Random H, Random M]{\includegraphics[width = 3in]{./img/random_H_random_M}} ~
-\subfloat[Random H, batch M]{\includegraphics[width = 3in]{./img/random_H_batch_M}}\\
-\subfloat[Batch H, Random M]{\includegraphics[width = 3in, height = 2in]{example-image}}~
-\subfloat[Batch H, Batch M]{\includegraphics[width = 3in]{./img/lakkarajus_H_batch_M}} \\
-\subfloat[Independent H and Random M]{\includegraphics[width = 3in, height = 2in]{example-image}} ~
-\subfloat[Independent H, Batch M]{\includegraphics[width = 3in]{./img/independent_H_batch_M}}\\
-\subfloat[Probabilistic H, Random M]{\includegraphics[width = 3in]{./img/probabilistic_H_random_M}}~
-\subfloat[Probabilistic H, Batch M]{\includegraphics[width = 3in]{./img/probabilistic_H_batch_M}}\\
-
-\caption{Figures with different configs.}
+\subfloat[Random H, Random M]{\includegraphics[width = 3in]{./img/_deciderH_random_deciderM_random_maxR_0_9coefZ1_0_all}} ~
+\subfloat[Random H, batch M]{\includegraphics[width = 3in]{./img/_deciderH_random_deciderM_batch_maxR_0_9coefZ1_0_all}}\\
+\subfloat[Batch H, Random M]{\includegraphics[width = 3in]{./img/_deciderH_batch_deciderM_random_maxR_0_9coefZ1_0_all}}~
+\subfloat[Batch H, Batch M]{\includegraphics[width = 3in]{./img/_deciderH_batch_deciderM_batch_maxR_0_9coefZ1_0_all}} \\
+\subfloat[Independent H and Random M]{\includegraphics[width = 3in]{./img/_deciderH_independent_deciderM_random_maxR_0_9coefZ1_0_all}} ~
+\subfloat[Independent H, Batch M]{\includegraphics[width = 3in]{./img/_deciderH_independent_deciderM_batch_maxR_0_9coefZ1_0_all}}\\
+\subfloat[Probabilistic H, Random M]{\includegraphics[width = 3in, height = 1.5in]{example-image}}~
+\subfloat[Probabilistic H, Batch M]{\includegraphics[width = 3in, height = 1.5in]{example-image}}\\
+
+\caption{Figures with different deciders (N=5k, 50 judges, max$(r)=0.9,~ \beta_z=\gamma_z=1$).}
 \label{some example}
 \end{figure}
 
+%%
+
 \begin{figure}[H]
-\subfloat[Probabilistic H, batch M, $\beta_Z=\gamma_Z=5$]{\includegraphics[width = 3in]{./img/probabilistic_H_batch_M_coef_betaZ_5}} ~
-\subfloat[Probabilistic H, batch M, max$(r)=0.5$.]{\includegraphics[width = 3in]{./img/sl_rmax05}}
-\caption{Figures with different configs. (cont.)}
+\subfloat[Random H, Random M]{\includegraphics[width = 3in]{./img/_deciderH_random_deciderM_random_maxR_0_9coefZ5_0_all}} ~
+\subfloat[Random H, batch M]{\includegraphics[width = 3in]{./img/_deciderH_random_deciderM_batch_maxR_0_9coefZ5_0_all}}\\
+\subfloat[Batch H, Random M]{\includegraphics[width = 3in]{./img/_deciderH_batch_deciderM_random_maxR_0_9coefZ5_0_all}}~
+\subfloat[Batch H, Batch M]{\includegraphics[width = 3in]{./img/_deciderH_batch_deciderM_batch_maxR_0_9coefZ5_0_all}} \\
+\subfloat[Independent H and Random M]{\includegraphics[width = 3in]{./img/_deciderH_independent_deciderM_random_maxR_0_9coefZ5_0_all}} ~
+\subfloat[Independent H, Batch M]{\includegraphics[width = 3in]{./img/_deciderH_independent_deciderM_batch_maxR_0_9coefZ5_0_all}}\\
+\subfloat[Probabilistic H, Random M]{\includegraphics[width = 3in]{./img/_deciderH_probabilistic_deciderM_random_maxR_0_9coefZ5_0_all}}~
+\subfloat[Probabilistic H, Batch M]{\includegraphics[width = 3in, height = 1.5in]{example-image}}\\
+
+\caption{Figures with different deciders (N=5k, 50 judges, max$(r)=0.9, \beta_z=\gamma_z=5$).}
 \label{some example}
 \end{figure}
+
+%%%
+%
+%\begin{figure}[H]
+%\subfloat[Random H, Random M]{\includegraphics[width = 3in]{./img/random_H_random_M}} ~
+%\subfloat[Random H, batch M]{\includegraphics[width = 3in]{./img/random_H_batch_M}}\\
+%\subfloat[Batch H, Random M]{\includegraphics[width = 3in, height = 1.5in]{example-image}}~
+%\subfloat[Batch H, Batch M]{\includegraphics[width = 3in]{./img/lakkarajus_H_batch_M}} \\
+%\subfloat[Independent H and Random M]{\includegraphics[width = 3in, height = 1.5in]{example-image}} ~
+%\subfloat[Independent H, Batch M]{\includegraphics[width = 3in]{./img/independent_H_batch_M}}\\
+%\subfloat[Probabilistic H, Random M]{\includegraphics[width = 3in]{./img/probabilistic_H_random_M}}~
+%\subfloat[Probabilistic H, Batch M]{\includegraphics[width = 3in]{./img/probabilistic_H_batch_M}}\\
+%
+%\caption{Figures with different configs.}
+%\label{some example}
+%\end{figure}
+%
+%%%
+%
+%
+%\begin{figure}[H]
+%\subfloat[Probabilistic H, batch M, $\beta_Z=\gamma_Z=5$]{\includegraphics[width = 3in]{./img/probabilistic_H_batch_M_coef_betaZ_5}} ~
+%\subfloat[Probabilistic H, batch M, max$(r)=0.5$.]{\includegraphics[width = 3in]{./img/sl_rmax05}}
+%\caption{Figures with different configs. (cont.)}
+%\label{some example}
+%\end{figure}
+
+\newpage
+
+\section{Technical details}
+
+\note{Riku}{From KDD: ''In addition, authors can provide an optional two (2) page supplement at the end of their submitted paper (it needs to be in the same PDF file and start at page 10) focused on reproducibility. This supplement can only be used to include (i) information necessary for reproducing the experimental results, insights, or conclusions reported in the paper (e.g., various algorithmic and model parameters and configurations, hyper-parameter search spaces, details related to dataset filtering and train/test splits, software versions, detailed hardware configuration, etc.), and (ii) any pseudo-code, or proofs that due to space limitations, could not be included in the main nine-page manuscript, but that help in reproducibility (see reproducibility policy below for more details).''}
+
+Specify the following for the technical appendix:
+
+\begin{itemize}
+\item Computing environment, versions of
+	\begin{itemize}
+	\item Python
+	\item Stan
+	\item ???
+	\end{itemize}
+\item Full model specification
+\item Replication specifics, see above from their requirements
+\end{itemize}
+
+\subsection{Model definition} \label{sec:model_definition}
+
+\note{Copied from sec 3.5}{Riku}
+
+The causal diagram of Figure~\ref{fig:causalmodel} provides the structure of causal relationships for quantities of interest.
+%
+In addition, we consider \judgeAmount instances $\{\human_j, j = 1, 2, \ldots, \judgeAmount\}$ of decision makers \human.
+%
+For the purposes of Bayesian modelling, we present the hierarchical model and explicate our assumptions about the relationships and the quantities below.
+%
+Note that index $j$ refers to decision maker $\human_j$ and \invlogit is the standard logistic function.
+
+\noindent
+\hrulefill
+\begin{align}
+\prob{\unobservable = \unobservableValue} & = (2\pi)^{-\nicefrac{1}{2}}\exp(-\unobservableValue^2/2)  \nonumber \\
+\prob{\decision = 0~|~\leniency_j = \leniencyValue, \obsFeatures = \obsFeaturesValue, \unobservable = \unobservableValue} & = \invlogit(\alpha_j + \gamma_\obsFeaturesValue\obsFeaturesValue + \gamma_\unobservableValue \unobservableValue + \epsilon_\decisionValue),  \label{eq:judgemodel} \\
+	\text{where}~ \alpha_{j} & \approx \logit(\leniencyValue_j) \label{eq:leniencymodel}\\
+\prob{\outcome=0~|~\decision, \obsFeatures=\obsFeaturesValue, \unobservable=\unobservableValue} & =
+	\begin{cases}
+		0,~\text{if}~\decision = 0\\
+		\invlogit(\alpha_\outcomeValue + \beta_\obsFeaturesValue \obsFeaturesValue + \beta_\unobservableValue \unobservableValue + \epsilon_\outcomeValue),~\text{o/w} \label{eq:defendantmodel}
+	\end{cases}
+\end{align}
+\hrulefill
+
+
+As stated in the equations above, we consider normalized features \obsFeatures and \unobservable.
+%
+Moreover, the probability that the decision maker makes a positive decision takes the form of a logistic function (Equation~\ref{eq:judgemodel}).
+% 
+Note that we are making the simplifying assumption that coefficients $\gamma$ are the same for all defendants, but decision makers are allowed to differ in intercept $\alpha_j \approx \logit(\leniencyValue_j)$ so as to model varying leniency levels among them (Eq. \ref{eq:leniencymodel}).
+%
+The probability that the outcome is successful conditional on a positive decision (Eq.~\ref{eq:defendantmodel}) is also provided by a logistic function, applied on the same features as the logistic formula of equation \ref{eq:judgemodel}.
+%
+In general, these two logistic functions may differ in their coefficients.
+%
+However, in many settings, a decision maker would be considered good if the two functions were the same -- i.e., if the probability to make a positive decision was the same as the probability to obtain a successful outcome after a positive decision.
+
+For the Bayesian modelling, the priors for the coefficients $\gamma_\obsFeaturesValue, ~\beta_\obsFeaturesValue, ~\gamma_\unobservableValue$ and $\beta_\unobservableValue$ were defined using the gamma-mixture representation of Student's t-distribution with $6$ degrees of freedom.
+%
+The gamma-mixture is obtained by first sampling a variance parameter from Gamma($\nicefrac{\nu}{2},~\nicefrac{\nu}{2}$) distribution and then drawing the  coefficient from zero-mean Gaussian distribution with variance equal to the inverse of the scale parameter.
+%
+Student's t-distribution was chosen for prior over the Gaussian for its better robustness against outliers \cite{ohagan1979outlier}.
+%
+The scale parameters $\eta_\unobservableValue, ~\eta_{\beta_\obsFeaturesValue}$ and $\eta_{\gamma_\obsFeaturesValue}$ were sampled independently from Gamma$(\nicefrac{6}{2},~\nicefrac{6}{2})$ and then the coefficients were sampled from Gaussian distribution with expectation $0$ and variance parameters $\eta_\unobservableValue^{-1}, ~\eta_{\beta_\obsFeaturesValue}^{-1}$ and $\eta_{\gamma_\obsFeaturesValue}^{-1}$ as shown below. The coefficients for the unobserved confounder \unobservable were bounded to the positive values to ensure identifiability.
+\begin{align}
+\eta_\unobservableValue, ~\eta_{\beta_\obsFeaturesValue}, ~\eta_{\gamma_\obsFeaturesValue} & \sim \text{Gamma}(3, 3) \\
+\gamma_\unobservableValue, ~\beta_\unobservableValue & \sim N_+(0, \eta_\unobservableValue^{-1}) \\
+\gamma_\obsFeaturesValue & \sim N(0, \eta_{\gamma_\obsFeaturesValue}^{-1}) \\
+\beta_\obsFeaturesValue & \sim N(0, \eta_{\beta_\obsFeaturesValue}^{-1})
+\end{align}
+
+The intercepts for the \judgeAmount decision-makers and outcome \outcome were defined to have hierarchical Gaussian priors with variances $\sigma_\decisionValue^2$ and $\sigma_\outcomeValue^2$ as shown below. Note that the decision-makers have a joint variance parameter $\sigma_\decisionValue^2$.
+\begin{align}
+\sigma_\decisionValue^2, ~\sigma_\outcomeValue^2 & \sim N_+(0, \tau^2) \\
+\alpha_j & \sim N(0, \sigma_\decisionValue^2) \\
+\alpha_\outcomeValue & \sim N(0, \sigma_\outcomeValue^2)
+\end{align}
+%
+The variance parameters $\sigma_\decisionValue^2$ and $\sigma_\outcomeValue^2$ were drawn independently from bounded zero-mean Gaussian distributions.
+% 
+The Gaussians were restricted to the positive real numbers and both had mean $0$ and variance $\tau^2=1$ -- other values were tested but observed to have no effect.
+
+
+
 \end{document}

paper/experiments.tex

@@ -30,7 +30,7 @@ We employed the following decision makers in our experiments:
 $y$ given features
 %$y \sim x$ or $y\sim x+z$ 
 and releases $r$ portion of the subjects.
-\item \textbf{Independent}: Each subject is released with respect to a cumulative distribution function based on the logistic regression model. \acomment{EXPLAIN BETTER. MAKE DETERMINISTIC?}
+\item \textbf{Independent}: Each subject is released with respect to a cumulative distribution function based on the logistic regression model. Given a subject with features \obsFeatures and \unobservable, we make decision \decision based on the value of expression $\invlogit(\gamma_\obsFeaturesValue\obsFeaturesValue + \gamma_\unobservableValue\unobservableValue)$ compared to the inverse cumulative distribution of a random variable $\invlogit(\gamma_\obsFeaturesValue\obsFeatures + \gamma_\unobservableValue\unobservable)$. If, for a decision-maker with leniency $\leniencyValue$, $\invlogit(\gamma_\obsFeaturesValue\obsFeaturesValue + \gamma_\unobservableValue\unobservableValue + \epsilon_\decisionValue) < F^{-1}(r)$ subject can be assigned with a positive decision. \acomment{EXPLAIN BETTER. MAKE DETERMINISTIC?}
 \item \textbf{Probabilistic}: Each subject is released with probability based on the logistic regression model, where the leniency is inputted through $\alpha_j$.
 \end{itemize}
 Decision makers in the data (\human) have access to \unobservable. Evaluated decision makers do not have access to \unobservable. All parameters of the models are for evaluated decision makers are learned from the training data set.