sl.tex

\documentclass[sigconf,anonymous]{acmart}
% \documentclass[sigconf]{acmart}
%
% TO DO
% - Use 'case' instead of 'subject', unless we talk explicitly about people
% - Use 'recent' intead of 'state-of-the-art' to refer to lakkaraju.

% For camera-ready version: change these
\settopmatter{printacmref=false}
\settopmatter{printccs=false}
\setcopyright{none}

\sloppy

\usepackage{tikz}
\usepackage{tikz-cd}
\usetikzlibrary{shapes}
\usetikzlibrary{arrows,automata, positioning}

% Packages
\usepackage{type1cm}     % type1 computer modern font
\usepackage{graphicx}     % advanced figures
\usepackage{xspace}     % fix space in macros
\usepackage{balance}     % to better equalize the last page
\usepackage{multirow}     % multi rows for tables
\usepackage[font={bf}, tableposition=top]{caption}     % captions on top for tables
\usepackage{bold-extra}     % bold + {small capital, italic}
\usepackage{siunitx}          % \num for decimal grouping
\usepackage[vlined,linesnumbered,ruled,noend]{algorithm2e}     % algorithms
\usepackage{booktabs}     % nicer tables
%\usepackage[hyphens]{url}     % handle long urls
%\usepackage[bookmarks, pdftex, colorlinks=false]{hyperref}     % clickable references
%\usepackage[square,numbers]{natbib}     % better references
\usepackage{microtype}    % compress text
\usepackage{units}     % nicer slanted fractions
\usepackage{mathtools}     % amsmath++
%\usepackage{amssymb}     % math symbols
\usepackage{amsmath}
\usepackage{relsize}
\usepackage{caption}
\captionsetup{belowskip=6pt,aboveskip=2pt} % to save space.
%\usepackage{subcaption}
% \usepackage{multicolumn}
\usepackage[]{inputenc}
\usepackage{xfrac}
\RequirePackage{graphicx,color}
\usepackage[font={small}]{subfig} % subfig, 4 figures in a row
\usepackage{pifont}
\usepackage{footnote} % show footnotes in tables
\makesavenoteenv{table}



\newtheorem{problem}{Problem}

%\newcommand{\ourtitle}{Evaluating Decision Makers over Selectively Labeled Data}


%\newcommand{\ourtitle}{A Causal Approach to\\Evaluating Decision Makers over Selectively Labeled Data}

\newcommand{\ourtitle}{Evaluating Decision Makers over Selectively Labeled Data:\\
A Causal Modeling Approach
}

% A in the Presence of Unobservables and Selective Labels
% A Causal Treatment for Unobservables and Selective Labels
% Incomplete Data
%-unobservables
%-selective labels

%-causal
%-bayesian


\input{macros}
\usepackage{chato-notes}


\title{\ourtitle}

%\author{Michael Mathioudakis}
%\affiliation{%
 % \institution{University of Helsinki}
 % \city{Helsinki} 
 % \country{Finland} 
%}
%\email{michael.mathioudakis@helsinki.fi}


\begin{abstract}
Today, AI systems replace humans in an increasing number of decisions affecting people's lives.
%
Therefore, it is important to evaluate the performance of such systems {\it offline}, i.e., before they are deployed in real settings --
and compare it to the performance of human decisions they aim to replace.
%
The data which such evaluation is performed on has two major challenges, biasing any direct evaluations of considered decision makers.
%
First, in most cases the data does not include all factors that play a role in the decisions recorded in it.
%
Second, the past decision in the data may skew the data, and, in particular, any possible outcomes recorded in it.
%Another major challenge in such cases is that often past decisions have skewed the data on which the evaluation is performed. 
%
For example, when a bank decides whether a customer should be granted a loan, it is desired to grant loans to customers who would honor its conditions, but not to ones who would violate them.
%
However, we can directly evaluate only the decision to grant the loan, while we cannot observe whether customers who were not granted the loan would indeed violate its conditions. 
%
%THIS IS NOT SKEW THIS IS MISSING DATA
% 
Such bias appears in the decisions of both human and AI decision makers -- and should be properly taken into account for evaluation.
%
%Further complications arise since commonly not all features that the decisions are based on are observed. DISCUSS UNOBSERVABLES IN THE INTRO?
%
In this paper, we develop a Bayesian approach towards this end.
%
We use a detailed causal model of the decision making process, taking into account also unobserved features.
Based on this model, we evaluate counterfactual outcomes to correct any aforementioned biases. 
 % to infer unobserved outcomes.
%
Compared to previous methods for this setting, the approach estimates the quality of decisions more accurately and with lower variance. 
%
The approach is also %demonstrated to be 
robust to different variations in the decision mechanisms in the data.
%
\end{abstract}


\begin{document}


\fancyhead{}
\maketitle

\renewcommand{\shortauthors}{Authors}


\input{introduction}

\input{setting}

\input{imputation}

\input{experiments} 

\input{related} 

\input{conclusions}


% \textbf{Acknowledgments.}
%The computational resources must be mentioned. 

%\clearpage
% \balance
\bibliographystyle{ACM-Reference-Format}
\bibliography{biblio}
%\balancecolumns % GM June 2007

%\clearpage

\input{appendix}

\end{document}