\subtitle{Evaluating Performance}
\date{Monday (AM), 14 May 2018}
\begin{document}
\begin{frame}
\titlepage
\end{frame}
% What do we want?
% What do we measure?
% How do we measure it?
\begin{frame}{What is Player Experience?}
\begin{block}{Player experience}
Collection of events that \textbf{occur} to the player \textbf{during} the game
\note{Should be clear - it is only the events that occur because of the game that are important}
\end{block}
\end{frame}
\begin{frame}{What is Player Experience?}
\begin{block}{Scenario}
Jeffrey is playing an online RTS game, and he is playing with a friend online against two other people.
\end{block}
\begin{block}{Question}
Which of these are a part of the player experience and which are not?\note{All happen while the person is playing a game}
\end{block}
\begin{tabularx}{7.5cm}{X | l}
Losing a Unit & \uncover<2->{Yes} \\
Laundry Finishing & \uncover<3->{No} \\
Collecting resource & \uncover<4->{Yes} \\
New message in chat window & \uncover<5->{Yes} \\
Unit Moving & \uncover<6->{Yes}
\end{tabularx}
\note{\\ Anything that occurs during the game and as part of the game is part of the player experience. Which of these can be detected by an AI?}
\end{frame}
\section{Metrics}
\begin{frame}
Collect data on how players/bots work
\begin{block}{Activity}
What kinds of features can we collect?
\end{block}
\end{frame}
\begin{frame}{Data from humans}
\begin{itemize}[<+->]
\item{High-level human experience}
\begin{itemize}
\item Final game scores?
\item How long did they play for?
\end{itemize}
\item{Biosignals}
\begin{itemize}
\item Where did they look?
\item Galvanic skin response
\item BCI
\end{itemize}
\item{Surveys and interviews}
\begin{itemize}
\item Likert Scales
\item Why did you feel that way?
\end{itemize}
\end{itemize}
\end{frame}
\begin{frame}{Data from bots}
\begin{itemize}[<+->]
\item Internal State
\begin{itemize}
\item Will depend on bot architecture
\item Measure state visits in FSM
\item Did the game make full use of the AI?
\end{itemize}
\item How many times does a bot face a difficult choice?
\begin{itemize}
\item What is a difficult choice? \note{Difficult Choice: MCTS - near identical branches, GA - No Convergence}
\end{itemize}
\end{itemize}
\end{frame}
\begin{frame}{Data from either}
\note{Some things can be measured regardless of if a human or AI is playing \begin{itemize}}
\begin{itemize}[<+->]
\item Final Score distribution\note{\item How high, variation?}
\item Game Duration \note{\item Length, range of lengths}
\item Score ``Drama'' \note{\item Runaway victory?, keep changing hands? loop?}
\item Statistical distribution of states \note{\item Some states not used at all? Some overused?}
\item Degree of challenge \note{\item How to measure this?}
\end{itemize}
\note{\end{itemize}}
\end{frame}
\begin{frame}{Data from populations}
\begin{itemize}
\item Variability of scores
\item Skill-depth
\end{itemize}
\end{frame}
\section{Action Sequences}
\begin{frame}{Data from either}
\begin{itemize}
\item Actions taken
\item Record the sequence of button-pushes
\end{itemize}
\end{frame}
\begin{frame}{Entropy}
\note{\begin{itemize}}
\begin{itemize}[<+->]
\item Sometimes used to \textbf{interpret} aspects of player experience
\begin{itemize}
\item $H(X) = \sum_{i=1}^{n} P(x_{i})I(x_{i}) = -\sum_{i=1}^{n}P(x_{i})\log_{b}P(x_{i})$ \note{\item We won't worry too much about the middle definition}
\item Take a fair coin - how much entropy?
\item $H(fairCoint) = \sum_{i=1}^{2}(\frac{1}{2})\log_{2}(\frac{1}{2}) = -\sum_{i=1}^{2}(\frac{1}{2}) \times (-1) = 1 $ \note{\item Because it is a fair coin - each toss can tell us nothing}
\item How about an unfair coin? What is the entropy for a coin of probability 0.9?
\note{\item Whiteboard time if students stuck: \begin{itemize}}
\note{\item Answer is: $ H(dodgyCoin) = \sum_{i=1}^{2}(0.9)\log_{2}(0.9) = $}
\note{\item Continued: $ -\Big( (0.9 \log_{2}0.9) + (0.1 \log_{2}0.1) \Big) = 0.47 $}
\note{\end{itemize}}
\end{itemize}
\end{itemize}
\begin{center}
\uncover<6->{\includegraphics[scale=0.4]{entropy}\footnote<6->{Borrowed from \href{https://en.wikipedia.org/wiki/Entropy_(information_theory)}{wikipedia}}}
\end{center}
\note{\end{itemize}}
\end{frame}
\begin{frame}{A Game Example}
%\setbeamercovered{invisible}
\note{\begin{itemize}}
\begin{columns}
\note{\item Some sample 2D location visit counts}
\begin{column}{0.3\textwidth}
\begin{tabularx}{\linewidth}{l | l | l | l}
loc & 0 & 1 & 2 \\
\hline
0 & 10 & 20 & 15 \\
1 & 12 & 35 & 13 \\
2 & 15 & 20 & 10 \\
\end{tabularx}
\end{column}
\note{\item Converted into visit counts as fraction of total and then into probability of having visited that location}
\begin{column}{0.7\textwidth}
\begin{tabularx}{\linewidth}{l | l | l | l}
loc & visits & p(loc) & calc \\
\hline
0,0 & 10 & 0.067 & $0.067\log_{2}(0.067)$\\
\uncover<2>{
0,1 & 12 & 0.08 & $0.080\log_{2}(0.008)$ \\
0,2 & 15 & 0.1 & $0.100\log_{2}(0.100)$ \\
1,0 & 20 & 0.134 & $0.134\log_{2}(0.134)$ \\
1,1 & 35 & 0.234 & $0.234\log_{2}(0.234)$ \\
1,2 & 20 & 0.134 & $0.134\log_{2}(0.134)$ \\
2,0 & 15 & 0.1 & $0.100\log_{2}(0.100)$ \\
2,1 & 13 & 0.0867 & $0.0867\log_{2}(0.0867)$ \\
2,2 & 10 & 0.067 & $0.067\log_{2}(0.067)$ \\
\hline}
& 150 & Total: & \\
\end{tabularx}
\end{column}
\note{\item Then we just perform the math as a giant summation. Computers are good at this}
\note{\item Except computers are not keen on 0's}
\end{columns}
\note{\end{itemize}}
\end{frame}
%% METRICS
% Simon's raw vs computed metrics.
%% SKILL
% Evaluating skill depth
\begin{frame}{Skill Ratings}
\note{\begin{itemize}}
\begin{itemize}[<+->]
\item How \textbf{good} is a player? \note{\item And how do we represent this?}
\item What is the \textbf{issue} with win rates? \note{\item Based on observations, was it enough? Watch F1 at one track and use those observations for another?}
\item If A $>$ B and B $>$ C is A $>$C? \note{\item Usually this is the case in games}
\note{\item Does close win rates prove a lack of skill depth? No, current set of players doesn't demonstrate it. Like me and Joe playing Pool}
\end{itemize}
\note{\end{itemize}}
\end{frame}
\begin{frame}{Elo Ratings}
\note{\begin{itemize}}
\begin{itemize}[<+->]
\item Elo is based on probability \note{\item Designed for chess}
\item $Elo(A) - Elo(B) = P(A$ beats $B)$ \note{\item Point difference between players denotes the probability of winning}
\note{\item Advantage of 100 points = 64\% chance of winning
\item Advantage of 200 points = 76\% chance of winning
\item Works by taking points from the loser and giving them to the winner. Number transfered proportional to difference between points
}
\end{itemize}
\begin{center}
\only<3>{\includegraphics[scale=0.5]{elo}\footnote{Borrowed from \href{http://liquipedia.net/starcraft/Elo_rating}{liquipedia}}}
\end{center}
\note{\end{itemize}}
\end{frame}
\end{document}