evaluation.tex

\subtitle{Evaluating Performance}
\date{Monday (AM), 14 May 2018}
\begin{document}
	
	\begin{frame}
	\titlepage
	\end{frame}

	% What do we want?
	% What do we measure?
	% How do we measure it?
	
	\begin{frame}{What is Player Experience?}
		\begin{block}{Player experience}
			Collection of events that occur to the player during the game
			\note{Should be clear - it is only the events that occur because of the game that are important}	
		\end{block}
	\end{frame}

	\begin{frame}{What is Player Experience?}
		\begin{block}{Scenario}
			Jeffrey is playing a game in his bedroom. The game is an online RTS game, and he is playing with a friend online against two other people.			
		\end{block}
		\begin{block}{Question}
		Which of these are a part of the player experience and which are not?\note{All happen while the person is playing a game}
		\end{block}
		\begin{itemize}
			\item<2->{Losing a unit} \uncover<7->{Yes}
			\item<3->{Laundry finishing} \uncover<8->{No}
			\item<4->{Collecting resource} \uncover<9->{Yes}
			\item<5->{New message in chat window} \uncover<10->{Yes}
			\item<6->{Unit moving} \uncover<11->{Yes}
		\end{itemize}
	\note{\\ Anything that occurs during the game and as part of the game is part of the player experience. Which of these can be detected by an AI?}
	\end{frame}
	
	\section{Metrics}
	
	\begin{frame}
		Collect data on how players/bots work
		
		\begin{block}{Activity}
			What kinds of features can we collect?
		\end{block}
	\end{frame}

	\begin{frame}{Data from humans}
		\begin{itemize}[<+->]
			\item{High-level human experience}
				\begin{itemize}
					\item Final game scores?
					\item How long did they play for?
				\end{itemize}
			\item{Biosignals}
				\begin{itemize}
					\item Where did they look?
					\item Galvanic skin response
					\item BCI
				\end{itemize}
			\item{Surveys and interviews}
				\begin{itemize}
					\item Likert Scales
					\item Why did you feel that way?
				\end{itemize}
		\end{itemize}
	\end{frame}

	\begin{frame}{Data from bots}
		\begin{itemize}[<+->]
			\item Internal State
				\begin{itemize}
					\item Will depend on bot architecture
					\item Measure state visits in FSM
					\item Did the game make full use of the AI?
				\end{itemize}
			\item How many times does a bot face a difficult choice?
				\begin{itemize}
					\item What is a difficult choice? \note{Difficult Choice: MCTS - near identical branches, GA - No Convergence}
				\end{itemize}
		\end{itemize}
	\end{frame}

	\begin{frame}{Data from either}
		\note{Some things can be measured regardless of if a human or AI is playing	\begin{itemize}[<+->]}
		\begin{itemize}[<+->]
			\item Final Score distribution\note{\item How high, variation?}
			\item Game Duration \note{\item Length, range of lengths}
			\item Score ``Drama'' \note{\item Runaway victory?, keep changing hands? loop?}
			\item Statistical distribution of states \note{\item Some states not used at all? Some overused?}
			\item Degree of challenge \note{\item How to measure this?}
		\end{itemize}
	\note{\end{itemize}}
	\end{frame}

	\begin{frame}{Data from populations}
	Variability of scores, skill-depth
	\end{frame}


	\section{Action Sequences}

	\begin{frame}{Data from either}
		Actions taken, Record the sequence of button-pushes
	\end{frame}

	\begin{frame}{Entropy}
	
	\end{frame}

	%% METRICS
	% Simon's raw vs computed metrics.
	
	%% SKILL
	% Evaluating skill depth
	
\end{document}