metrics.tex

\subtitle{Metrics}
\date{Tuesday, 22 May 2018}

\tikzset{
	treenode/.style = {align=center, inner sep=0pt, text centered,
		font=\sffamily},
	arn_n/.style = {treenode, circle, white, font=\sffamily\bfseries, draw=black,
		fill=black, text width=1.5em},% arbre rouge noir, noeud noir
	arn_r/.style = {treenode, circle, red, draw=red, 
		text width=1.5em, very thick},% arbre rouge noir, noeud rouge
	arn_x/.style = {treenode, rectangle, draw=black,
		minimum width=0.5em, minimum height=0.5em},% arbre rouge noir, nil	
	box/.style = {draw, align=center, text centered, rectangle, draw=black, minimum height=1.25cm, minimum width=2cm}
}

\DeclareMathOperator*{\argmax}{arg\,max}


\begin{document}
	
	\begin{frame}
	\titlepage
	\end{frame}

	\begin{frame}{Metrics}
		\begin{itemize}[<+->]
			\item We talked about this last week...
			\item Have a look at the evaluation slides
			\item We also did this in the asteroids code.
		\end{itemize}
	\end{frame}

\section{Evaluation}

\begin{frame}{Basic}
\begin{itemize}
	\item Look at AppMetrics
	\item Add your parameters using addParameter
	\item Write your fitness function
	\item get the best and print it out
\end{itemize}
\end{frame}

	\begin{frame}{What we need}
	\begin{itemize}[<+->]
		\item AI agents (We've done this yesterday)
		\item Some Parameters (EntityProp, EntityCost, etc...)
		\item An evaluation function
		\item Some maps? (One will do...)
	\end{itemize}
	\end{frame}

	\begin{frame}{AI Agents}
	\begin{itemize}[<+->]
		\item We did these yesterday.
		\item We should load the AIFactory
		\item Then use the factory to build agents in the evaluate
		\item ai.buildAI("ProRuleRushRangedBlue", settings),
	\end{itemize}
	\end{frame}

	\begin{frame}{Some Parameters}
	\begin{itemize}[<+->]
		\item evo.addParameter(new EntityProp("red\_knight", "defRanged", 0, 10, 1)); 
		\item A min, max and step size
		\item The entity and property
	\end{itemize}
	\end{frame}

	\begin{frame}{Evaluation Function}
	\begin{itemize}[<+->]
		\item Run the games
		\item Collect some \textbf{metrics}
		\item Report the game fitness
		\item (offline) analyse the metrics...
	\end{itemize}	
	\end{frame}

\begin{frame}[fragile]{Basic}
	\begin{minted}[breaklines,tabsize=2,fontsize=\footnotesize]{Java}
public Double evaluate(GameSettings settings) {
}
	\end{minted}
\end{frame}

\begin{frame}[fragile]{Less Basic}
	\begin{minted}[breaklines,tabsize=2,fontsize=\footnotesize]{Java}
public Double evaluate(GameSettings settings) {
	GameState start = map.buildState(settings);
	int[] winCounts = new int[2];
	for (int i=0; i<10; i++) {
		Controller[] controllers = new Controller[] {
			ai.buildAI("ProRuleRushRangedBlue", settings),
			ai.buildAI("ProRuleRushRed", settings)
		};
		GameState state = new GameState(start);
		GameMetrics metrics = runGame(state, settings, controllers);
		
		Integer winner = metrics.getWinner();
		if (winner != null) {
			winCounts[winner]++;
		}
	}
}
	\end{minted}
\end{frame}

\section{Stats}
\begin{frame}[fragile]{Stats}
	\begin{itemize}
		\item Stats about the games
		\item Stats about the turns
		\item You can write files per game played - see the example
		\item lots (and lots) of files...
	\end{itemize}
\end{frame}

\begin{frame}[fragile]{Graphs}
	\begin{figure}
	\includegraphics[width=\textwidth]{tbs-peices}
	\caption{What can we learn from this?}
	\end{figure}
\end{frame}

\begin{frame}{Results}
	\begin{table}
		\begin{tabular}{l|l|l}
		defRanged&atkRanged&fitness\\\hline
		1 & 5 & 30\\
		10 & 6 & 20\\
		4 & 2 & 20\\
		10 & 2 & 20
		\end{tabular}
	
		\pause So not a very good fitness function then...
	\end{table}
\end{frame}

\section{User Testing}

\begin{frame}{A|B Testing}
	\begin{itemize}[<+->]
		\item Put two versions of the game in front of someone
		\item Ask them what one they enjoyed most
		\item Ideally should use a large sample size (or stats)
		\item But this isn't meant to be rigorous
	\end{itemize}
\end{frame}

\begin{frame}{Qualitative Data}
	\begin{itemize}[<+->]
		\item In inform out conclusions we could ask other things
		\begin{itemize}
			\item Basic demographic data,
			\item Experience with games,
			\item Open ended questions,
			\item Game Metrics...
		\end{itemize}
	\end{itemize}
\end{frame}

\section{Advice}

\begin{frame}{AI experiments}
	\begin{itemize}[<+->]
		\item Fast iterations - don't spend too long waiting for stuff
		\item Drop turn limits, repeats, generation counts...
		\item See what kind of data you're getting and adapt your fitness.
	\end{itemize}
\end{frame}

\end{document}