% Created 2023-10-23 Mon 20:13
% Intended LaTeX compiler: pdflatex
\documentclass[11pt]{article}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{longtable}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage[normalem]{ulem}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{capt-of}
\usepackage{hyperref}
\author{Niclas Dobbertin}
\date{\today}
\title{Analysis}
\hypersetup{
 pdfauthor={Niclas Dobbertin},
 pdftitle={Analysis},
 pdfkeywords={},
 pdfsubject={},
 pdfcreator={Emacs 29.1 (Org mode 9.7)}, 
 pdflang={English}}
\usepackage{biblatex}
\addbibresource{/home/niclas/bib/references.bib}
\begin{document}

\maketitle
\tableofcontents

\section{Imports}
\label{sec:orgf19bf7c}
\begin{verbatim}
import pandas as pd
import pickle
from pathlib import Path

\end{verbatim}
\section{Constants}
\label{sec:orgb587203}
\begin{verbatim}
data_path = Path("/home/niclas/repos/uni/master_thesis/experiment/data")

procedures = ["1", "2", "3", "4", "5", "6", "overall"]
\end{verbatim}
\section{Import Data}
\label{sec:org3427b7b}
\begin{verbatim}
def unpickle(pkl):
    with open(pkl, "rb") as f:
        data = pickle.load(f)
    return data
\end{verbatim}
\subsection{Conditions}
\label{sec:org9e15909}
\begin{verbatim}
conditions = [x.stem for x in data_path.iterdir() if x.is_dir()]
conditions
\end{verbatim}

\begin{center}
\begin{tabular}{lll}
random & fixed & blocked\\[0pt]
\end{tabular}
\end{center}
\subsection{Data}
\label{sec:org65d4664}
\begin{verbatim}
data = {}
for condition in conditions:
    data[condition] = {}
    for vp in (data_path / condition).iterdir():
        data[condition][vp.stem] = unpickle(vp / "vp.pkl")
\end{verbatim}

\begin{verbatim}
None
\end{verbatim}
\section{Basic statistics}
\label{sec:orgea2a5f1}
\subsection{Total percent correct}
\label{sec:org2eef721}
To find out how well VP solved the tasked, we calculate the accuracy for train
and test phase.

\begin{verbatim}
def percent_correct(vp):
    train = [x for x in vp.keys() if "train" in x]
    test = [x for x in vp.keys() if "test" in x]

    train_total = len(train) * len(vp[train[0]]["procedure_order"])
    test_total = len(test) * len(vp[test[0]]["procedure_order"])

    train_correct = 0
    test_correct = 0

    def count_correct(trials):
        trials_correct = 0
        for sample in trials:
            for proc in vp[sample]["procedure_order"]:
                vp_ans = vp[sample][proc]["answer"]
                for c in vp_ans:
                    if not c.isdigit():
                        vp_ans = vp_ans.replace(c, "")
                vp_ans = int(vp_ans)
                if vp_ans == vp[sample]["water_sample"][proc][0]:
                    trials_correct += 1
        return trials_correct

    return count_correct(train) / train_total, count_correct(test) / test_total
\end{verbatim}

\begin{verbatim}
condition = "random"
df = pd.DataFrame([percent_correct(data[condition][vp]) for vp in data[condition].keys()], columns=["train", "test"])
df
\end{verbatim}

\begin{verbatim}
      train      test
0  0.822222  0.820000
1  0.966667  0.800000
2  0.973333  0.980000
3  0.911111  0.960000
4  0.906667  0.980000
5  0.924444  0.943333
6  0.957778  0.926667
7  0.857778  0.946667
8  0.962222  0.970000
9  0.982222  0.986667
\end{verbatim}

Most subjects have an accuracy of over 95\% in both training and test phase.
Some however are notably lower, under 90\% in either training or test phase, or
both.
This could be a systematic misunderstanding of specific equations, that are
present in both, or only one of the two phases.
To investigate, we look at the per procedure accuracy per subject.

\begin{verbatim}
pass
\end{verbatim}

\begin{verbatim}
None
\end{verbatim}
\end{document}