1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
|
% Created 2023-10-23 Mon 20:13
% Intended LaTeX compiler: pdflatex
\documentclass[11pt]{article}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{longtable}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage[normalem]{ulem}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{capt-of}
\usepackage{hyperref}
\author{Niclas Dobbertin}
\date{\today}
\title{Analysis}
\hypersetup{
pdfauthor={Niclas Dobbertin},
pdftitle={Analysis},
pdfkeywords={},
pdfsubject={},
pdfcreator={Emacs 29.1 (Org mode 9.7)},
pdflang={English}}
\usepackage{biblatex}
\addbibresource{/home/niclas/bib/references.bib}
\begin{document}
\maketitle
\tableofcontents
\section{Imports}
\label{sec:orgf19bf7c}
\begin{verbatim}
import pandas as pd
import pickle
from pathlib import Path
\end{verbatim}
\section{Constants}
\label{sec:orgb587203}
\begin{verbatim}
data_path = Path("/home/niclas/repos/uni/master_thesis/experiment/data")
procedures = ["1", "2", "3", "4", "5", "6", "overall"]
\end{verbatim}
\section{Import Data}
\label{sec:org3427b7b}
\begin{verbatim}
def unpickle(pkl):
with open(pkl, "rb") as f:
data = pickle.load(f)
return data
\end{verbatim}
\subsection{Conditions}
\label{sec:org9e15909}
\begin{verbatim}
conditions = [x.stem for x in data_path.iterdir() if x.is_dir()]
conditions
\end{verbatim}
\begin{center}
\begin{tabular}{lll}
random & fixed & blocked\\[0pt]
\end{tabular}
\end{center}
\subsection{Data}
\label{sec:org65d4664}
\begin{verbatim}
data = {}
for condition in conditions:
data[condition] = {}
for vp in (data_path / condition).iterdir():
data[condition][vp.stem] = unpickle(vp / "vp.pkl")
\end{verbatim}
\begin{verbatim}
None
\end{verbatim}
\section{Basic statistics}
\label{sec:orgea2a5f1}
\subsection{Total percent correct}
\label{sec:org2eef721}
To find out how well VP solved the tasked, we calculate the accuracy for train
and test phase.
\begin{verbatim}
def percent_correct(vp):
train = [x for x in vp.keys() if "train" in x]
test = [x for x in vp.keys() if "test" in x]
train_total = len(train) * len(vp[train[0]]["procedure_order"])
test_total = len(test) * len(vp[test[0]]["procedure_order"])
train_correct = 0
test_correct = 0
def count_correct(trials):
trials_correct = 0
for sample in trials:
for proc in vp[sample]["procedure_order"]:
vp_ans = vp[sample][proc]["answer"]
for c in vp_ans:
if not c.isdigit():
vp_ans = vp_ans.replace(c, "")
vp_ans = int(vp_ans)
if vp_ans == vp[sample]["water_sample"][proc][0]:
trials_correct += 1
return trials_correct
return count_correct(train) / train_total, count_correct(test) / test_total
\end{verbatim}
\begin{verbatim}
condition = "random"
df = pd.DataFrame([percent_correct(data[condition][vp]) for vp in data[condition].keys()], columns=["train", "test"])
df
\end{verbatim}
\begin{verbatim}
train test
0 0.822222 0.820000
1 0.966667 0.800000
2 0.973333 0.980000
3 0.911111 0.960000
4 0.906667 0.980000
5 0.924444 0.943333
6 0.957778 0.926667
7 0.857778 0.946667
8 0.962222 0.970000
9 0.982222 0.986667
\end{verbatim}
Most subjects have an accuracy of over 95\% in both training and test phase.
Some however are notably lower, under 90\% in either training or test phase, or
both.
This could be a systematic misunderstanding of specific equations, that are
present in both, or only one of the two phases.
To investigate, we look at the per procedure accuracy per subject.
\begin{verbatim}
pass
\end{verbatim}
\begin{verbatim}
None
\end{verbatim}
\end{document}
|
