Merge branch 'master' into 1920x1080

8 files changed, 893 insertions, 0 deletions

diff --git a/experiment/EK_Muster-Einverstaendniserklaerung_2022.odt b/experiment/EK_Muster-Einverstaendniserklaerung_2022.odt
new file mode 100644
index 0000000..8634478
--- /dev/null
+++ b/experiment/EK_Muster-Einverstaendniserklaerung_2022.odt
diff --git a/experiment/EK_Muster-Einverstaendniserklaerung_2022.pdf b/experiment/EK_Muster-Einverstaendniserklaerung_2022.pdf
new file mode 100644
index 0000000..c21c686
--- /dev/null
+++ b/experiment/EK_Muster-Einverstaendniserklaerung_2022.pdf
diff --git a/experiment/analysis/analysis.org b/experiment/analysis/analysis.org
new file mode 100644
index 0000000..e726046
--- /dev/null
+++ b/experiment/analysis/analysis.org
@@ -0,0 +1,103 @@
+#+title: Analysis
+#+PROPERTY: header-args:python+ :session *python* :exports both :tangle yes
+
+* Imports
+#+begin_src python :results none
+import pandas as pd
+from pathlib import Path
+from pprint import pprint
+
+import tools
+
+#+end_src
+
+* Constants
+#+begin_src python :results none
+data_path = Path("/home/niclas/repos/uni/master_thesis/experiment/data")
+
+procedures = ["1", "2", "3", "4", "5", "6", "overall"]
+#+end_src
+
+* Import Data
+** Conditions
+#+begin_src python
+conditions = [x.stem for x in data_path.iterdir() if x.is_dir()]
+conditions
+#+end_src
+
+#+RESULTS:
+| random | fixed | blocked |
+
+** Data
+#+begin_src python :results none
+data = {}
+for condition in conditions:
+    data[condition] = {}
+    for vp in (data_path / condition).iterdir():
+        data[condition][vp.stem] = tools.unpickle(vp / "vp.pkl")
+
+data_train, data_test = tools.train_test_split(data)
+#+end_src
+
+* Basic statistics
+** Total percent correct
+To find out how well VP solved the tasked, we calculate the accuracy for train
+and test phase.
+
+#+begin_src python
+condition = "random"
+df = pd.DataFrame([tools.total_accuracy(data[condition][vp], procedures) for vp in data[condition].keys()], index=data[condition].keys(), columns=["train", "test"])
+df
+#+end_src
+
+#+RESULTS:
+#+begin_example
+         train      test
+vp12  0.822222  0.820000
+vp19  0.966667  0.800000
+vp15  0.973333  0.980000
+vp17  0.911111  0.960000
+vp20  0.906667  0.980000
+vp10  0.924444  0.943333
+vp16  0.957778  0.926667
+vp13  0.857778  0.946667
+vp18  0.962222  0.970000
+vp14  0.982222  0.986667
+#+end_example
+
+Most subjects have an accuracy of over 95% in both training and test phase.
+Some however are notably lower, under 90% in either training or test phase, or
+both.
+This could be a systematic misunderstanding of specific equations, that are
+present in both, or only one of the two phases.
+To investigate, we look at the per procedure accuracy per subject.
+
+#+begin_src python
+condition = "random"
+proc_accs = [
+    tools.count_correct(data[condition][vp], data[condition][vp].keys(), procedures)
+    for vp in data[condition].keys()
+]
+for vp in proc_accs:
+    for proc in vp.keys():
+        vp[proc] /= len(next(iter(data[condition].values())).keys())
+df = pd.DataFrame(proc_accs, index=data[condition].keys())
+df
+#+end_src
+
+#+RESULTS:
+#+begin_example
+          1      2      3      4      5      6  overall
+vp12  0.992  0.592  0.392  0.976  0.960  1.000    0.016
+vp19  1.000  0.992  0.000  0.576  0.992  0.992    0.848
+vp15  0.992  0.992  0.960  0.392  0.592  1.000    0.928
+vp17  0.392  0.968  0.584  1.000  1.000  0.992    0.648
+vp20  0.992  0.376  0.952  0.976  0.976  0.560    0.784
+vp10  0.968  0.360  0.592  0.984  0.984  0.992    0.712
+vp16  0.976  0.600  0.376  0.976  0.992  1.000    0.752
+vp13  0.384  0.960  0.928  0.560  0.992  0.968    0.568
+vp18  0.976  0.976  0.960  0.392  0.600  0.984    0.904
+vp14  0.992  0.976  0.992  0.976  0.400  0.600    0.968
+#+end_example
+
+We can see that most vp have around 2 procedures with accuracy of around 50%
diff --git a/experiment/analysis/analysis.pdf b/experiment/analysis/analysis.pdf
new file mode 100644
index 0000000..751625d
--- /dev/null
+++ b/experiment/analysis/analysis.pdf
diff --git a/experiment/analysis/analysis.tex b/experiment/analysis/analysis.tex
new file mode 100644
index 0000000..5b56bc8
--- /dev/null
+++ b/experiment/analysis/analysis.tex
@@ -0,0 +1,150 @@
+% Created 2023-10-28 Sat 19:43
+% Intended LaTeX compiler: pdflatex
+\documentclass[11pt]{article}
+\usepackage[utf8]{inputenc}
+\usepackage[T1]{fontenc}
+\usepackage{graphicx}
+\usepackage{longtable}
+\usepackage{wrapfig}
+\usepackage{rotating}
+\usepackage[normalem]{ulem}
+\usepackage{amsmath}
+\usepackage{amssymb}
+\usepackage{capt-of}
+\usepackage{hyperref}
+\author{Niclas Dobbertin}
+\date{\today}
+\title{Analysis}
+\hypersetup{
+ pdfauthor={Niclas Dobbertin},
+ pdftitle={Analysis},
+ pdfkeywords={},
+ pdfsubject={},
+ pdfcreator={Emacs 29.1 (Org mode 9.7)}, 
+ pdflang={English}}
+\usepackage{biblatex}
+\addbibresource{/home/niclas/bib/references.bib}
+\begin{document}
+
+\maketitle
+\tableofcontents
+
+\section{Imports}
+\label{sec:orgbdc2c77}
+\begin{verbatim}
+import pandas as pd
+from pathlib import Path
+from pprint import pprint
+
+import tools
+
+\end{verbatim}
+\section{Constants}
+\label{sec:orgcb8c537}
+\begin{verbatim}
+data_path = Path("/home/niclas/repos/uni/master_thesis/experiment/data")
+
+procedures = ["1", "2", "3", "4", "5", "6", "overall"]
+\end{verbatim}
+\section{Import Data}
+\label{sec:org87e67b0}
+\subsection{Conditions}
+\label{sec:orga12f2b6}
+\begin{verbatim}
+conditions = [x.stem for x in data_path.iterdir() if x.is_dir()]
+conditions
+\end{verbatim}
+
+\begin{center}
+\begin{tabular}{lll}
+random & fixed & blocked\\[0pt]
+\end{tabular}
+\end{center}
+\subsection{Data}
+\label{sec:orgcac95cb}
+\begin{verbatim}
+data = {}
+for condition in conditions:
+    data[condition] = {}
+    for vp in (data_path / condition).iterdir():
+        data[condition][vp.stem] = tools.unpickle(vp / "vp.pkl")
+\end{verbatim}
+
+\begin{verbatim}
+None
+\end{verbatim}
+\subsection{Useful Subdata}
+\label{sec:org4384120}
+\begin{verbatim}
+# data_correct = {conditons[0]: {}, conditons[1]: {}, conditons[2]: {}}
+pass
+# for condition in conditions:
+#     data_correct[condition] = None
+\end{verbatim}
+
+\begin{verbatim}
+None
+\end{verbatim}
+\section{Basic statistics}
+\label{sec:org44d0851}
+\subsection{Total percent correct}
+\label{sec:org461b551}
+To find out how well VP solved the tasked, we calculate the accuracy for train
+and test phase.
+
+\begin{verbatim}
+condition = "random"
+df = pd.DataFrame([tools.total_accuracy(data[condition][vp], procedures) for vp in data[condition].keys()], index=data[condition].keys(), columns=["train", "test"])
+df
+\end{verbatim}
+
+\begin{verbatim}
+         train      test
+vp12  0.822222  0.820000
+vp19  0.966667  0.800000
+vp15  0.973333  0.980000
+vp17  0.911111  0.960000
+vp20  0.906667  0.980000
+vp10  0.924444  0.943333
+vp16  0.957778  0.926667
+vp13  0.857778  0.946667
+vp18  0.962222  0.970000
+vp14  0.982222  0.986667
+\end{verbatim}
+
+Most subjects have an accuracy of over 95\% in both training and test phase.
+Some however are notably lower, under 90\% in either training or test phase, or
+both.
+This could be a systematic misunderstanding of specific equations, that are
+present in both, or only one of the two phases.
+To investigate, we look at the per procedure accuracy per subject.
+
+\begin{verbatim}
+condition = "random"
+proc_accs = [
+    tools.count_correct(data[condition][vp], data[condition][vp].keys(), procedures)
+    for vp in data[condition].keys()
+]
+for vp in proc_accs:
+    for proc in vp.keys():
+        vp[proc] /= len(next(iter(data[condition].values())).keys())
+df = pd.DataFrame(proc_accs, index=data[condition].keys())
+df
+\end{verbatim}
+
+\begin{verbatim}
+          1      2      3      4      5      6  overall
+vp12  0.992  0.592  0.392  0.976  0.960  1.000    0.016
+vp19  1.000  0.992  0.000  0.576  0.992  0.992    0.848
+vp15  0.992  0.992  0.960  0.392  0.592  1.000    0.928
+vp17  0.392  0.968  0.584  1.000  1.000  0.992    0.648
+vp20  0.992  0.376  0.952  0.976  0.976  0.560    0.784
+vp10  0.968  0.360  0.592  0.984  0.984  0.992    0.712
+vp16  0.976  0.600  0.376  0.976  0.992  1.000    0.752
+vp13  0.384  0.960  0.928  0.560  0.992  0.968    0.568
+vp18  0.976  0.976  0.960  0.392  0.600  0.984    0.904
+vp14  0.992  0.976  0.992  0.976  0.400  0.600    0.968
+\end{verbatim}
+
+We can see that most vp have around 2 procedures with accuracy of around 50\%
+\end{document}
\ No newline at end of file
diff --git a/experiment/analysis/tools.py b/experiment/analysis/tools.py
new file mode 100644
index 0000000..d32ccd3
--- /dev/null
+++ b/experiment/analysis/tools.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+
+import pickle
+from copy import deepcopy
+
+def unpickle(pkl):
+    with open(pkl, "rb") as f:
+        data = pickle.load(f)
+    return data
+
+
+def count_correct(vp, trials, procedures):
+    trials_correct = {}
+    for proc in procedures:
+        trials_correct[proc] = 0
+    for sample in trials:
+        for proc in vp[sample]["procedure_order"]:
+            vp_ans = vp[sample][proc]["answer"]
+            for c in vp_ans:
+                if not c.isdigit():
+                    vp_ans = vp_ans.replace(c, "")
+            vp_ans = int(vp_ans)
+            if vp_ans == vp[sample]["water_sample"][proc][0]:
+                trials_correct[proc] += 1
+    return trials_correct
+
+
+def total_accuracy(vp, procedures):
+    train = [x for x in vp.keys() if "train" in x]
+    test = [x for x in vp.keys() if "test" in x]
+
+    train_total = len(train) * len(vp[train[0]]["procedure_order"])
+    test_total = len(test) * len(vp[test[0]]["procedure_order"])
+
+    acc_train = count_correct(vp, train, procedures)
+    acc_test = count_correct(vp, test, procedures)
+
+    acc_train = sum([acc_train[x] for x in acc_train.keys()]) / train_total
+    acc_test = sum([acc_test[x] for x in acc_test.keys()]) / test_total
+
+    return acc_train, acc_test
+
+
+def train_test_split(data):
+    def delete_trials(data, string):
+        new_dict = {}
+        for cond in data.keys():
+            new_dict[cond] = {}
+            for vp in data[cond].keys():
+                new_dict[cond][vp] = {}
+                for trial in data[cond][vp].keys():
+                    if string in trial:
+                        new_dict[cond][vp][trial] = data[cond][vp][trial]
+        return new_dict
+    data_train = delete_trials(data, "train")
+    data_test = delete_trials(data, "test")
+
+    return data_train, data_test
+
+print("imported tools")
diff --git a/experiment/frensch_procedures.py b/experiment/frensch_procedures.py
new file mode 100644
index 0000000..3c64471
--- /dev/null
+++ b/experiment/frensch_procedures.py
@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+import random
+
+
+class WaterSample:
+    def __init__(
+        self,
+        solid: int,
+        algae: int,
+        lime: tuple[int, int, int, int],
+        toxin: tuple[int, int, int, int],
+    ):
+        self.solid = solid
+        self.algae = algae
+        self.lime = lime
+        self.toxin = toxin
+
+    def procedure_dict(self):
+        procedures = {
+            "1": (self.index1(), self.index1_str()),
+            "2": (self.index2(), self.index2_str()),
+            "3": (self.index3(), self.index3_str()),
+            "4": (self.index4(), self.index4_str()),
+            "5": (self.index5(), self.index5_str()),
+            "6": (self.index6(), self.index6_str()),
+            "overall": (self.overall(), self.overall_str()),
+        }
+
+        return procedures
+
+    def water_sample_dict(self):
+        sample = {
+            "1": (self.index1(), self.index1_str()),
+            "2": (self.index2(), self.index2_str()),
+            "3": (self.index3(), self.index3_str()),
+            "4": (self.index4(), self.index4_str()),
+            "5": (self.index5(), self.index5_str()),
+            "6": (self.index6(), self.index6_str()),
+            "overall": (self.overall(), self.overall_str()),
+            "solid": self.solid,
+            "algae": self.algae,
+            "lime": self.lime,
+            "toxin": self.toxin,
+        }
+        return sample
+
+    def select_procedures(self):
+        procedures = [
+            (self.index1, self.index1_str),
+            (self.index2, self.index2_str),
+            (self.index3, self.index3_str),
+            (self.index4, self.index4_str),
+            (self.index5, self.index5_str),
+            (self.index6, self.index6_str),
+        ]
+        random.shuffle(procedures)
+        training_procedures = procedures[:-1]
+        training_procedures.append((self.overall, self.overall_str))
+
+        return training_procedures, procedures[-1]
+
+    def index1(self):
+        return (self.lime[3] - self.lime[1]) * self.solid
+
+    def index1_str(self):
+        return "(Sandstein_4 - Sandstein_2) * Mineralien"
+
+    def index2(self):
+        return (2 * self.algae) + min(self.lime)
+
+    def index2_str(self):
+        return "(2 * Algen) + Sandstein_min"
+
+    def index3(self):
+        return max(self.toxin) + min(self.toxin)
+
+    def index3_str(self):
+        return "Gifte_max + Gifte_min"
+
+    def index4(self):
+        return (self.solid * 2) - self.toxin[3]
+
+    def index4_str(self):
+        return "(Mineralien * 2) - Gifte_4"
+
+    def index5(self):
+        return max(self.lime[2], (self.toxin[2] - self.toxin[1]))
+
+    def index5_str(self):
+        return "Das Höhere von (Gifte_3 - Gifte_2), (Sandstein_3)"
+
+    def index6(self):
+        return min(self.algae, (self.lime[0] + self.toxin[0]))
+
+    def index6_str(self):
+        return "Das Kleinere von (Sandstein_1 + Gifte_1), (Algen)"
+
+    def overall(self):
+        return 100 - max(
+            self.index1(), self.index2(), self.index3(), self.index4(), self.index5()
+        )
+
+    def overall_str(self):
+        return "100 - dem Höchstem aller Ergebnisse"
+
+    def print_all(self):
+        print(f"Solid: {self.solid}")
+        print(f"Algae: {self.algae}")
+        print(f"Lime: {self.lime}")
+        print(f"Toxin: {self.toxin}")
+        print(f"Index 1: {self.index1_str()} = {self.index1()}")
+        print(f"Index 2: {self.index2_str()} = {self.index2()}")
+        print(f"Index 3: {self.index3_str()} = {self.index3()}")
+        print(f"Index 4: {self.index4_str()} = {self.index4()}")
+        print(f"Index 5: {self.index5_str()} = {self.index5()}")
+        print(f"Index 6: {self.index6_str()} = {self.index6()}")
+        print(f"Overall Quality: {self.overall_str()} = {self.overall()}")
+
+
+# No step should produce a negative number, greater/lesser of comparisons should not
+# use equal numbers
+def constrained_WaterSample():
+    water_sample = random_WaterSample()
+    resample = True
+    while resample:
+        water_sample = random_WaterSample()
+        resample = False
+        # check for negative results
+        for proc in water_sample.procedure_dict().keys():
+            if water_sample.procedure_dict()[proc][0] < 0:
+                resample = True
+        # check for greater/lesser equality
+        # procedure 5
+        if (water_sample.toxin[2] - water_sample.toxin[1]) == water_sample.lime[2]:
+            resample = True
+        # procedure 6
+        if (water_sample.lime[0] + water_sample.toxin[0]) == water_sample.algae:
+            resample = True
+    return water_sample
+
+
+def random_WaterSample():
+    solid = random.randint(1, 9)
+    algae = random.randint(1, 9)
+    lime = tuple(random.randint(1, 9) for _ in range(4))
+    toxin = tuple(random.randint(1, 9) for _ in range(4))
+
+    return WaterSample(solid, algae, lime, toxin)
diff --git a/experiment/frensch_task.py b/experiment/frensch_task.py
new file mode 100644
index 0000000..1949d51
--- /dev/null
+++ b/experiment/frensch_task.py
@@ -0,0 +1,430 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import pickle
+import random
+from collections import namedtuple
+from pprint import pprint
+
+import pandas as pd
+from psychopy import core, event, gui, visual
+
+import frensch_procedures
+
+DisplayVariable = namedtuple("DisplayVariable", ["name", "values"])
+DisplayProcedure = namedtuple("DisplayProcedure", ["procedure", "solution"])
+
+intro_text = """Vielen Dank dass Sie bei unserem Experiment zum menschlichen Lernen teilnehmen!
+
+In diesem Experiment arbeiten Sie in einem Labor, welches die Wasserqualität analysiert.
+Dafür bekommen Sie einige Wasserproben und müssen für jede Probe verschiedene Kennwerte ermitteln.
+
+Jede Wasserprobe besitzt bereits verschiedene gemessene Werte, wie der Algengehalt, welche für die Berechnungen benutzt werden.
+
+
+(Leertaste zum Fortfahren)
+"""
+
+intro2_text = """Im folgenden müssen sie verschiedene Rechenaufgaben lösen um die Kennwerte zu berechnen.
+Verwenden Sie die gewohnten Rechenregeln und geben sie Ihre Lösung bitte immer als zweistellige Zahl ein, und bestätigen mit Enter.
+Die Werte der Variablen (z.B. Algen) werden oben am Bildschirm angezeigt.
+
+Manche Variablen haben mehrere mögliche Werte; "Gifte_2" besagt z.B., dass der zweite Wert der Giftwerte zu verwenden ist.
+"_max/_min" besagt, dass der maximale/minimale Wert dieser Variable zu verwenden ist.
+
+Als letzte Berechnung müssen Sie den Gesamtwert der Wasserqualität aus ihren Ergebnissen berechnen.
+
+Nach jeder Aufgabe können Sie kurz pausieren.
+
+Drücken Sie die Leertaste um zu beginnen"""
+
+
+def experiment_shutdown():
+    WIN.close()
+    core.quit()
+
+
+WIN = visual.Window((1920, 1080), fullscr=True, units="pix")
+MONITOR_FPS = 60
+TRAIN_TRIALS = 75
+TEST_TRIALS = 50
+# TRAIN_TRIALS = 3
+# TEST_TRIALS = 3
+ORDER_CONDITIONS = ["fixed", "random", "blocked"]
+PROCEDURE_KEYS = ["1", "2", "3", "4", "5", "6", "overall"]
+
+# Cancel experiment anytime with Esc
+event.globalKeys.add(key="escape", func=experiment_shutdown, name="shutdown")
+
+
+def generate_variable_display(varx: list[DisplayVariable], x_positions: list[int]):
+    assert len(varx) == len(x_positions)
+
+    stims = []
+
+    def gen_value_stims(values, x, y, offset):
+        for value in values:
+            y -= offset
+            value_stim = visual.TextBox2(
+                WIN,
+                pos=(x, y),
+                text=value,
+                # size=200,
+                letterHeight=75,
+                alignment="center",
+            )
+            stims.append(value_stim)
+
+    y = 550
+    offset = 100
+
+    for var, x_pos in zip(varx, x_positions):
+        stim_var = visual.TextBox2(
+            WIN,
+            pos=[x_pos, y],
+            text=var.name,
+            # size=[1000, 1000],
+            letterHeight=40,
+            alignment="center",
+        )
+        stims.append(stim_var)
+
+        gen_value_stims(var.values, x_pos, y, offset)
+
+    return stims
+
+
+def generate_procedure_display(procedure: DisplayProcedure, position):
+    stim_procedure = visual.TextBox2(
+        WIN,
+        pos=position,
+        text=procedure.procedure,
+        size=[1000000, 1000],
+        letterHeight=40,
+        alignment="center",
+    )
+    return stim_procedure
+
+
+def generate_all_watersamples(n):
+    samples = []
+    for _ in range(n):
+        samples.append(frensch_procedures.constrained_WaterSample())
+    return samples
+
+
+def run_blocked_trials(water_samples, procedure_keys):
+    results = {}
+    for proc_idx, proc in enumerate(procedure_keys):
+        for sample_idx, sample in enumerate(water_samples):
+
+            if sample_idx % 6 == 0:
+                pause_after_trial.draw()
+                WIN.flip()
+                event.waitKeys(keyList=["space"])
+
+            cur_key = f"train_{sample_idx}"
+            if not cur_key in results.keys():
+                results[cur_key] = {}
+            results[cur_key]["procedure_order"] = tuple(procedure_keys)
+            results[cur_key]["water_sample"] = sample.water_sample_dict()
+
+            solid = DisplayVariable("Mineralien", [sample.solid])
+            algae = DisplayVariable("Algen", [sample.algae])
+            lime = DisplayVariable("Sandstein", sample.lime)
+            toxin = DisplayVariable("Gifte", sample.toxin)
+            x_positions = [-800, -400, 400, 800]
+
+            stims = generate_variable_display([solid, algae, lime, toxin], x_positions)
+
+            procedures = sample.procedure_dict()
+
+            proc_x = -600
+            proc_y = -100
+            answ_x = 200
+            answ_y = -100
+            y_offset = 80
+
+            for prev in procedure_keys[:proc_idx]:
+                if not prev:
+                    continue
+                p = DisplayProcedure(procedures[prev][1], procedures[prev][0])
+                p = generate_procedure_display(p, (proc_x, proc_y))
+                stims.append(p)
+                proc_y -= y_offset
+
+                stim_answer_equals = visual.TextBox2(
+                    WIN,
+                    "=",
+                    letterHeight=50,
+                    pos=(answ_x - 100, answ_y),
+                    size=[150, 70],
+                    alignment="center",
+                )
+                stims.append(stim_answer_equals)
+
+                stim_answer_box = visual.TextBox2(
+                    WIN,
+                    results[f"train_{sample_idx}"][prev]["answer"],
+                    letterHeight=50,
+                    pos=(answ_x, answ_y),
+                    size=[150, 70],
+                    editable=True,
+                    fillColor="white",
+                    color="black",
+                    alignment="center",
+                )
+                stims.append(stim_answer_box)
+                answ_y -= y_offset
+
+            p = DisplayProcedure(procedures[proc][1], procedures[proc][0])
+            p = generate_procedure_display(p, (proc_x, proc_y))
+            stims.append(p)
+            proc_y -= y_offset
+
+            stim_answer_equals = visual.TextBox2(
+                WIN,
+                "=",
+                letterHeight=50,
+                pos=(answ_x - 100, answ_y),
+                size=[150, 70],
+                alignment="center",
+            )
+            stims.append(stim_answer_equals)
+
+            print(sample_idx)
+            stim_answer_box = visual.TextBox2(
+                WIN,
+                "",
+                letterHeight=50,
+                pos=(answ_x, answ_y),
+                size=[150, 70],
+                editable=True,
+                fillColor="white",
+                color="black",
+                alignment="center",
+            )
+            stims.append(stim_answer_box)
+            answ_y -= y_offset
+
+            not_finished = True
+            answer = "not answered"
+            start_time = core.monotonicClock.getTime()
+            while not_finished:
+                stim_answer_box.hasFocus = True
+                for stim in stims:
+                    stim.draw()
+                WIN.flip()
+                answer = stim_answer_box.text
+                if "\n" in answer:
+                    if answer[0].isdigit() and answer[1].isdigit():
+                        not_finished = False
+                    else:
+                        stim_answer_box.text = answer[:-1]
+                if len(answer) > 2:
+                    stim_answer_box.text = stim_answer_box.text[:2]
+            answer_time = core.monotonicClock.getTime() - start_time
+            answer = (answer.replace("\n", ""), answer_time)
+            results[cur_key][proc] = {"answer": answer[0], "time": answer[1]}
+
+        pause_new_proc.draw()
+        WIN.flip()
+        event.waitKeys(keyList=["space"])
+
+    return results
+
+
+def run_trial(water_sample, procedure_keys: list, condition):
+    water_sample.print_all()
+
+    if condition == "random":
+        overall = procedure_keys.pop()
+        random.shuffle(procedure_keys)
+        procedure_keys.append(overall)
+
+    solid = DisplayVariable("Mineralien", [water_sample.solid])
+    algae = DisplayVariable("Algen", [water_sample.algae])
+    lime = DisplayVariable("Sandstein", water_sample.lime)
+    toxin = DisplayVariable("Gifte", water_sample.toxin)
+    x_positions = [-800, -400, 400, 800]
+
+    stims = generate_variable_display([solid, algae, lime, toxin], x_positions)
+
+    procedures = water_sample.procedure_dict()
+
+    answers = []
+    proc_x = -600
+    proc_y = -100
+    answ_x = 200
+    answ_y = -100
+    y_offset = 80
+    for proc in procedure_keys:
+        p = DisplayProcedure(procedures[proc][1], procedures[proc][0])
+        p = generate_procedure_display(p, (proc_x, proc_y))
+        stims.append(p)
+        proc_y -= y_offset
+
+        stim_answer_equals = visual.TextBox2(
+            WIN,
+            "=",
+            letterHeight=50,
+            pos=(answ_x - 100, answ_y),
+            size=[150, 70],
+            alignment="center",
+        )
+        stims.append(stim_answer_equals)
+
+        stim_answer_box = visual.TextBox2(
+            WIN,
+            "",
+            letterHeight=50,
+            pos=(answ_x, answ_y),
+            size=[150, 70],
+            editable=True,
+            fillColor="white",
+            color="black",
+            alignment="center",
+        )
+        stims.append(stim_answer_box)
+        answ_y -= y_offset
+
+        not_finished = True
+        answer = "not answered"
+        start_time = core.monotonicClock.getTime()
+        while not_finished:
+            stim_answer_box.hasFocus = True
+            for stim in stims:
+                stim.draw()
+            WIN.flip()
+            answer = stim_answer_box.text
+            if "\n" in answer:
+                if answer[0].isdigit() and answer[1].isdigit():
+                    not_finished = False
+                else:
+                    stim_answer_box.text = answer[:-1]
+            if len(answer) > 2:
+                stim_answer_box.text = stim_answer_box.text[:2]
+        answer_time = core.monotonicClock.getTime() - start_time
+        answers.append((answer.replace("\n", ""), answer_time))
+
+    # event.waitKeys(keyList=["space"])
+
+    return tuple(answers), tuple(procedure_keys)
+
+
+condition_dlg = gui.Dlg(title="Experiment Condition")
+condition_dlg.addText("Condition")
+condition_dlg.addField("condition")
+CONDITION = condition_dlg.show()[0]
+
+assert CONDITION in ORDER_CONDITIONS
+
+pause = visual.TextBox2(
+    WIN,
+    """Drücken Sie die Leertaste um mit der nächsten Wasserprobe fortzufahren""",
+    letterHeight=50,
+    alignment="center",
+)
+
+pause_new_proc = visual.TextBox2(
+    WIN,
+    """Drücken Sie die Leertaste um mit dem nächsten Kennwert fortzufahren""",
+    letterHeight=50,
+    alignment="center",
+)
+
+pause_after_trial = visual.TextBox2(
+    WIN,
+    """Drücken Sie die Leertaste um fortzufahren""",
+    letterHeight=50,
+    alignment="center",
+)
+
+intro = visual.TextBox2(
+    WIN, intro_text, letterHeight=40, alignment="center", size=(100000, 100000)
+)
+intro.draw()
+WIN.flip()
+event.waitKeys(keyList=["space"])
+
+intro2 = visual.TextBox2(
+    WIN, intro2_text, letterHeight=40, alignment="center", size=(100000, 100000)
+)
+intro2.draw()
+WIN.flip()
+event.waitKeys(keyList=["space"])
+
+train_procedures = PROCEDURE_KEYS[:-1]
+random.shuffle(train_procedures)
+transfer_procedure = train_procedures[-1]
+train_procedures = train_procedures[:-1]
+train_procedures.append(PROCEDURE_KEYS[-1])
+
+all_samples = generate_all_watersamples(TRAIN_TRIALS + TEST_TRIALS)
+
+if CONDITION != "blocked":
+    results = {}
+    for i in range(TRAIN_TRIALS):
+        print(train_procedures)
+        answer, procedure_keys = run_trial(all_samples[i], train_procedures, CONDITION)
+        answer_dict = {}
+        answer_dict["procedure_order"] = procedure_keys
+        answer_dict["water_sample"] = all_samples[i].water_sample_dict()
+        for proc, key in zip(answer, procedure_keys):
+            answer_dict[key] = {"answer": proc[0], "time": proc[1]}
+
+        results[f"train_{i}"] = answer_dict
+
+        pause.draw()
+        WIN.flip()
+        event.waitKeys(keyList=["space"])
+else:
+    results = run_blocked_trials(all_samples[:TRAIN_TRIALS], train_procedures)
+
+
+phase = visual.TextBox2(
+    WIN,
+    """Sie haben den ersten Teil geschafft! Der zweite Teil ist etwas kürzer als der erste
+
+Drücken Sie die Leertaste um anzufangen.""",
+    letterHeight=40,
+    alignment="center",
+)
+phase.draw()
+WIN.flip()
+event.waitKeys(keyList=["space"])
+
+train_procedures[2] = transfer_procedure
+for i in range(TEST_TRIALS):
+    print(train_procedures)
+    answer, procedure_keys = run_trial(
+        all_samples[TRAIN_TRIALS + i], train_procedures, "fixed"
+    )
+    print(procedure_keys)
+    answer_dict = {}
+    answer_dict["procedure_order"] = procedure_keys
+    answer_dict["water_sample"] = all_samples[TRAIN_TRIALS + i].water_sample_dict()
+    for proc, key in zip(answer, procedure_keys):
+        answer_dict[key] = {"answer": proc[0], "time": proc[1]}
+
+    results[f"test_{i}"] = answer_dict
+
+    pause.draw()
+    WIN.flip()
+    event.waitKeys(keyList=["space"])
+
+pprint(results)
+
+df = pd.DataFrame.from_dict(results, orient="index")
+df.to_csv("vp_results.csv")
+
+with open("vp.pkl", "wb") as file:
+    pickle.dump(results, file)
+
+outro_text = "Das Experiment ist nun vorüber.\n\nVielen Dank für Ihre Teilnahme!"
+outro = visual.TextBox2(
+    WIN, outro_text, letterHeight=40, alignment="center", size=(100000, 100000)
+)
+outro.draw()
+WIN.flip()
+event.waitKeys(keyList=["space"])