Actual benchmark results

All results are stored in a json encoded format, which can easily be read and converted into a pandas.DataFrame for simple analysis.

After that we just need to filter the Dataframe to create the overviews.

Note

The way an engine is implemented, encapsulating the code in a single function call, gives misleading results in the benchmarks for the naive implementations, which are no longer executed in the globals() context. Therefore the results for the ‘naive’, ‘inner-func’ and ‘outer-func’ implementations are very similar, although they differ when executed properly in a jupyter notebook for example.

view_names = ["base", "slight01", "medium01"]
host_name = "NUC"
engine_names = ["naive", "inner-func", "outer-func", "naive-caching", "optimized-caching"]

Show code cell source Hide code cell source

from pathlib import Path
import json

import ipywidgets as ipw
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd

sns.set()
plt.rcParams['figure.dpi'] = 300

results_dir = Path("/Users/jneines/development/pycon_berlin_2023/mandelbrot/results/")

results = []
for results_file in results_dir.glob("*/*/*.json"):
    with results_file.open("r") as fd:
        result = json.loads(fd.read())
        results.append(result)
        
df = pd.DataFrame(results)

# filter

dfs = df[
    (df["view"].isin(view_names))
    & (df["hostname"] == host_name)
    & (df["engine"].isin(engine_names))
].reset_index(drop=True)
print(f"We have {len(dfs)} results to analyze.")

# display

tabs = ipw.Tab()

tabs_children = []
tab_names = []
for view_name, _dfs in dfs.groupby("view"):
    tab_names.append(view_name)
    with plt.ioff():
        output = ipw.Output()
        with output:
            fig, axes = plt.subplots()
            sns.violinplot(data=_dfs, x="engine", y="calculation_time", order=engine_names, ax=axes)
            axes.set_ylim(0, None)
            axes.set_ylabel("Calculation time [s]")
            axes.set_xlabel(None)

            axes.set_xticklabels(["\n" * (x_position%2) + label.get_text() for x_position, label in enumerate(axes.get_xticklabels())])
            axes.set_title(view_name)

            plt.tight_layout()
            plt.show()
        tabs_children.append(output)
        
tabs.children = tabs_children
for index, tab_name in enumerate(tab_names):
    tabs.set_title(index, tab_name)

tabs

We have 90 results to analyze.