diff --git a/docs/images/boxplot-naive-vs-biochatter.pdf b/docs/images/boxplot-naive-vs-biochatter.pdf index 3e3f2363..109a4004 100644 Binary files a/docs/images/boxplot-naive-vs-biochatter.pdf and b/docs/images/boxplot-naive-vs-biochatter.pdf differ diff --git a/docs/images/scatter-per-quantisation-name.pdf b/docs/images/scatter-per-quantisation-name.pdf index f77dc960..f1c9a795 100644 Binary files a/docs/images/scatter-per-quantisation-name.pdf and b/docs/images/scatter-per-quantisation-name.pdf differ diff --git a/docs/images/scatter-quantisation-accuracy.pdf b/docs/images/scatter-quantisation-accuracy.pdf new file mode 100644 index 00000000..86e28a60 Binary files /dev/null and b/docs/images/scatter-quantisation-accuracy.pdf differ diff --git a/docs/images/scatter-quantisation-accuracy.png b/docs/images/scatter-quantisation-accuracy.png new file mode 100644 index 00000000..42f82219 Binary files /dev/null and b/docs/images/scatter-quantisation-accuracy.png differ diff --git a/docs/images/scatter-size-accuracy.pdf b/docs/images/scatter-size-accuracy.pdf new file mode 100644 index 00000000..114b0824 Binary files /dev/null and b/docs/images/scatter-size-accuracy.pdf differ diff --git a/docs/images/scatter-size-accuracy.png b/docs/images/scatter-size-accuracy.png new file mode 100644 index 00000000..8d4f10c6 Binary files /dev/null and b/docs/images/scatter-size-accuracy.png differ diff --git a/docs/scripts/hooks.py b/docs/scripts/hooks.py index 94204d2f..b55494fb 100644 --- a/docs/scripts/hooks.py +++ b/docs/scripts/hooks.py @@ -401,6 +401,52 @@ def plot_comparison_naive_biochatter(overview): # TODO publish this test and other related ones on website as well? + # calculate correlation between LLM size and accuracy for all tasks + # convert size to float, make Unknown = 300, replace commas with dots + size = overview_melted["Size"].apply( + lambda x: 300 if x == "Unknown" else float(x.replace(",", ".")) + ) + print(size.corr(overview_melted["Accuracy"])) + # plot scatter plot + plt.figure(figsize=(6, 4)) + sns.scatterplot(x=size, y=overview_melted["Accuracy"]) + plt.xlabel("Model size (billions of parameters)") + plt.ylabel("Accuracy") + plt.title("Scatter plot of model size vs accuracy") + plt.savefig( + f"docs/images/scatter-size-accuracy.png", + bbox_inches="tight", + dpi=300, + ) + plt.savefig( + f"docs/images/scatter-size-accuracy.pdf", + bbox_inches="tight", + ) + plt.close() + + # calculate correlation between quantisation and accuracy for all tasks + # convert quantisation to float, make >= 16-bit* = 16, replace -bit with nothing + quantisation = overview_melted["Quantisation"].apply( + lambda x: 16 if x == ">= 16-bit*" else float(x.replace("-bit", "")) + ) + print(quantisation.corr(overview_melted["Accuracy"])) + # plot scatter plot + plt.figure(figsize=(6, 4)) + sns.scatterplot(x=quantisation, y=overview_melted["Accuracy"]) + plt.xlabel("Quantisation (bits)") + plt.ylabel("Accuracy") + plt.title("Scatter plot of quantisation vs accuracy") + plt.savefig( + f"docs/images/scatter-quantisation-accuracy.png", + bbox_inches="tight", + dpi=300, + ) + plt.savefig( + f"docs/images/scatter-quantisation-accuracy.pdf", + bbox_inches="tight", + ) + plt.close() + def melt_and_process(overview): overview_melted = overview.melt(