Evalica¶

Open in Colab Binder

Evalica

In [1]:
import evalica
import pandas as pd
import plotly.express as px
from evalica import Winner
from plotly.graph_objects import Figure
In [2]:
evalica.__version__
Out[2]:
'0.3.0'
In [3]:
df_food = pd.read_csv("food.csv", dtype=str)

df_food["winner"] = df_food["winner"].map({
    "left": Winner.X,
    "right": Winner.Y,
    "tie": Winner.Draw,
})

df_food.head(5)
Out[3]:
left right winner
0 Pizza Sushi Winner.X
1 Burger Pasta Winner.Y
2 Tacos Pizza Winner.X
3 Sushi Tacos Winner.Y
4 Burger Pizza Winner.X
In [4]:
df_food["left_id"], df_food["right_id"], index = evalica.indexing(df_food["left"], df_food["right"])
In [5]:
matrices = evalica.matrices(df_food["left_id"], df_food["right_id"], df_food["winner"], index)
In [6]:
pd.DataFrame(matrices.win_matrix, index=index, columns=index)  # win matrix
Out[6]:
Pizza Burger Tacos Sushi Pasta
Pizza 0.0 0.0 0.0 2.0 1.0
Burger 3.0 0.0 1.0 2.0 0.0
Tacos 4.0 1.0 0.0 1.0 4.0
Sushi 0.0 4.0 1.0 0.0 1.0
Pasta 0.0 2.0 1.0 0.0 0.0
In [7]:
pd.DataFrame(matrices.tie_matrix, index=index, columns=index)  # tie matrix
Out[7]:
Pizza Burger Tacos Sushi Pasta
Pizza 0.0 0.0 0.0 0.0 1.0
Burger 0.0 0.0 0.0 0.0 0.0
Tacos 0.0 0.0 0.0 0.0 0.0
Sushi 0.0 0.0 0.0 0.0 1.0
Pasta 1.0 0.0 0.0 1.0 0.0
In [8]:
count_result = evalica.counting(df_food["left"], df_food["right"], df_food["winner"])
count_result.scores.to_frame()
Out[8]:
counting
Tacos 10.0
Sushi 6.5
Burger 6.0
Pasta 4.0
Pizza 3.5
In [9]:
avr_result = evalica.average_win_rate(df_food["left"], df_food["right"], df_food["winner"])
avr_result.scores.to_frame()
Out[9]:
average_win_rate
Tacos 0.700000
Sushi 0.479167
Burger 0.458333
Pizza 0.437500
Pasta 0.425000
In [10]:
bt_result = evalica.bradley_terry(df_food["left"], df_food["right"], df_food["winner"])
bt_result.scores.to_frame()
Out[10]:
bradley_terry
Tacos 2.509025
Sushi 1.101156
Burger 0.854906
Pasta 0.740381
Pizza 0.571837
In [11]:
newman_result = evalica.newman(df_food["left"], df_food["right"], df_food["winner"])
newman_result.scores.to_frame()
Out[11]:
newman
Tacos 2.665211
Sushi 1.090627
Burger 0.829660
Pasta 0.710154
Pizza 0.536813
In [12]:
eigen_result = evalica.eigen(df_food["left"], df_food["right"], df_food["winner"])
eigen_result.scores.to_frame()
Out[12]:
eigen
Burger 0.528534
Pizza 0.505309
Sushi 0.454870
Pasta 0.439948
Tacos 0.254665
In [13]:
elo_result = evalica.elo(df_food["left"], df_food["right"], df_food["winner"])
elo_result.scores.to_frame()
Out[13]:
elo
Tacos 1013.358777
Sushi 1002.098059
Burger 998.026093
Pasta 994.191306
Pizza 992.325765
In [14]:
df_bt_pairwise = evalica.pairwise_frame(bt_result.scores)

df_bt_pairwise
Out[14]:
Tacos Sushi Burger Pasta Pizza
Tacos 0.500000 0.694986 0.745861 0.772149 0.814391
Sushi 0.305014 0.500000 0.562945 0.597955 0.658195
Burger 0.254139 0.437055 0.500000 0.535895 0.599201
Pasta 0.227851 0.402045 0.464105 0.500000 0.564221
Pizza 0.185609 0.341805 0.400799 0.435779 0.500000
In [15]:
def visualize(df_pairwise: pd.DataFrame) -> Figure:
    fig = px.imshow(df_pairwise, color_continuous_scale="RdBu", text_auto=".2f")

    fig.update_layout(xaxis_title="Loser", yaxis_title="Winner", xaxis_side="top")

    fig.update_traces(hovertemplate="Winner: %{y}<br>Loser: %{x}<br>Fraction of Wins: %{z}")

    return fig
In [16]:
visualize(df_bt_pairwise)
In [ ]: