루브릭 평가: 생성형 AI 평가를 위한 종합 프레임워크

# Sample weight specification:
WEIGHTS_DICT = {
   "[A] Clarity": 0.05,
   "[A] Naturalness": 0.05,
   "[A] Volume Consistency": 0.05,
   "[A] Background Noise": 0.05,
   "[A] Pitch and Tone": 0.05,
   "[A] Audio Format": 0.05,
   "[S] Grammar and Syntax": 0.03,
   "[S] Coherence": 0.06,
   "[S] Pronunciation Accuracy": 0.03,
   "[S] Fluency": 0.05,
   "[S] Prosody": 0.05,
   "[S] Handling of Complex Sentences": 0.03,
   }

def weights_to_df(weights_dict: dict[str, float]) -> pd.DataFrame:
   """Load and process weights from JSON file into numpy array."""
   # Create weights array in the same order as PROPERTIES
   weights = [weights_dict[c] for c in COLUMNS]
   weights = np.array(weights)
   weights = weights / np.sum(weights) # normalize such that they sum to 1

   print(f"Loaded {len(weights)} weights, sum: {np.sum(weights):.6f}")
   return pd.DataFrame(weights[None], columns=COLUMNS)

weights = weights_to_df(WEIGHTS_DICT)

# Plot 1: Weighted Comparison Heatmap
def plot_weighted_comparison_heatmap():
   """Plot 1: Heatmap comparing raw scores vs weighted contributions."""
   # Calculate means using pandas
   model1_means = model1_df.mean()
   model2_means = model2_df.mean()
   weights_series = weights.iloc[0]  # Extract weights as Series
   uniform_weights = np.ones((model1_df.shape[1],)) / model1_df.shape[1]


   # Calculate weighted contributions
   uniform_model1_contributions = model1_means * uniform_weights
   uniform_model2_contributions = model2_means * uniform_weights


   weighted_model1_contributions = model1_means * weights_series
   weighted_model2_contributions = model2_means * weights_series


   # Create figure with subplots
   fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(20, 12))


   # Prepare data for heatmap
   raw_data = pd.DataFrame(
       {
           "Model 1": uniform_model1_contributions,
           "Model 2": uniform_model2_contributions,
       }
   ).T


   weighted_data = pd.DataFrame(
       {
           "Model 1 (Weighted)": weighted_model1_contributions,
           "Model 2 (Weighted)": weighted_model2_contributions,
       }
   ).T


   # Plot 1: Raw scores with weights
   sns.heatmap(
       raw_data,
       annot=True,
       fmt=".3f",
       cmap="RdYlGn",
       ax=ax1,
       cbar_kws={"label": "Score Value"},
   )
   ax1.set_title("Weighted uniformly")
   ax1.set_xticklabels(raw_data.columns, rotation=45, ha="right")


   # Plot 2: Weighted contributions
   sns.heatmap(
       weighted_data,
       annot=True,
       fmt=".3f",
       cmap="RdYlGn",
       ax=ax2,
       cbar_kws={"label": "Weighted Contribution"},
   )
   ax2.set_title("Weighted by rubric")
   ax2.set_xticklabels(weighted_data.columns, rotation=45, ha="right")


   plt.tight_layout()
   plt.show()


# Plot 2: Category Analysis (Clean Pandas Approach)
def plot_category_analysis():
   """Plot 2: Box plots showing performance by category."""


   # Prepare data for plotting
   plot_data_uniform = []
   plot_data_weighted = []


   print("By Group:")
   for grp_name, _ in PROPERTIES:
       cat = grp_name[0]
       columns = model1_df.columns[model1_df.columns.str.slice(1, 2) == cat]
       # Uniform weights
       group_weights = np.ones((columns.shape[0])) / columns.shape[0]
       group1 = (model1_df[columns] * group_weights).mean(axis=1)
       group2 = (model2_df[columns] * group_weights).mean(axis=1)


       for score in group1:
           plot_data_uniform.append(
               {"Category": grp_name, "Model": "Model 1", "Score": score}
           )
       for score in group2:
           plot_data_uniform.append(
               {"Category": grp_name, "Model": "Model 2", "Score": score}
           )


       # Rubric weights
       group_weights = (weights[columns] / weights[columns].sum(axis=1)[0]).iloc[0]
       group1 = (model1_df[columns] * group_weights).mean(axis=1)
       group2 = (model2_df[columns] * group_weights).mean(axis=1)


       for score in group1:
           plot_data_weighted.append(
               {"Category": grp_name, "Model": "Model 1", "Score": score}
           )
       for score in group2:
           plot_data_weighted.append(
               {"Category": grp_name, "Model": "Model 2", "Score": score}
           )


   df_uniform = pd.DataFrame(plot_data_uniform)
   df_weighted = pd.DataFrame(plot_data_weighted)


   # Uniform weights
   _, axs = plt.subplots(1, 2, figsize=(12, 8))
   ax = sns.boxplot(data=df_uniform, x="Category", y="Score", hue="Model", ax=axs[0])
   ax.set_title("Uniform Weights")
   ax.set_ylabel("Average Score")
   ax.legend(title="Model")
   ax.set_xticklabels(ax.get_xticklabels(), rotation=45)


   # Rubric weights
   ax = sns.boxplot(data=df_weighted, x="Category", y="Score", hue="Model", ax=axs[1])
   ax.set_title("Rubric Weights")
   ax.set_ylabel("Average Score")
   ax.legend(title="Model")
   ax.set_xticklabels(ax.get_xticklabels(), rotation=45)
   # plt.xticks(rotation=45)
   plt.tight_layout()
   plt.show()


plot_category_analysis()

"""
Script to explain CDF interpretation with real evaluation data.
"""


def create_real_data_cdf(
   model1_scores, model2_scores, weights, score_threshold: float = 0.925
):
   model1_scores = model1_df @ weights.T
   model2_scores = model2_df @ weights.T


   # Create CDF plot
   fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6))


   # Plot 1: Histograms for context
   ax1.hist(
       model1_scores,
       alpha=0.6,
       label="Model 1",
       bins=30,
       color="lightcoral",
       density=True,
   )
   ax1.hist(
       model2_scores,
       alpha=0.6,
       label="Model 2",
       bins=30,
       color="lightblue",
       density=True,
   )
   ax1.set_xlabel("Weighted Score")
   ax1.set_ylabel("Density")
   ax1.set_title("Real Evaluation Score Distributions (Histograms)")
   ax1.legend()
   ax1.grid(True, alpha=0.3)


   # Plot 2: CDF
   sorted_m1 = np.sort(model1_scores.to_numpy().flatten())
   sorted_m2 = np.sort(model2_scores.to_numpy().flatten())
   y1 = np.arange(1, len(sorted_m1) + 1) / len(sorted_m1)
   y2 = np.arange(1, len(sorted_m2) + 1) / len(sorted_m2)


   ax2.plot(sorted_m1, y1, label="Model 1", color="red", linewidth=3)
   ax2.plot(sorted_m2, y2, label="Model 2", color="blue", linewidth=3)


   # Find probabilities at threshold
   prob_m1 = np.mean(model1_scores <= score_threshold)
   prob_m2 = np.mean(model2_scores <= score_threshold)


   # Add vertical line at threshold
   ax2.axvline(x=score_threshold, color="gray", linestyle="--", alpha=0.7)
   ax2.axhline(y=prob_m1, color="red", linestyle=":", alpha=0.7)
   ax2.axhline(y=prob_m2, color="blue", linestyle=":", alpha=0.7)


   # Add annotations
   ax2.annotate(
       f"At score {score_threshold}:\nModel 1: {prob_m1:.1%} ≤ {score_threshold}\nModel 2: {prob_m2:.1%} ≤ {score_threshold}",
       xy=(score_threshold, 0.5),
       xytext=(0.85, 0.3),
       bbox=dict(boxstyle="round", facecolor="wheat", alpha=0.8),
       arrowprops=dict(arrowstyle="->", color="gray"),
   )


   ax2.set_xlabel("Weighted Score")
   ax2.set_ylabel("Cumulative Probability")
   ax2.set_title("Real Data Cumulative Distribution Function (CDF)")
   ax2.legend()
   ax2.grid(True, alpha=0.3)


   plt.tight_layout()
   plt.savefig("real_data_cdf_explanation.png", dpi=300, bbox_inches="tight")
   plt.show()


create_real_data_cdf(model1_df, model2_df, weights)