From ad49a96ec20e6226e1e946788e49bd0c6ada9f51 Mon Sep 17 00:00:00 2001 From: Molly Hickman Date: Fri, 2 May 2025 16:13:29 -0400 Subject: [PATCH] reinstating the calculate all peer scores --- functions.py | 153 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) diff --git a/functions.py b/functions.py index 29a05b2..42435d8 100644 --- a/functions.py +++ b/functions.py @@ -1188,3 +1188,156 @@ def parse_options_array(options_str): # Simple fallback: just split by comma and strip quotes return [p.strip().strip('"\'') for p in cleaned.split(',')] + +def calculate_peer_score_numeric(row, bot_col, pro_col='pro_median'): + """Calculate peer score for numeric questions""" + try: + # Check if bot didn't provide a forecast + if pd.isna(row[bot_col]): + return np.nan + + resolution_value = row['resolution'] + + # Get the CDF values + bot_cdf = row[bot_col] + pro_median_cdf = row[pro_col] + + # Handle special cases + if resolution_value == 'below_lower_bound': + # Use first point in CDF + if isinstance(bot_cdf, (list, np.ndarray)) and len(bot_cdf) > 0: + bot_prob = bot_cdf[0] + else: + return np.nan + + if isinstance(pro_median_cdf, (list, np.ndarray)) and len(pro_median_cdf) > 0: + pro_median_prob = pro_median_cdf[0] + else: + return np.nan + + elif resolution_value == 'above_upper_bound': + # Use (1 - last point in CDF) + if isinstance(bot_cdf, (list, np.ndarray)) and len(bot_cdf) > 0: + bot_prob = 1 - bot_cdf[-1] + else: + return np.nan + + if isinstance(pro_median_cdf, (list, np.ndarray)) and len(pro_median_cdf) > 0: + pro_median_prob = 1 - pro_median_cdf[-1] + else: + return np.nan + + else: + # Convert to float if it's a numeric resolution + try: + resolution_float = float(resolution_value) + + # Convert CDF to PMF + if isinstance(bot_cdf, (list, np.ndarray)) and isinstance(pro_median_cdf, (list, np.ndarray)): + # Convert CDFs to PMFs + bot_pmf = np.diff(np.concatenate([[0], bot_cdf])) + pro_pmf = np.diff(np.concatenate([[0], pro_median_cdf])) + + # Use nominal_location_to_cdf_location to find the appropriate bucket + cdf_location = nominal_location_to_cdf_location(resolution_float, row) + + # Find the appropriate bucket index + bucket_index = min(int(cdf_location * (len(bot_pmf) - 1)), len(bot_pmf) - 1) + + # Get probabilities + bot_prob = bot_pmf[bucket_index] + pro_median_prob = pro_pmf[bucket_index] + else: + return np.nan + except: + return np.nan + + # Ensure non-zero probabilities + bot_prob = max(bot_prob, 1e-10) + pro_median_prob = max(pro_median_prob, 1e-10) + + # Calculate peer score and divide by 2 for continuous questions + return np.log(bot_prob / pro_median_prob) / 2 + + except Exception as e: + # Print the specific error for debugging + return np.nan + +def calculate_peer_score_binary(row, bot_col, pro_col='pro_median'): + """Calculate peer score for binary questions""" + if row['resolution'] == 'yes': + return np.log(row[bot_col] / row[pro_col]) + else: # resolution is 'no' + return np.log((1 - row[bot_col]) / (1 - row[pro_col])) + +def parse_cdf_string(cdf_string): + """Parse CDF string into numpy array""" + return np.array([float(x) for x in cdf_string.strip('[]').split(',')]) + +def calculate_peer_score_multiple_choice(row, bot_col, pro_col='pro_median'): + """Calculate peer score for multiple choice questions""" + # Check if bot didn't provide a forecast (NaN) + if pd.isna(row[bot_col]): + return np.nan + + # Get the resolution value and options + resolution_value = row['resolution'] + options = row['options_parsed'] if 'options_parsed' in row else row['options'] + + # Find the index of the resolution in options array + resolution_str = str(resolution_value) + + try: + resolution_index = options.index(resolution_str) + + # Get the forecasts + bot_pmf_raw = row[bot_col] + pro_pmf_raw = row[pro_col] + + # Parse string representations of arrays if needed + if isinstance(bot_pmf_raw, str): + bot_pmf = [float(x) for x in bot_pmf_raw.strip('[]').split(',')] + else: + bot_pmf = bot_pmf_raw + + if isinstance(pro_pmf_raw, str): + pro_pmf = [float(x) for x in pro_pmf_raw.strip('[]').split(',')] + else: + pro_pmf = pro_pmf_raw + + # Get the probabilities at the correct index + bot_prob = bot_pmf[resolution_index] + pro_prob = pro_pmf[resolution_index] + + # Calculate peer score + return np.log(bot_prob / pro_prob) + except Exception as e: + # If any error occurs, return NaN + return np.nan + +def calculate_peer_score(row, bot_col, pro_col='pro_median'): + """Calculate peer score based on question type""" + if row['type'] == 'binary': + return calculate_peer_score_binary(row, bot_col, pro_col) + elif row['type'] == 'multiple_choice': + return calculate_peer_score_multiple_choice(row, bot_col, pro_col) + elif row['type'] == 'numeric': + return calculate_peer_score_numeric(row, bot_col, pro_col) + else: + # Unknown question type; return NaN + return np.nan + +def calculate_all_peer_scores(df, all_bots, pro_col='pro_median'): + """Calculate peer scores for all bots""" + # Create a new DataFrame to store peer scores + df_peer = df.copy() + + # Calculate peer score for each bot + for bot in all_bots: + df_peer[bot] = 100 * df.apply(lambda row: calculate_peer_score(row, bot, pro_col), axis=1) + + # Calculate peer score for bot_team_median + df_peer["bot_team_median"] = 100 * df.apply( + lambda row: calculate_peer_score(row, 'bot_median', pro_col), axis=1) + + return df_peer