From 9e8454750737d187387242195b9a564e376dd147 Mon Sep 17 00:00:00 2001 From: James Chua Date: Thu, 26 Oct 2023 20:24:52 +0100 Subject: [PATCH 01/28] wip --- cot_transparency/tasks.py | 42 ++++- james_ignored_reasoning.py | 21 +++ .../ignored_reasoning/stage_two_analysis.py | 2 +- scripts/james_test_stream.py | 33 ++++ stage_one.py | 156 +++++++++++++++++- 5 files changed, 245 insertions(+), 9 deletions(-) create mode 100644 james_ignored_reasoning.py create mode 100644 scripts/james_test_stream.py diff --git a/cot_transparency/tasks.py b/cot_transparency/tasks.py index 58a184df..3a661383 100644 --- a/cot_transparency/tasks.py +++ b/cot_transparency/tasks.py @@ -1,14 +1,16 @@ +from operator import call import random from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path from typing import Literal, Optional, Type, Union +from altair import overload from pydantic import BaseModel from retry import retry from tqdm import tqdm -from cot_transparency.apis import call_model_api -from cot_transparency.apis.base import InferenceResponse +from cot_transparency.apis import UniversalCaller, call_model_api +from cot_transparency.apis.base import InferenceResponse, ModelCaller from cot_transparency.apis.rate_limiting import exit_event from cot_transparency.data_models.config import OpenaiInferenceConfig from cot_transparency.data_models.io import ( @@ -65,6 +67,7 @@ def __call_or_raise( task: Union[TaskSpec, StageTwoTaskSpec], config: OpenaiInferenceConfig, formatter: Type[PromptFormatter], + caller: ModelCaller, raise_on: Union[Literal["all"], Literal["any"]] = "any", ) -> list[ModelOutput]: if isinstance(task, StageTwoTaskSpec): @@ -72,7 +75,7 @@ def __call_or_raise( else: stage_one_task_spec = task - raw_responses: InferenceResponse = call_model_api(task.messages, config) + raw_responses: InferenceResponse = caller.call(task.messages, config) def get_model_output_for_response(raw_response: str) -> Union[ModelOutput, AnswerNotFound]: parsed_response: str | None = formatter.parse_answer( @@ -123,11 +126,12 @@ def call_model_and_raise_if_not_suitable( task: Union[TaskSpec, StageTwoTaskSpec], config: OpenaiInferenceConfig, formatter: Type[PromptFormatter], + caller: ModelCaller, retries: int = 20, raise_on: Union[Literal["all"], Literal["any"]] = "any", ) -> list[ModelOutput]: responses = retry(exceptions=AtLeastOneFailed, tries=retries)(__call_or_raise)( - task=task, config=config, formatter=formatter, raise_on=raise_on + task=task, config=config, formatter=formatter, raise_on=raise_on,caller=caller, ) return responses @@ -136,20 +140,42 @@ def call_model_and_catch( task: Union[TaskSpec, StageTwoTaskSpec], config: OpenaiInferenceConfig, formatter: Type[PromptFormatter], + caller: ModelCaller, retries: int = 20, raise_on: Union[Literal["all"], Literal["any"]] = "any", ) -> list[ModelOutput]: try: return call_model_and_raise_if_not_suitable( - task=task, config=config, formatter=formatter, retries=retries, raise_on=raise_on + task=task, config=config, formatter=formatter, retries=retries, raise_on=raise_on,caller=caller, ) except AtLeastOneFailed as e: return e.model_outputs +@overload +def task_function( + task: StageTwoTaskSpec, + raise_after_retries: bool, + caller: ModelCaller, + raise_on: Union[Literal["all"], Literal["any"]] = "any", + num_retries: int = 10, +) -> list[StageTwoTaskOutput]: + ... + +@overload +def task_function( + task: TaskSpec, + raise_after_retries: bool, + caller: ModelCaller, + raise_on: Union[Literal["all"], Literal["any"]] = "any", + num_retries: int = 10, +) -> list[TaskOutput]: + ... + def task_function( task: Union[TaskSpec, StageTwoTaskSpec], raise_after_retries: bool, + caller: ModelCaller, raise_on: Union[Literal["all"], Literal["any"]] = "any", num_retries: int = 10, ) -> Union[list[TaskOutput], list[StageTwoTaskOutput]]: @@ -162,6 +188,7 @@ def task_function( formatter=formatter, retries=num_retries, raise_on=raise_on, + caller=caller, ) if raise_after_retries else call_model_and_catch( @@ -170,6 +197,7 @@ def task_function( formatter=formatter, retries=num_retries, raise_on=raise_on, + caller=caller, ) ) @@ -223,8 +251,10 @@ def run_with_caching( elif isinstance(task_to_run[0], StageTwoTaskSpec): loaded_dict = get_loaded_dict_stage2(paths) + output_count = 0 for task_output in loaded_dict.values(): for output in task_output.outputs: + output_count += 1 completed_outputs[output.task_spec.uid()] = output # print number that are None @@ -290,7 +320,7 @@ def kill_and_save(loaded_dict: LoadedJsonType): for task in tasks_to_run: future_instance_outputs.append( executor.submit( - task_function, task, raise_after_retries=raise_after_retries, raise_on=raise_on, num_retries=num_retries + task_function, task, raise_after_retries=raise_after_retries, raise_on=raise_on, num_retries=num_retries # type: ignore ) ) diff --git a/james_ignored_reasoning.py b/james_ignored_reasoning.py new file mode 100644 index 00000000..4819c507 --- /dev/null +++ b/james_ignored_reasoning.py @@ -0,0 +1,21 @@ +from cot_transparency.formatters.transparency.s1_baselines import ZeroShotCOTUnbiasedTameraTFormatter +from scripts.ignored_reasoning.stage_two import main as stage_two_main +from scripts.ignored_reasoning.stage_two_analysis import aoc_plot, plot_adding_mistakes +from stage_one import main as stage_one_main + +if __name__ == "__main__": + # generate COT + exp_dir = "experiments/james_ignored" + stage_one_main( + tasks=["truthful_qa"], + formatters=[ZeroShotCOTUnbiasedTameraTFormatter.name()], + example_cap=10, + models=["gpt-3.5-turbo", "claude-2"], + temperature=1.0, + exp_dir=exp_dir, + batch=20, + ) + stage_two_dir = "experiments/james_ignored" + stage_two_main(input_exp_dir=exp_dir, mistake_model="claude-instant-1", exp_dir=stage_two_dir) + plot_adding_mistakes(exp_dir=stage_two_dir, show_plots=True) + aoc_plot(exp_dir=stage_two_dir, show_plots=True) diff --git a/scripts/ignored_reasoning/stage_two_analysis.py b/scripts/ignored_reasoning/stage_two_analysis.py index adb76121..39ff2ff2 100644 --- a/scripts/ignored_reasoning/stage_two_analysis.py +++ b/scripts/ignored_reasoning/stage_two_analysis.py @@ -329,7 +329,7 @@ def aoc_plot( # Mistakes AoC df_mistakes = df[~df.was_truncated] df_mistakes = df_mistakes.groupby("stage_one_hash").apply(check_same_answer).reset_index(drop=True) - df_mistakes = drop_not_found(df_mistakes) + df_mistakes = drop_not_found(df_mistakes) # type: ignore aoc_mistakes = get_aoc(df_mistakes) # Early Answering AoC diff --git a/scripts/james_test_stream.py b/scripts/james_test_stream.py new file mode 100644 index 00000000..31dd0cdc --- /dev/null +++ b/scripts/james_test_stream.py @@ -0,0 +1,33 @@ +import asyncio +from cot_transparency.apis.base import InferenceResponse, ModelCaller +from cot_transparency.data_models.config import OpenaiInferenceConfig +from cot_transparency.data_models.messages import ChatMessage +from stage_one import stage_one_stream + + +class MockCaller(ModelCaller): + # A caller that can call (mostly) any model + # This exists so that James can easily attach a cache to a single caller with with_file_cache + # He uses a single caller in his script because sometimes its Claude, sometimes its GPT-3.5 + def call( + self, + messages: list[ChatMessage], + config: OpenaiInferenceConfig, + ) -> InferenceResponse: + output = "Let's think step by step... Therefore the best answer is: (A)" + return InferenceResponse(raw_responses=[output]) + + +async def main(): + await stage_one_stream( + formatters=["ZeroShotCOTUnbiasedFormatter"], + dataset="cot_testing", + example_cap=400, + raise_after_retries=False, + temperature=1.0, + caller=MockCaller(), + ).tqdm(None).run_to_completion() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/stage_one.py b/stage_one.py index 1f259af6..d5c5c189 100644 --- a/stage_one.py +++ b/stage_one.py @@ -4,7 +4,11 @@ from typing import Literal, Optional, Type, Sequence import fire +from grugstream import Observable + from slist import Slist +from cot_transparency.apis import UniversalCaller +from cot_transparency.apis.base import ModelCaller from cot_transparency.data_models.config import config_from_default from cot_transparency.data_models.data.bbh import BBH_TASK_LIST @@ -24,7 +28,7 @@ ) from cot_transparency.data_models.data.bbh_biased_wrong_cot import BiasedWrongCOTBBH from cot_transparency.data_models.example_base import DataExampleBase -from cot_transparency.data_models.models import TaskSpec +from cot_transparency.data_models.models import TaskOutput, TaskSpec from cot_transparency.formatters.base_class import StageOneFormatter @@ -40,7 +44,7 @@ ZeroShotCOTSycophancyFormatter, ZeroShotCOTUnbiasedFormatter, ) -from cot_transparency.tasks import TaskSetting, run_with_caching +from cot_transparency.tasks import TaskSetting, run_with_caching, task_function from cot_transparency.util import get_exp_dir_name # ok to train on the test set since we test on completely different datasets @@ -196,6 +200,153 @@ def get_list_of_examples( return data # type: ignore +def stage_one_stream( + tasks: Sequence[str] = [], + dataset: Optional[str] = None, + models: Sequence[str] = ["gpt-3.5-turbo", "gpt-4"], + formatters: Sequence[str] = [ZeroShotCOTSycophancyFormatter.name(), ZeroShotCOTUnbiasedFormatter.name()], + # Pass in a list of interventions to run, indicate None to run no intervention as well + interventions: Sequence[str | None] = [], + exp_dir: Optional[str] = None, + experiment_suffix: str = "", + example_cap: Optional[int] = 1000000, + save_file_every: int = 50, + batch: int = 20, + repeats_per_question: int = 1, + temperature: Optional[float] = None, + raise_after_retries: bool = True, + raise_on: Literal["all", "any"] = "all", + num_retries: int = 10, + max_tokens: Optional[int] = None, + n_responses_per_request: Optional[int] = None, + retry_answers_with_none: bool = False, + caller: ModelCaller = UniversalCaller(), +) -> Observable[TaskOutput]: + if dataset is not None: + # we are using a dataset + assert len(tasks) == 0, "You have defined a dataset and a task, you can only define one" + tasks = TASK_LIST[dataset] + else: + assert tasks, "You must define a task or a dataset" + + for model in models: + if "llama" in model.lower(): + assert batch == 1, "Llama only supports batch size of 1" + print("Number of models to run:", len(models)) + + # match formatter name wildcard + formatters = match_wildcard_formatters(formatters) + + assert len(formatters) > 0, "You must define at least one formatter" + + tasks = validate_tasks(tasks) + print("Number of tasks to run:", len(tasks)) + validated_formatters = get_valid_stage1_formatters(formatters) + print("Number of formatters to run:", len(validated_formatters)) + validated_interventions = get_valid_stage1_interventions(interventions) + print("Number of interventions to run:", len(validated_interventions)) + exp_dir = get_exp_dir_name(exp_dir, experiment_suffix, sub_dir="stage_one") + + task_settings: list[TaskSetting] = create_task_settings( + tasks=tasks, models=models, formatters=validated_formatters, interventions=validated_interventions + ) + + tasks_to_run: list[TaskSpec] = [] + print("Number of settings to run:", len(task_settings)) + for setting in task_settings: + task = setting.task + model = setting.model + formatter = setting.formatter + # Shuffle the data BEFORE we cap it + # Pass 42 to maintain the same shuffle that we had in the past, though slist wants a string instead + data: Slist[DataExampleBase] = get_list_of_examples(task, dataset=dataset).shuffle(typing.cast(str, 42)) + out_file_path: Path = ( + Path(f"{exp_dir}/{task}/{model}/{formatter.name()}.json") + if setting.intervention is None + else Path(f"{exp_dir}/{task}/{model}/{formatter.name()}_and_{setting.intervention.name()}.json") + ) + + if example_cap: + data = data.take(example_cap) + + # Config Overrides Start ---------------------- + config = config_from_default(model).model_copy(deep=True) + if issubclass(formatter, FormattersForTransparency): + few_shot_stops = ["\n\nHuman:", "\n\nAssistant:", "\n\nQuestion:"] + if isinstance(config.stop, list): + config.stop += few_shot_stops + else: + config.stop = few_shot_stops + config.max_tokens = 300 + config.temperature = 0.8 + config.top_p = 0.95 + + # if you are using an intervention, we need to add SINGLE_SHOT_SEP to the stop list + if setting.intervention: + if isinstance(config.stop, list): + config.stop += [FEW_SHOT_STOP_TOKEN] + else: + config.stop = [FEW_SHOT_STOP_TOKEN] + + if temperature is not None: + config.temperature = temperature + + assert config.model == model + + if not formatter.is_cot: + config.max_tokens = 1 + + if max_tokens is not None: + config.max_tokens = max_tokens + + if n_responses_per_request is not None: + config.n = n_responses_per_request + + # Config Overrides End ---------------------- + + if raise_after_retries and temperature == 0: + raise ValueError("Must set --raise_after_retires=False when temperature is 0 as it will always fail") + + for item in data: + for i in range(repeats_per_question): + messages = ( + setting.intervention.intervene(question=item, formatter=formatter, model=model) + if setting.intervention + else formatter.format_example(question=item, model=model) + ) + # Save the format spec defined by the formatter + new_item: DataExampleBase = item.model_copy() + format_spec = formatter.get_data_format_spec() + new_item.data_format = format_spec + task_spec = TaskSpec( + task_name=task, + inference_config=config, + messages=messages, + out_file_path=out_file_path, + ground_truth=new_item.ground_truth, + formatter_name=formatter.name(), + task_hash=new_item.hash(), + biased_ans=new_item.biased_ans, + data_example=new_item.model_dump(), + repeat_idx=i, + intervention_name=setting.intervention.name() if setting.intervention else None, + ) + tasks_to_run.append(task_spec) + return ( + Observable.from_iterable(tasks_to_run) + .map_blocking_par( + lambda task_spec: task_function( + task=task_spec, + raise_after_retries=raise_after_retries, + raise_on=raise_on, + caller=caller, + ), + max_par=batch, + ) + .flatten_list() + ) + + def main( tasks: Sequence[str] = [], dataset: Optional[str] = None, @@ -216,6 +367,7 @@ def main( max_tokens: Optional[int] = None, n_responses_per_request: Optional[int] = None, retry_answers_with_none: bool = False, + caller: ModelCaller = UniversalCaller(), ): if dataset is not None: # we are using a dataset From ec8ddbaecbcd1f97f2cb016c34425b542219424e Mon Sep 17 00:00:00 2001 From: James Chua Date: Thu, 26 Oct 2023 20:37:52 +0100 Subject: [PATCH 02/28] add early answering --- scripts/ignored_reasoning/stage_two.py | 1 + scripts/james_test_stream.py | 41 +++++++++++++++++++++----- stage_one.py | 2 +- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/scripts/ignored_reasoning/stage_two.py b/scripts/ignored_reasoning/stage_two.py index 91846663..2df7fb38 100644 --- a/scripts/ignored_reasoning/stage_two.py +++ b/scripts/ignored_reasoning/stage_two.py @@ -53,6 +53,7 @@ def filter_cot_by_possible_ends(cot_steps: list[str]) -> list[str]: return filtered_steps + def get_early_answering_tasks( stage_one_output: TaskOutput, exp_dir: str, diff --git a/scripts/james_test_stream.py b/scripts/james_test_stream.py index 31dd0cdc..42536e2a 100644 --- a/scripts/james_test_stream.py +++ b/scripts/james_test_stream.py @@ -1,7 +1,11 @@ import asyncio + +from zipp import Path from cot_transparency.apis.base import InferenceResponse, ModelCaller from cot_transparency.data_models.config import OpenaiInferenceConfig from cot_transparency.data_models.messages import ChatMessage +from cot_transparency.tasks import task_function +from scripts.ignored_reasoning.stage_two import get_early_answering_tasks from stage_one import stage_one_stream @@ -19,14 +23,35 @@ def call( async def main(): - await stage_one_stream( - formatters=["ZeroShotCOTUnbiasedFormatter"], - dataset="cot_testing", - example_cap=400, - raise_after_retries=False, - temperature=1.0, - caller=MockCaller(), - ).tqdm(None).run_to_completion() + stage_one_cache_dir = Path("experiments/stage_one.jsonl") + stage_one_caller = MockCaller() + stage_two_cache_dir = Path("experiments/stage_two.jsonl") + obs = ( + stage_one_stream( + formatters=["ZeroShotCOTUnbiasedFormatter"], + dataset="cot_testing", + example_cap=400, + raise_after_retries=False, + temperature=1.0, + caller=MockCaller(), + ) + .tqdm(None) + .map( + lambda task_output: get_early_answering_tasks( + stage_one_output=task_output, + exp_dir="not_used", + temperature=None, + n_samples_per_cot=4, + full_answers_only=False, + ) + ) + .flatten_list() + .map_blocking_par( + lambda stage_two_spec: task_function(task=stage_two_spec, raise_after_retries=False, caller=MockCaller()) + ) + .flatten_list() + ) + await obs.run_to_completion() if __name__ == "__main__": diff --git a/stage_one.py b/stage_one.py index d5c5c189..4fd63e68 100644 --- a/stage_one.py +++ b/stage_one.py @@ -334,7 +334,7 @@ def stage_one_stream( tasks_to_run.append(task_spec) return ( Observable.from_iterable(tasks_to_run) - .map_blocking_par( + .map_blocking_par( lambda task_spec: task_function( task=task_spec, raise_after_retries=raise_after_retries, From eaaf740571d9507267e74c7a02b78885a180c5c3 Mon Sep 17 00:00:00 2001 From: James Chua Date: Thu, 26 Oct 2023 21:07:29 +0100 Subject: [PATCH 03/28] make typings happy --- scripts/ignored_reasoning/stage_two_analysis.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/scripts/ignored_reasoning/stage_two_analysis.py b/scripts/ignored_reasoning/stage_two_analysis.py index 39ff2ff2..a80faf77 100644 --- a/scripts/ignored_reasoning/stage_two_analysis.py +++ b/scripts/ignored_reasoning/stage_two_analysis.py @@ -106,12 +106,17 @@ def convert_stage2_experiment_to_dataframe(exp: StageTwoExperimentJsonFormat) -> def get_data_frame_from_exp_dir(exp_dir: str) -> pd.DataFrame: - loaded_dict = ExpLoader.stage_two(exp_dir, final_only=True) - dfs = [] - for exp in loaded_dict.values(): - df = convert_stage2_experiment_to_dataframe(exp) - dfs.append(df) - df = pd.concat(dfs) + df = convert_stage2_experiment_to_dataframe(loaded) + df["is_correct"] = (df.parsed_response == df.ground_truth).astype(int) + # filter out the NOT_FOUND rows + n_not_found = len(df[df.parsed_response == "NOT_FOUND"]) + print(f"Number of NOT_FOUND rows: {n_not_found}") + df = df[df.parsed_response != "NOT_FOUND"] + return df # type: ignore + + +def get_data_frame_from_exp_dir_items(items: Sequence[StageTwoTaskOutput]) -> pd.DataFrame: + df = convert_stage2_experiment_to_dataframe(items) df["is_correct"] = (df.parsed_response == df.ground_truth).astype(int) # filter out the NOT_FOUND rows n_not_found = len(df[df.parsed_response == "NOT_FOUND"]) From af57e802b886ac18fec12eca954e625023bea06d Mon Sep 17 00:00:00 2001 From: James Chua Date: Thu, 26 Oct 2023 21:14:28 +0100 Subject: [PATCH 04/28] refactor --- cot_transparency/tasks.py | 41 ++++++++++++----- scripts/ignored_reasoning/stage_two.py | 1 - .../ignored_reasoning/stage_two_analysis.py | 45 ++++++++++++++++--- scripts/james_test_stream.py | 38 +++++++++------- stage_one.py | 2 +- 5 files changed, 95 insertions(+), 32 deletions(-) diff --git a/cot_transparency/tasks.py b/cot_transparency/tasks.py index 3a661383..2f4fe380 100644 --- a/cot_transparency/tasks.py +++ b/cot_transparency/tasks.py @@ -1,4 +1,3 @@ -from operator import call import random from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path @@ -8,8 +7,8 @@ from pydantic import BaseModel from retry import retry from tqdm import tqdm +from cot_transparency.apis import UniversalCaller -from cot_transparency.apis import UniversalCaller, call_model_api from cot_transparency.apis.base import InferenceResponse, ModelCaller from cot_transparency.apis.rate_limiting import exit_event from cot_transparency.data_models.config import OpenaiInferenceConfig @@ -131,7 +130,11 @@ def call_model_and_raise_if_not_suitable( raise_on: Union[Literal["all"], Literal["any"]] = "any", ) -> list[ModelOutput]: responses = retry(exceptions=AtLeastOneFailed, tries=retries)(__call_or_raise)( - task=task, config=config, formatter=formatter, raise_on=raise_on,caller=caller, + task=task, + config=config, + formatter=formatter, + raise_on=raise_on, + caller=caller, ) return responses @@ -146,36 +149,54 @@ def call_model_and_catch( ) -> list[ModelOutput]: try: return call_model_and_raise_if_not_suitable( - task=task, config=config, formatter=formatter, retries=retries, raise_on=raise_on,caller=caller, + task=task, + config=config, + formatter=formatter, + retries=retries, + raise_on=raise_on, + caller=caller, ) except AtLeastOneFailed as e: return e.model_outputs + @overload -def task_function( +def task_function( task: StageTwoTaskSpec, raise_after_retries: bool, - caller: ModelCaller, + caller: ModelCaller = ..., raise_on: Union[Literal["all"], Literal["any"]] = "any", num_retries: int = 10, ) -> list[StageTwoTaskOutput]: ... + @overload -def task_function( +def task_function( task: TaskSpec, raise_after_retries: bool, - caller: ModelCaller, + caller: ModelCaller = ..., raise_on: Union[Literal["all"], Literal["any"]] = "any", num_retries: int = 10, ) -> list[TaskOutput]: ... +@overload def task_function( task: Union[TaskSpec, StageTwoTaskSpec], raise_after_retries: bool, - caller: ModelCaller, + caller: ModelCaller = ..., + raise_on: Union[Literal["all"], Literal["any"]] = "any", + num_retries: int = 10, +) -> Union[list[TaskOutput], list[StageTwoTaskOutput]]: + ... + + +def task_function( + task: Union[TaskSpec, StageTwoTaskSpec], + raise_after_retries: bool, + caller: ModelCaller = UniversalCaller(), raise_on: Union[Literal["all"], Literal["any"]] = "any", num_retries: int = 10, ) -> Union[list[TaskOutput], list[StageTwoTaskOutput]]: @@ -320,7 +341,7 @@ def kill_and_save(loaded_dict: LoadedJsonType): for task in tasks_to_run: future_instance_outputs.append( executor.submit( - task_function, task, raise_after_retries=raise_after_retries, raise_on=raise_on, num_retries=num_retries # type: ignore + lambda: task_function(task=task, raise_after_retries=True, raise_on=raise_on, num_retries=num_retries) ) ) diff --git a/scripts/ignored_reasoning/stage_two.py b/scripts/ignored_reasoning/stage_two.py index 2df7fb38..91846663 100644 --- a/scripts/ignored_reasoning/stage_two.py +++ b/scripts/ignored_reasoning/stage_two.py @@ -53,7 +53,6 @@ def filter_cot_by_possible_ends(cot_steps: list[str]) -> list[str]: return filtered_steps - def get_early_answering_tasks( stage_one_output: TaskOutput, exp_dir: str, diff --git a/scripts/ignored_reasoning/stage_two_analysis.py b/scripts/ignored_reasoning/stage_two_analysis.py index a80faf77..649bcc0c 100644 --- a/scripts/ignored_reasoning/stage_two_analysis.py +++ b/scripts/ignored_reasoning/stage_two_analysis.py @@ -1,9 +1,10 @@ from typing import Optional import fire +from git import Sequence from matplotlib import pyplot as plt from analysis import get_general_metrics from cot_transparency.data_models.models import ( - StageTwoExperimentJsonFormat, + StageTwoTaskOutput, TaskOutput, ) import pandas as pd @@ -13,6 +14,7 @@ import seaborn as sns import numpy as np + # Used to produce human readable names on plots NAMES_MAP = { "model": "Model", @@ -72,9 +74,9 @@ def get_auc(group: pd.DataFrame) -> float: return areas -def convert_stage2_experiment_to_dataframe(exp: StageTwoExperimentJsonFormat) -> pd.DataFrame: +def convert_stage2_experiment_to_dataframe(exp: Sequence[StageTwoTaskOutput]) -> pd.DataFrame: out = [] - for task_output in exp.outputs: + for task_output in exp: d_with_config = get_general_metrics(task_output) d_with_config["model"] = task_output.task_spec.inference_config.model d_with_config["task_name"] = task_output.task_spec.stage_one_output.task_spec.task_name @@ -106,7 +108,12 @@ def convert_stage2_experiment_to_dataframe(exp: StageTwoExperimentJsonFormat) -> def get_data_frame_from_exp_dir(exp_dir: str) -> pd.DataFrame: - df = convert_stage2_experiment_to_dataframe(loaded) + loaded_dict = ExpLoader.stage_two(exp_dir, final_only=True) + dfs = [] + for exp in loaded_dict.values(): + df = convert_stage2_experiment_to_dataframe(exp.outputs) + dfs.append(df) + df = pd.concat(dfs) df["is_correct"] = (df.parsed_response == df.ground_truth).astype(int) # filter out the NOT_FOUND rows n_not_found = len(df[df.parsed_response == "NOT_FOUND"]) @@ -265,7 +272,35 @@ def plot_early_answering( col: str = "original_cot_trace_length", hue: str = "model", ): - df = get_data_frame_from_exp_dir(exp_dir) + items: list[StageTwoTaskOutput] = [] + loaded_dict = ExpLoader.stage_two(exp_dir, final_only=True) + for vals in loaded_dict.values(): + outputs = vals.outputs + items.extend(outputs) + + return plot_early_answering_from_list( + items=items, + show_plots=show_plots, + inconsistent_only=inconsistent_only, + aggregate_over_tasks=aggregate_over_tasks, + model_filter=model_filter, + length_filter=length_filter, + col=col, + hue=hue, + ) + + +def plot_early_answering_from_list( + items: Sequence[StageTwoTaskOutput], + show_plots: bool = False, + inconsistent_only: bool = False, + aggregate_over_tasks: bool = False, + model_filter: Optional[str] = None, + length_filter: Optional[list[int]] = None, + col: str = "original_cot_trace_length", + hue: str = "model", +): + df = get_data_frame_from_exp_dir_items(items=items) # drop formatters that have Mistake in the name df = df[~df.has_mistake] df = df_filters(df, inconsistent_only, aggregate_over_tasks, model_filter, length_filter) # type: ignore diff --git a/scripts/james_test_stream.py b/scripts/james_test_stream.py index 42536e2a..355ed963 100644 --- a/scripts/james_test_stream.py +++ b/scripts/james_test_stream.py @@ -1,11 +1,12 @@ import asyncio +from pathlib import Path -from zipp import Path from cot_transparency.apis.base import InferenceResponse, ModelCaller from cot_transparency.data_models.config import OpenaiInferenceConfig from cot_transparency.data_models.messages import ChatMessage from cot_transparency.tasks import task_function from scripts.ignored_reasoning.stage_two import get_early_answering_tasks +from scripts.ignored_reasoning.stage_two_analysis import plot_early_answering_from_list from stage_one import stage_one_stream @@ -24,19 +25,21 @@ def call( async def main(): stage_one_cache_dir = Path("experiments/stage_one.jsonl") - stage_one_caller = MockCaller() + + stage_one_caller = MockCaller().with_file_cache(stage_one_cache_dir) stage_two_cache_dir = Path("experiments/stage_two.jsonl") - obs = ( - stage_one_stream( - formatters=["ZeroShotCOTUnbiasedFormatter"], - dataset="cot_testing", - example_cap=400, - raise_after_retries=False, - temperature=1.0, - caller=MockCaller(), - ) - .tqdm(None) - .map( + stage_two_caller = MockCaller().with_file_cache(stage_two_cache_dir) + stage_one_obs = stage_one_stream( + formatters=["ZeroShotCOTUnbiasedFormatter"], + dataset="cot_testing", + example_cap=400, + raise_after_retries=False, + temperature=1.0, + caller=stage_one_caller, + ).tqdm(None) + + early_answer_obs = ( + stage_one_obs.map( lambda task_output: get_early_answering_tasks( stage_one_output=task_output, exp_dir="not_used", @@ -47,11 +50,16 @@ async def main(): ) .flatten_list() .map_blocking_par( - lambda stage_two_spec: task_function(task=stage_two_spec, raise_after_retries=False, caller=MockCaller()) + lambda stage_two_spec: task_function( + task=stage_two_spec, raise_after_retries=False, caller=stage_two_caller + ) ) .flatten_list() ) - await obs.run_to_completion() + stage_one_caller.save_cache() + stage_two_caller.save_cache() + early_answer_results = await early_answer_obs.to_list() + plot_early_answering_from_list(items=early_answer_results, show_plots=True) if __name__ == "__main__": diff --git a/stage_one.py b/stage_one.py index 4fd63e68..d5c5c189 100644 --- a/stage_one.py +++ b/stage_one.py @@ -334,7 +334,7 @@ def stage_one_stream( tasks_to_run.append(task_spec) return ( Observable.from_iterable(tasks_to_run) - .map_blocking_par( + .map_blocking_par( lambda task_spec: task_function( task=task_spec, raise_after_retries=raise_after_retries, From 33c4ea9fa29b815754d80b6bd0df5e4f4f499e4d Mon Sep 17 00:00:00 2001 From: James Chua Date: Thu, 26 Oct 2023 21:33:35 +0100 Subject: [PATCH 05/28] save wip, still need the recompletion part --- scripts/ignored_reasoning/stage_two.py | 111 +++++++++++------- ...mes_test_stream.py => stage_two_stream.py} | 36 +++++- 2 files changed, 100 insertions(+), 47 deletions(-) rename scripts/{james_test_stream.py => stage_two_stream.py} (69%) diff --git a/scripts/ignored_reasoning/stage_two.py b/scripts/ignored_reasoning/stage_two.py index 91846663..225c81f4 100644 --- a/scripts/ignored_reasoning/stage_two.py +++ b/scripts/ignored_reasoning/stage_two.py @@ -3,6 +3,7 @@ from typing import List, Literal, Optional, Type import fire +from git import Sequence from cot_transparency.data_models.config import CONFIG_MAP from cot_transparency.data_models.data.task_name_map import task_name_to_data_example @@ -108,6 +109,60 @@ def get_early_answering_tasks( return outputs +def create_mistake_task_spec_for_stage_one( + stage_one_output: TaskOutput, + exp_dir: str, + mistake_adding_model: str, + mistake_adding_temperature: float, + n_mistake_insertion_points: int, +) -> list[StageTwoTaskSpec]: + DataExample = task_name_to_data_example(stage_one_output.task_spec.task_name) + data_example = stage_one_output.task_spec.read_data_example_or_raise(DataExample) + original_question: str = data_example.get_parsed_input() + + cot_steps = get_cot_steps(stage_one_output.first_raw_response) + cot_steps = filter_cot_by_possible_ends(cot_steps) + + rng = random.Random(original_question) + # we don't want to insert mistakes at the first step, because then there is no sentence to make a mistake in + # as the first one is blank cot + if len(cot_steps) == 0: + print("WARNING - skipping task as len(cot_steps) == 0") + return [] + + sample_idxs = [0, len(cot_steps) - 1] # we always do the first and last one to get good aoc numbers + if len(cot_steps) > 2: + mistake_idxs = rng.sample(range(1, len(cot_steps) - 1), min(n_mistake_insertion_points, len(cot_steps) - 2)) + sample_idxs.extend(mistake_idxs) + sample_idxs = sorted(list(set(sample_idxs))) + + config = CONFIG_MAP[mistake_adding_model].copy() + config.max_tokens = 100 + config.temperature = mistake_adding_temperature + config.stop = ["\n", "```"] + + original_model_that_generated_cot = stage_one_output.task_spec.inference_config.model + path = Path( + f"{exp_dir}/mistakes_stage1/s1-{stage_one_output.task_spec.formatter_name}/cot-{original_model_that_generated_cot}/{stage_one_output.task_spec.task_name}/" + f"/mistake-{config.model}/{FewShotGenerateMistakeFormatter.name()}.json" + ) + out: list[StageTwoTaskSpec] = [] + for i in sample_idxs: + messages = FewShotGenerateMistakeFormatter.format_example( + original_question=original_question, sentence=cot_steps[i] + ) + task_spec = StageTwoTaskSpec( + stage_one_output=stage_one_output, + inference_config=config, + formatter_name=FewShotGenerateMistakeFormatter.name(), + messages=messages, + out_file_path=path, + trace_info=TraceInfo(original_cot=cot_steps, mistake_inserted_idx=i), + ) + out.append(task_spec) + return out + + def get_mistakes( stage_one_outputs: list[TaskOutput], exp_dir: str, @@ -121,49 +176,15 @@ def get_mistakes( # e.g Tamera et al use a non RLHF model to generate mistakes specs: list[StageTwoTaskSpec] = [] for stage_one_output in stage_one_outputs: - DataExample = task_name_to_data_example(stage_one_output.task_spec.task_name) - data_example = stage_one_output.task_spec.read_data_example_or_raise(DataExample) - original_question: str = data_example.get_parsed_input() - - cot_steps = get_cot_steps(stage_one_output.first_raw_response) - cot_steps = filter_cot_by_possible_ends(cot_steps) - - rng = random.Random(original_question) - # we don't want to insert mistakes at the first step, because then there is no sentence to make a mistake in - # as the first one is blank cot - if len(cot_steps) == 0: - print("WARNING - skipping task as len(cot_steps) == 0") - continue - - sample_idxs = [0, len(cot_steps) - 1] # we always do the first and last one to get good aoc numbers - if len(cot_steps) > 2: - mistake_idxs = rng.sample(range(1, len(cot_steps) - 1), min(n_mistake_insertion_points, len(cot_steps) - 2)) - sample_idxs.extend(mistake_idxs) - sample_idxs = sorted(list(set(sample_idxs))) - - config = CONFIG_MAP[mistake_adding_model].copy() - config.max_tokens = 100 - config.temperature = mistake_adding_temperature - config.stop = ["\n", "```"] - - original_model_that_generated_cot = stage_one_output.task_spec.inference_config.model - path = Path( - f"{exp_dir}/mistakes_stage1/s1-{stage_one_output.task_spec.formatter_name}/cot-{original_model_that_generated_cot}/{stage_one_output.task_spec.task_name}/" - f"/mistake-{config.model}/{FewShotGenerateMistakeFormatter.name()}.json" - ) - for i in sample_idxs: - messages = FewShotGenerateMistakeFormatter.format_example( - original_question=original_question, sentence=cot_steps[i] - ) - task_spec = StageTwoTaskSpec( + specs.extend( + create_mistake_task_spec_for_stage_one( stage_one_output=stage_one_output, - inference_config=config, - formatter_name=FewShotGenerateMistakeFormatter.name(), - messages=messages, - out_file_path=path, - trace_info=TraceInfo(original_cot=cot_steps, mistake_inserted_idx=i), + exp_dir=exp_dir, + mistake_adding_model=mistake_adding_model, + mistake_adding_temperature=mistake_adding_temperature, + n_mistake_insertion_points=n_mistake_insertion_points, ) - specs.append(task_spec) + ) print(f"1. Generating mistakes using {mistake_adding_model}") print("n specs", len(specs)) @@ -184,6 +205,10 @@ def get_mistakes( outputs = run_with_caching_stage_two(save_mistake_generating_file_every, batch, specs) + return filter_mistakes_output(outputs) + + +def filter_mistakes_output(outputs: Sequence[StageTwoTaskOutput]) -> list[StageTwoTaskOutput]: # Discard outputs where there was no output filtered_outputs = [output for output in outputs if output.first_parsed_response is not None] n_discarded = len(outputs) - len(filtered_outputs) @@ -194,9 +219,7 @@ def get_mistakes( filtered_outputs = [i for i in outputs if "NO_REASONING" not in i.first_parsed_response] # type: ignore n_discarded = len(outputs) - len(filtered_outputs) print(f"Discarding {n_discarded} outputs where the mistake model didn't deem there to be a reasoning step") - outputs = filtered_outputs - - return outputs + return filtered_outputs def recomplete_cot_with_inserted_mistake( diff --git a/scripts/james_test_stream.py b/scripts/stage_two_stream.py similarity index 69% rename from scripts/james_test_stream.py rename to scripts/stage_two_stream.py index 355ed963..f8187e0b 100644 --- a/scripts/james_test_stream.py +++ b/scripts/stage_two_stream.py @@ -4,8 +4,14 @@ from cot_transparency.apis.base import InferenceResponse, ModelCaller from cot_transparency.data_models.config import OpenaiInferenceConfig from cot_transparency.data_models.messages import ChatMessage +from cot_transparency.data_models.models import StageTwoTaskOutput from cot_transparency.tasks import task_function -from scripts.ignored_reasoning.stage_two import get_early_answering_tasks +from scripts.ignored_reasoning.stage_two import ( + create_mistake_task_spec_for_stage_one, + filter_mistakes_output, + get_early_answering_tasks, + get_mistakes, +) from scripts.ignored_reasoning.stage_two_analysis import plot_early_answering_from_list from stage_one import stage_one_stream @@ -56,11 +62,35 @@ async def main(): ) .flatten_list() ) - stage_one_caller.save_cache() - stage_two_caller.save_cache() early_answer_results = await early_answer_obs.to_list() plot_early_answering_from_list(items=early_answer_results, show_plots=True) + mistakes_obs = ( + stage_one_obs.map( + lambda task_output: create_mistake_task_spec_for_stage_one( + stage_one_output=task_output, + exp_dir="not_used", + mistake_adding_temperature=1.0, + n_mistake_insertion_points=4, + mistake_adding_model="claude-instant-1", + ) + ) + .flatten_list() + .map_blocking_par( + lambda stage_two_spec: task_function( + task=stage_two_spec, raise_after_retries=False, caller=stage_two_caller + ) + ) + .flatten_list() + ) + mistakes_results = await mistakes_obs.to_list() + filtered_results: list[StageTwoTaskOutput] = filter_mistakes_output(mistakes_results) + # todo: you need to get the + + + stage_two_caller.save_cache() + stage_one_caller.save_cache() + if __name__ == "__main__": asyncio.run(main()) From 543f78aec3ee94a9dc519e08ebca358771768747 Mon Sep 17 00:00:00 2001 From: James Chua Date: Thu, 26 Oct 2023 22:45:27 +0100 Subject: [PATCH 06/28] add recomputation --- scripts/ignored_reasoning/stage_two.py | 149 +++++++++++++++---------- scripts/stage_two_stream.py | 31 +++-- 2 files changed, 114 insertions(+), 66 deletions(-) diff --git a/scripts/ignored_reasoning/stage_two.py b/scripts/ignored_reasoning/stage_two.py index 225c81f4..2b333ba6 100644 --- a/scripts/ignored_reasoning/stage_two.py +++ b/scripts/ignored_reasoning/stage_two.py @@ -1,9 +1,10 @@ import random from pathlib import Path -from typing import List, Literal, Optional, Type +from typing import List, Literal, NewType, Optional, Type import fire from git import Sequence +from cot_transparency.apis.base import ModelCaller from cot_transparency.data_models.config import CONFIG_MAP from cot_transparency.data_models.data.task_name_map import task_name_to_data_example @@ -28,7 +29,7 @@ from cot_transparency.formatters.transparency.trace_manipulation import get_cot_steps from cot_transparency.formatters.transparency.util import FullCOTCompletionFormatter, FullCOTFormatter from cot_transparency.apis.openai.set_key import set_keys_from_env -from cot_transparency.tasks import run_with_caching_stage_two +from cot_transparency.tasks import run_with_caching_stage_two, task_function from cot_transparency.util import get_exp_dir_name from stage_one import get_valid_stage1_formatters @@ -222,20 +223,19 @@ def filter_mistakes_output(outputs: Sequence[StageTwoTaskOutput]) -> list[StageT return filtered_outputs -def recomplete_cot_with_inserted_mistake( - generated_mistakes: list[StageTwoTaskOutput], - exp_dir: str, - save_completing_with_mistakes_every: int = 50, - batch: int = 10, -) -> List[StageTwoTaskOutput]: - mistakes_inserted_at_last_position: list[StageTwoTaskOutput] = [] - specs: list[StageTwoTaskSpec] = [] +# A Newtype so that you won't get confused with all the different types +RecomputeTaskSpec = NewType("RecomputeTaskSpec", StageTwoTaskSpec) - for generated_mistake in generated_mistakes: - if generated_mistake.first_parsed_response is None or "NO_REASONING" in generated_mistake.first_parsed_response: - print("WARNING - skipping task as NO_REASONING found in the trace passed to the mistake generator") - continue +def mistakes_into_completed_cot_spec( + mistake: StageTwoTaskOutput, + exp_dir: str, +) -> Optional[RecomputeTaskSpec]: + generated_mistake = mistake + if generated_mistake.first_parsed_response is None or "NO_REASONING" in generated_mistake.first_parsed_response: + print("WARNING - skipping task as NO_REASONING found in the trace passed to the mistake generator") + return None + else: stage_one_output = generated_mistake.task_spec.stage_one_output config = stage_one_output.task_spec.inference_config.copy() @@ -243,7 +243,7 @@ def recomplete_cot_with_inserted_mistake( f"{exp_dir}/mistakes_stage2/s1-{stage_one_output.task_spec.formatter_name}/{stage_one_output.task_spec.task_name}/{config.model}/{CompletePartialCOT.name()}.json" ) - trace_info = generated_mistake.task_spec.trace_info + trace_info: TraceInfo | None = generated_mistake.task_spec.trace_info assert trace_info is not None trace_info.sentence_with_mistake = generated_mistake.first_parsed_response @@ -262,15 +262,35 @@ def recomplete_cot_with_inserted_mistake( trace_info=trace_info, ) - # if the mistake was the last step in the reasoning trace, then we don't need to complete the COT - # so just make a task output with no response - if trace_info.mistake_inserted_idx == len(trace_info.original_cot) - 1: - output = StageTwoTaskOutput( - task_spec=task_spec, inference_output=ModelOutput(raw_response="", parsed_response="") - ) - mistakes_inserted_at_last_position.append(output) - else: - specs.append(task_spec) + return RecomputeTaskSpec(task_spec) + + +def execute_recomputation(task_spec: RecomputeTaskSpec, caller: ModelCaller) -> list[StageTwoTaskOutput]: + # if the mistake was the last step in the reasoning trace, then we don't need to complete the COT + # so just make a task output with no response + trace_info = task_spec.trace_info + assert trace_info + if trace_info.mistake_inserted_idx == len(trace_info.original_cot) - 1: + output = StageTwoTaskOutput( + task_spec=task_spec, inference_output=ModelOutput(raw_response="", parsed_response="") + ) + return [output for _ in range(task_spec.inference_config.n)] + else: + # There should be only one response? + return task_function(task=task_spec, raise_after_retries=False, caller=caller) + + +def recomplete_cot_with_inserted_mistake( + generated_mistakes: list[StageTwoTaskOutput], + exp_dir: str, + save_completing_with_mistakes_every: int = 50, + batch: int = 10, +) -> List[StageTwoTaskOutput]: + mistakes_inserted_at_last_position: list[StageTwoTaskOutput] = [] + specs: list[StageTwoTaskSpec] = [] + + for generated_mistake in generated_mistakes: + ... print("2. Regenerating COTs with mistakes, note skipping tasks where mistake was last step in COT") outputs = run_with_caching_stage_two(save_completing_with_mistakes_every, batch, specs) @@ -278,6 +298,48 @@ def recomplete_cot_with_inserted_mistake( return outputs + mistakes_inserted_at_last_position +def single_get_best_single_answer_tasks_given_mistakes( + cot_with_mistakes_outputs: StageTwoTaskOutput, + exp_dir: str, + temperature: Optional[float] = None, +) -> Optional[StageTwoTaskSpec]: + output = cot_with_mistakes_outputs + stage_one_output = output.task_spec.stage_one_output + config = stage_one_output.task_spec.inference_config.copy() + config.max_tokens = 30 # code-davinci-002 doesn't return answer unless we set this to greater than 1 + if temperature is not None: + config.temperature = temperature + + parsed_response: str | None = output.first_parsed_response + if parsed_response is None: + print("WARNING - skipping task as parsed_response is None") + return None + + if stage_one_output.task_spec.formatter_name == FewShotCOTUnbiasedCompletionNoRoleTameraTFormatter.name(): + Formatter = FullCOTCompletionFormatter + else: + Formatter = FullCOTFormatter + + path = Path( + f"{exp_dir}/mistakes_final/s1-{stage_one_output.task_spec.formatter_name}/{stage_one_output.task_spec.task_name}/{config.model}/{Formatter.name()}.json" + ) + trace_info = output.task_spec.trace_info + assert trace_info is not None + trace_info.regenerated_cot_post_mistake = parsed_response + cot_trace_with_mistake = trace_info.get_complete_modified_cot() + + final_task = StageTwoTaskSpec( + stage_one_output=output.task_spec.stage_one_output, + inference_config=config, + formatter_name=Formatter.name(), + messages=Formatter.format_example(stage_one_output.task_spec.messages, cot_trace_with_mistake, config.model), + out_file_path=path, + n_steps_in_cot_trace=len(get_cot_steps(cot_trace_with_mistake)), + trace_info=trace_info, + ) + return final_task + + def get_best_single_answer_tasks_given_mistakes( cots_with_mistakes_outputs: list[StageTwoTaskOutput], exp_dir: str, @@ -285,42 +347,11 @@ def get_best_single_answer_tasks_given_mistakes( ) -> list[StageTwoTaskSpec]: specs: List[StageTwoTaskSpec] = [] for output in cots_with_mistakes_outputs: - stage_one_output = output.task_spec.stage_one_output - config = stage_one_output.task_spec.inference_config.copy() - config.max_tokens = 30 # code-davinci-002 doesn't return answer unless we set this to greater than 1 - if temperature is not None: - config.temperature = temperature - - parsed_response: str | None = output.first_parsed_response - if parsed_response is None: - print("WARNING - skipping task as parsed_response is None") - continue - - if stage_one_output.task_spec.formatter_name == FewShotCOTUnbiasedCompletionNoRoleTameraTFormatter.name(): - Formatter = FullCOTCompletionFormatter - else: - Formatter = FullCOTFormatter - - path = Path( - f"{exp_dir}/mistakes_final/s1-{stage_one_output.task_spec.formatter_name}/{stage_one_output.task_spec.task_name}/{config.model}/{Formatter.name()}.json" - ) - trace_info = output.task_spec.trace_info - assert trace_info is not None - trace_info.regenerated_cot_post_mistake = parsed_response - cot_trace_with_mistake = trace_info.get_complete_modified_cot() - - final_task = StageTwoTaskSpec( - stage_one_output=output.task_spec.stage_one_output, - inference_config=config, - formatter_name=Formatter.name(), - messages=Formatter.format_example( - stage_one_output.task_spec.messages, cot_trace_with_mistake, config.model - ), - out_file_path=path, - n_steps_in_cot_trace=len(get_cot_steps(cot_trace_with_mistake)), - trace_info=trace_info, + final_task = single_get_best_single_answer_tasks_given_mistakes( + cot_with_mistakes_outputs=output, exp_dir=exp_dir, temperature=temperature ) - specs.append(final_task) + if final_task: + specs.append(final_task) return specs diff --git a/scripts/stage_two_stream.py b/scripts/stage_two_stream.py index f8187e0b..0f30ef11 100644 --- a/scripts/stage_two_stream.py +++ b/scripts/stage_two_stream.py @@ -1,16 +1,19 @@ import asyncio from pathlib import Path +from grugstream import Observable + from cot_transparency.apis.base import InferenceResponse, ModelCaller from cot_transparency.data_models.config import OpenaiInferenceConfig from cot_transparency.data_models.messages import ChatMessage -from cot_transparency.data_models.models import StageTwoTaskOutput +from cot_transparency.data_models.models import StageTwoTaskSpec from cot_transparency.tasks import task_function from scripts.ignored_reasoning.stage_two import ( create_mistake_task_spec_for_stage_one, - filter_mistakes_output, + execute_recomputation, get_early_answering_tasks, - get_mistakes, + mistakes_into_completed_cot_spec, + single_get_best_single_answer_tasks_given_mistakes, ) from scripts.ignored_reasoning.stage_two_analysis import plot_early_answering_from_list from stage_one import stage_one_stream @@ -65,7 +68,7 @@ async def main(): early_answer_results = await early_answer_obs.to_list() plot_early_answering_from_list(items=early_answer_results, show_plots=True) - mistakes_obs = ( + mistakes_obs: Observable[StageTwoTaskSpec] = ( stage_one_obs.map( lambda task_output: create_mistake_task_spec_for_stage_one( stage_one_output=task_output, @@ -82,11 +85,25 @@ async def main(): ) ) .flatten_list() + # We want only not None responses + .filter(lambda task: task.first_parsed_response is not None) + # Make another spec! + .map(lambda output: mistakes_into_completed_cot_spec(mistake=output, exp_dir="not_used")) + .flatten_optional() + # Execute recomputation + .map_blocking_par(lambda spec: execute_recomputation(task_spec=spec, caller=stage_two_caller)) + .flatten_list() + # Execute final best answer + .map_blocking_par( + lambda x: single_get_best_single_answer_tasks_given_mistakes( + cot_with_mistakes_outputs=x, exp_dir="not_used" + ) + ) + .flatten_optional() ) - mistakes_results = await mistakes_obs.to_list() - filtered_results: list[StageTwoTaskOutput] = filter_mistakes_output(mistakes_results) - # todo: you need to get the + await mistakes_obs.to_list() + # todo: you need to get the stage_two_caller.save_cache() stage_one_caller.save_cache() From 0d5764eb89354e158354ffcb2d61ae1102a9d9d5 Mon Sep 17 00:00:00 2001 From: James Chua Date: Thu, 26 Oct 2023 22:50:15 +0100 Subject: [PATCH 07/28] remove spammy print --- cot_transparency/formatters/transparency/util.py | 1 - scripts/stage_two_stream.py | 7 ++++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cot_transparency/formatters/transparency/util.py b/cot_transparency/formatters/transparency/util.py index 8e40ec4c..a2ef375d 100644 --- a/cot_transparency/formatters/transparency/util.py +++ b/cot_transparency/formatters/transparency/util.py @@ -27,7 +27,6 @@ def combine_question_with_cot(question: list[ChatMessage], cot_trace: str, model output[-1].role == MessageRole.assistant_if_completion and ModelType.from_model_name(model) in [ModelType.completion, ModelType.chat_with_append_assistant] ): - print("triggering") message = f"{output[-1].content}{cot_trace.rstrip()}" output.pop() else: diff --git a/scripts/stage_two_stream.py b/scripts/stage_two_stream.py index 0f30ef11..19d7f494 100644 --- a/scripts/stage_two_stream.py +++ b/scripts/stage_two_stream.py @@ -41,7 +41,7 @@ async def main(): stage_one_obs = stage_one_stream( formatters=["ZeroShotCOTUnbiasedFormatter"], dataset="cot_testing", - example_cap=400, + example_cap=300, raise_after_retries=False, temperature=1.0, caller=stage_one_caller, @@ -65,8 +65,8 @@ async def main(): ) .flatten_list() ) - early_answer_results = await early_answer_obs.to_list() - plot_early_answering_from_list(items=early_answer_results, show_plots=True) + # early_answer_results = await early_answer_obs.to_list() + # plot_early_answering_from_list(items=early_answer_results, show_plots=True) mistakes_obs: Observable[StageTwoTaskSpec] = ( stage_one_obs.map( @@ -102,6 +102,7 @@ async def main(): .flatten_optional() ) await mistakes_obs.to_list() + print("done with mistakes") # todo: you need to get the From 5ba398e0c1d135811cf3f843519d95e2fd599568 Mon Sep 17 00:00:00 2001 From: James Chua Date: Thu, 26 Oct 2023 22:58:30 +0100 Subject: [PATCH 08/28] it kinda runs but explodes due to cot LENGTH --- scripts/ignored_reasoning/stage_two_analysis.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/scripts/ignored_reasoning/stage_two_analysis.py b/scripts/ignored_reasoning/stage_two_analysis.py index 649bcc0c..344c4ad3 100644 --- a/scripts/ignored_reasoning/stage_two_analysis.py +++ b/scripts/ignored_reasoning/stage_two_analysis.py @@ -333,7 +333,21 @@ def plot_adding_mistakes( col: str = "original_cot_trace_length", hue: str = "model", ): - df = get_data_frame_from_exp_dir(exp_dir) + ... + + + +def plot_adding_mistakes_from_list( + items: Sequence[StageTwoTaskOutput], + show_plots: bool = False, + inconsistent_only: bool = False, + aggregate_over_tasks: bool = False, + model_filter: Optional[str] = None, + length_filter: Optional[list[int]] = None, + col: str = "original_cot_trace_length", + hue: str = "model", +): + df = convert_stage2_experiment_to_dataframe(items) df = df[~df.was_truncated] df = df_filters(df, inconsistent_only, aggregate_over_tasks, model_filter, length_filter) # type: ignore From 1d098d236800b13c8cc47b7fcef59797ec037d8a Mon Sep 17 00:00:00 2001 From: James Chua Date: Thu, 26 Oct 2023 22:58:40 +0100 Subject: [PATCH 09/28] wip --- .../ignored_reasoning/stage_two_analysis.py | 17 ++++++++++++++++- scripts/stage_two_stream.py | 19 +++++++++++++------ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/scripts/ignored_reasoning/stage_two_analysis.py b/scripts/ignored_reasoning/stage_two_analysis.py index 344c4ad3..5b97dd3b 100644 --- a/scripts/ignored_reasoning/stage_two_analysis.py +++ b/scripts/ignored_reasoning/stage_two_analysis.py @@ -2,6 +2,7 @@ import fire from git import Sequence from matplotlib import pyplot as plt +from scipy.__config__ import show from analysis import get_general_metrics from cot_transparency.data_models.models import ( StageTwoTaskOutput, @@ -333,8 +334,22 @@ def plot_adding_mistakes( col: str = "original_cot_trace_length", hue: str = "model", ): - ... + items: list[StageTwoTaskOutput] = [] + loaded_dict = ExpLoader.stage_two(exp_dir, final_only=True) + for vals in loaded_dict.values(): + outputs = vals.outputs + items.extend(outputs) + return plot_adding_mistakes_from_list( + items=items, + show_plots=show_plots, + inconsistent_only=inconsistent_only, + aggregate_over_tasks=aggregate_over_tasks, + model_filter=model_filter, + length_filter=length_filter, + col=col, + hue=hue, + ) def plot_adding_mistakes_from_list( diff --git a/scripts/stage_two_stream.py b/scripts/stage_two_stream.py index 19d7f494..acf57006 100644 --- a/scripts/stage_two_stream.py +++ b/scripts/stage_two_stream.py @@ -6,7 +6,7 @@ from cot_transparency.apis.base import InferenceResponse, ModelCaller from cot_transparency.data_models.config import OpenaiInferenceConfig from cot_transparency.data_models.messages import ChatMessage -from cot_transparency.data_models.models import StageTwoTaskSpec +from cot_transparency.data_models.models import StageTwoTaskOutput, StageTwoTaskSpec from cot_transparency.tasks import task_function from scripts.ignored_reasoning.stage_two import ( create_mistake_task_spec_for_stage_one, @@ -15,7 +15,7 @@ mistakes_into_completed_cot_spec, single_get_best_single_answer_tasks_given_mistakes, ) -from scripts.ignored_reasoning.stage_two_analysis import plot_early_answering_from_list +from scripts.ignored_reasoning.stage_two_analysis import plot_adding_mistakes, plot_adding_mistakes_from_list, plot_early_answering_from_list from stage_one import stage_one_stream @@ -68,7 +68,7 @@ async def main(): # early_answer_results = await early_answer_obs.to_list() # plot_early_answering_from_list(items=early_answer_results, show_plots=True) - mistakes_obs: Observable[StageTwoTaskSpec] = ( + mistakes_obs: Observable[StageTwoTaskOutput] = ( stage_one_obs.map( lambda task_output: create_mistake_task_spec_for_stage_one( stage_one_output=task_output, @@ -93,16 +93,23 @@ async def main(): # Execute recomputation .map_blocking_par(lambda spec: execute_recomputation(task_spec=spec, caller=stage_two_caller)) .flatten_list() - # Execute final best answer - .map_blocking_par( + # final best answer task spec + .map( lambda x: single_get_best_single_answer_tasks_given_mistakes( cot_with_mistakes_outputs=x, exp_dir="not_used" ) ) .flatten_optional() + .map_blocking_par( + lambda stage_two_spec: task_function( + task=stage_two_spec, raise_after_retries=False, caller=stage_two_caller + ) + ) + .flatten_list() ) - await mistakes_obs.to_list() + mistakes_results = await mistakes_obs.to_list() print("done with mistakes") + plot_adding_mistakes_from_list(mistakes_results) # todo: you need to get the From 4a780353033a14ce3a8b1e94838cc7b6e08acd1c Mon Sep 17 00:00:00 2001 From: James Chua Date: Thu, 26 Oct 2023 23:10:29 +0100 Subject: [PATCH 10/28] add different callers --- cot_transparency/apis/base.py | 11 ++++++----- scripts/stage_two_stream.py | 33 ++++++++++++++++++++++++++------- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/cot_transparency/apis/base.py b/cot_transparency/apis/base.py index 5b15b259..397f2397 100644 --- a/cot_transparency/apis/base.py +++ b/cot_transparency/apis/base.py @@ -147,12 +147,13 @@ def call( return self.cache[key].response else: response = self.model_caller.call(messages, config) + value = CachedValue( + response=response, + messages=messages, + config=config, + ) with self.save_lock: - self.cache[key] = CachedValue( - response=response, - messages=messages, - config=config, - ) + self.cache[key] = value self.__update_counter += 1 if self.__update_counter % self.write_every_n == 0: self.save_cache() diff --git a/scripts/stage_two_stream.py b/scripts/stage_two_stream.py index acf57006..4419cff7 100644 --- a/scripts/stage_two_stream.py +++ b/scripts/stage_two_stream.py @@ -15,11 +15,30 @@ mistakes_into_completed_cot_spec, single_get_best_single_answer_tasks_given_mistakes, ) -from scripts.ignored_reasoning.stage_two_analysis import plot_adding_mistakes, plot_adding_mistakes_from_list, plot_early_answering_from_list +from scripts.ignored_reasoning.stage_two_analysis import ( + plot_adding_mistakes, + plot_adding_mistakes_from_list, + plot_early_answering_from_list, +) from stage_one import stage_one_stream -class MockCaller(ModelCaller): +class MockCOTCaller(ModelCaller): + # A caller that can call (mostly) any model + # This exists so that James can easily attach a cache to a single caller with with_file_cache + # He uses a single caller in his script because sometimes its Claude, sometimes its GPT-3.5 + def call( + self, + messages: list[ChatMessage], + config: OpenaiInferenceConfig, + ) -> InferenceResponse: + output = ( + "Let's think step by step... \nStep 1: Hmmmm\nStep 2: Ok...\nStep 3: 1+1\nTherefore the best answer is: (A)" + ) + return InferenceResponse(raw_responses=[output]) + + +class MockMistakeCaller(ModelCaller): # A caller that can call (mostly) any model # This exists so that James can easily attach a cache to a single caller with with_file_cache # He uses a single caller in his script because sometimes its Claude, sometimes its GPT-3.5 @@ -28,16 +47,17 @@ def call( messages: list[ChatMessage], config: OpenaiInferenceConfig, ) -> InferenceResponse: - output = "Let's think step by step... Therefore the best answer is: (A)" + output = "Mistake: 5+2 = 1" return InferenceResponse(raw_responses=[output]) async def main(): stage_one_cache_dir = Path("experiments/stage_one.jsonl") - stage_one_caller = MockCaller().with_file_cache(stage_one_cache_dir) + stage_one_caller = MockCOTCaller().with_file_cache(stage_one_cache_dir) stage_two_cache_dir = Path("experiments/stage_two.jsonl") - stage_two_caller = MockCaller().with_file_cache(stage_two_cache_dir) + stage_two_caller = MockCOTCaller().with_file_cache(stage_two_cache_dir) + mock_mistake_caller = MockMistakeCaller().with_file_cache("experiments/stage_two_mistakes.jsonl") stage_one_obs = stage_one_stream( formatters=["ZeroShotCOTUnbiasedFormatter"], dataset="cot_testing", @@ -81,7 +101,7 @@ async def main(): .flatten_list() .map_blocking_par( lambda stage_two_spec: task_function( - task=stage_two_spec, raise_after_retries=False, caller=stage_two_caller + task=stage_two_spec, raise_after_retries=False, caller=mock_mistake_caller ) ) .flatten_list() @@ -111,7 +131,6 @@ async def main(): print("done with mistakes") plot_adding_mistakes_from_list(mistakes_results) - # todo: you need to get the stage_two_caller.save_cache() stage_one_caller.save_cache() From e65ea0b128b24f4dd07d932611a4a127671e70cd Mon Sep 17 00:00:00 2001 From: James Chua Date: Thu, 26 Oct 2023 23:21:20 +0100 Subject: [PATCH 11/28] decrease examples --- scripts/stage_two_stream.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/scripts/stage_two_stream.py b/scripts/stage_two_stream.py index 4419cff7..a8be36e2 100644 --- a/scripts/stage_two_stream.py +++ b/scripts/stage_two_stream.py @@ -1,5 +1,6 @@ import asyncio from pathlib import Path +from anyio import CapacityLimiter from grugstream import Observable @@ -61,11 +62,11 @@ async def main(): stage_one_obs = stage_one_stream( formatters=["ZeroShotCOTUnbiasedFormatter"], dataset="cot_testing", - example_cap=300, + example_cap=100, raise_after_retries=False, temperature=1.0, caller=stage_one_caller, - ).tqdm(None) + ) early_answer_obs = ( stage_one_obs.map( @@ -87,7 +88,7 @@ async def main(): ) # early_answer_results = await early_answer_obs.to_list() # plot_early_answering_from_list(items=early_answer_results, show_plots=True) - + tp = CapacityLimiter(50) mistakes_obs: Observable[StageTwoTaskOutput] = ( stage_one_obs.map( lambda task_output: create_mistake_task_spec_for_stage_one( @@ -102,7 +103,7 @@ async def main(): .map_blocking_par( lambda stage_two_spec: task_function( task=stage_two_spec, raise_after_retries=False, caller=mock_mistake_caller - ) + ),max_par=tp ) .flatten_list() # We want only not None responses @@ -111,7 +112,7 @@ async def main(): .map(lambda output: mistakes_into_completed_cot_spec(mistake=output, exp_dir="not_used")) .flatten_optional() # Execute recomputation - .map_blocking_par(lambda spec: execute_recomputation(task_spec=spec, caller=stage_two_caller)) + .map_blocking_par(lambda spec: execute_recomputation(task_spec=spec, caller=stage_two_caller), max_par=tp) .flatten_list() # final best answer task spec .map( @@ -123,14 +124,17 @@ async def main(): .map_blocking_par( lambda stage_two_spec: task_function( task=stage_two_spec, raise_after_retries=False, caller=stage_two_caller - ) + ), + max_par=tp ) .flatten_list() + .tqdm(None) ) mistakes_results = await mistakes_obs.to_list() print("done with mistakes") plot_adding_mistakes_from_list(mistakes_results) + # todo: you need to get the stage_two_caller.save_cache() stage_one_caller.save_cache() From 7c183c42d6ad19ef0ef2ddb1ceb418c44a1e6249 Mon Sep 17 00:00:00 2001 From: James Chua Date: Fri, 27 Oct 2023 00:22:02 +0100 Subject: [PATCH 12/28] it works!!! --- .../formatters/transparency/util.py | 11 +++- scripts/ignored_reasoning/stage_two.py | 2 +- scripts/stage_two_stream.py | 66 +++++++++++++++---- 3 files changed, 62 insertions(+), 17 deletions(-) diff --git a/cot_transparency/formatters/transparency/util.py b/cot_transparency/formatters/transparency/util.py index a2ef375d..eb560f29 100644 --- a/cot_transparency/formatters/transparency/util.py +++ b/cot_transparency/formatters/transparency/util.py @@ -1,3 +1,4 @@ +from tkinter import NO from typing import Optional, Type, Self from cot_transparency.data_models.example_base import DataExampleBase from cot_transparency.data_models.messages import MessageRole, StrictMessageRole @@ -79,7 +80,7 @@ def format_example(question: list[ChatMessage], cot_trace: str, model: str) -> l role=MessageRole.user if should_use_roles else MessageRole.none, content=GIVEN_ALL_OF_THE_ABOVE, ) - ) + ) output.append( ChatMessage( role=MessageRole.assistant if should_use_roles else MessageRole.none, @@ -94,9 +95,13 @@ def parse_answer(response: str, question: DataExampleBase, model: Optional[str] assert model is not None match ModelType.from_model_name(model): case ModelType.chat: - return extract_answer(response, question) + ans= extract_answer(response, question) + case ModelType.completion | ModelType.chat_with_append_assistant: - return extract_answer_non_cot(response) + ans = extract_answer_non_cot(response) + if ans is None: + print("breakpoint") + return ans SINGLE_MOST_LIKELY_ANSWER_COMPLETION = "Given all of the above the single most likely answer is: (" diff --git a/scripts/ignored_reasoning/stage_two.py b/scripts/ignored_reasoning/stage_two.py index 2b333ba6..a4bca1db 100644 --- a/scripts/ignored_reasoning/stage_two.py +++ b/scripts/ignored_reasoning/stage_two.py @@ -1,5 +1,6 @@ import random from pathlib import Path +from tkinter import NO from typing import List, Literal, NewType, Optional, Type import fire @@ -163,7 +164,6 @@ def create_mistake_task_spec_for_stage_one( out.append(task_spec) return out - def get_mistakes( stage_one_outputs: list[TaskOutput], exp_dir: str, diff --git a/scripts/stage_two_stream.py b/scripts/stage_two_stream.py index a8be36e2..9b2f7934 100644 --- a/scripts/stage_two_stream.py +++ b/scripts/stage_two_stream.py @@ -3,6 +3,7 @@ from anyio import CapacityLimiter from grugstream import Observable +from scipy.__config__ import show from cot_transparency.apis.base import InferenceResponse, ModelCaller from cot_transparency.data_models.config import OpenaiInferenceConfig @@ -39,6 +40,19 @@ def call( return InferenceResponse(raw_responses=[output]) +class MockFullCOTCaller(ModelCaller): + # A caller that can call (mostly) any model + # This exists so that James can easily attach a cache to a single caller with with_file_cache + # He uses a single caller in his script because sometimes its Claude, sometimes its GPT-3.5 + def call( + self, + messages: list[ChatMessage], + config: OpenaiInferenceConfig, + ) -> InferenceResponse: + output = "Therefore, the best answer is: (A)" + return InferenceResponse(raw_responses=[output]) + + class MockMistakeCaller(ModelCaller): # A caller that can call (mostly) any model # This exists so that James can easily attach a cache to a single caller with with_file_cache @@ -48,21 +62,27 @@ def call( messages: list[ChatMessage], config: OpenaiInferenceConfig, ) -> InferenceResponse: - output = "Mistake: 5+2 = 1" + any_has_normal_completion = False + # Hack to make it work for both early answering and mistakes + for msg in messages: + if "Given" in msg.content: + any_has_normal_completion = True + output = "Mistake: 5+2 = 1" if not any_has_normal_completion else "Therefore, the best answer is: (A)" return InferenceResponse(raw_responses=[output]) async def main(): stage_one_cache_dir = Path("experiments/stage_one.jsonl") - stage_one_caller = MockCOTCaller().with_file_cache(stage_one_cache_dir) + stage_one_caller = MockCOTCaller() stage_two_cache_dir = Path("experiments/stage_two.jsonl") - stage_two_caller = MockCOTCaller().with_file_cache(stage_two_cache_dir) - mock_mistake_caller = MockMistakeCaller().with_file_cache("experiments/stage_two_mistakes.jsonl") + stage_two_caller = MockCOTCaller() + mock_mistake_caller = MockMistakeCaller() + mock_final_answer_caller = MockFullCOTCaller() stage_one_obs = stage_one_stream( formatters=["ZeroShotCOTUnbiasedFormatter"], dataset="cot_testing", - example_cap=100, + example_cap=10, raise_after_retries=False, temperature=1.0, caller=stage_one_caller, @@ -91,6 +111,7 @@ async def main(): tp = CapacityLimiter(50) mistakes_obs: Observable[StageTwoTaskOutput] = ( stage_one_obs.map( + # Create mistake spec lambda task_output: create_mistake_task_spec_for_stage_one( stage_one_output=task_output, exp_dir="not_used", @@ -103,7 +124,8 @@ async def main(): .map_blocking_par( lambda stage_two_spec: task_function( task=stage_two_spec, raise_after_retries=False, caller=mock_mistake_caller - ),max_par=tp + ), + max_par=tp, ) .flatten_list() # We want only not None responses @@ -123,21 +145,39 @@ async def main(): .flatten_optional() .map_blocking_par( lambda stage_two_spec: task_function( - task=stage_two_spec, raise_after_retries=False, caller=stage_two_caller + task=stage_two_spec, raise_after_retries=False, caller=mock_final_answer_caller ), - max_par=tp + max_par=tp, ) .flatten_list() .tqdm(None) ) + baseline_no_mistakes = ( + stage_one_obs.map( + lambda stage_one_task: get_early_answering_tasks( + stage_one_output=stage_one_task, + exp_dir="not_used", + temperature=stage_one_task.task_spec.inference_config.temperature, + full_answers_only=True, + ) + ) + .flatten_list() + .map_blocking_par( + lambda stage_two_spec: task_function( + task=stage_two_spec, raise_after_retries=False, caller=mock_final_answer_caller + ), + max_par=tp, + ) + .flatten_list() + ) + mistakes_results = await mistakes_obs.to_list() + baseline_no_mistakes_results = await baseline_no_mistakes.to_list() print("done with mistakes") - plot_adding_mistakes_from_list(mistakes_results) - - # todo: you need to get the + plot_adding_mistakes_from_list(mistakes_results + baseline_no_mistakes_results, show_plots=True) - stage_two_caller.save_cache() - stage_one_caller.save_cache() + # stage_two_caller.save_cache() + # stage_one_caller.save_cache() if __name__ == "__main__": From 6b49d0732845658c4717b132b43ecca9a684f17e Mon Sep 17 00:00:00 2001 From: James Chua Date: Fri, 27 Oct 2023 00:30:41 +0100 Subject: [PATCH 13/28] save --- .../ignored_reasoning/stage_two_analysis.py | 33 +++++++++++++++++++ scripts/stage_two_stream.py | 19 +++++------ stage_one.py | 1 - 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/scripts/ignored_reasoning/stage_two_analysis.py b/scripts/ignored_reasoning/stage_two_analysis.py index 5b97dd3b..d604468c 100644 --- a/scripts/ignored_reasoning/stage_two_analysis.py +++ b/scripts/ignored_reasoning/stage_two_analysis.py @@ -416,6 +416,39 @@ def aoc_plot( if show_plots: plt.show() +def aoc_plot_from_list( + items: Sequence[StageTwoTaskOutput], + show_plots: bool = False, + inconsistent_only: bool = False, + aggregate_over_tasks: bool = False, + model_filter: Optional[str] = None, + length_filter: Optional[list[int]] = None, + hue: str = "stage_one_formatter_name", +): + df = get_data_frame_from_exp_dir_items(items) + df = df_filters(df, inconsistent_only, aggregate_over_tasks, model_filter, length_filter) + + # Mistakes AoC + df_mistakes = df[~df.was_truncated] + df_mistakes = df_mistakes.groupby("stage_one_hash").apply(check_same_answer).reset_index(drop=True) + df_mistakes = drop_not_found(df_mistakes) # type: ignore + aoc_mistakes = get_aoc(df_mistakes) + + # Early Answering AoC + df_early = df[~df.has_mistake] + df_early = df_early.groupby("stage_one_hash").apply(check_same_answer).reset_index(drop=True) + df_early = drop_not_found(df_early) # type: ignore + aoc_early = get_aoc(df_early) + + # baseline accuracies + baseline_accuracy(df, hue, "model") + + _aoc_point_plot(hue, df, aoc_mistakes, aoc_early, kind="bar") + _aoc_point_plot(hue, df, aoc_mistakes, aoc_early, kind="point") + + if show_plots: + plt.show() + def _aoc_point_plot(hue: str, df: pd.DataFrame, aoc_mistakes: pd.DataFrame, aoc_early: pd.DataFrame, kind="bar"): # two point plots side by side [mistakes, early answering, accuracy] diff --git a/scripts/stage_two_stream.py b/scripts/stage_two_stream.py index 9b2f7934..d7f1dcab 100644 --- a/scripts/stage_two_stream.py +++ b/scripts/stage_two_stream.py @@ -9,6 +9,7 @@ from cot_transparency.data_models.config import OpenaiInferenceConfig from cot_transparency.data_models.messages import ChatMessage from cot_transparency.data_models.models import StageTwoTaskOutput, StageTwoTaskSpec +from cot_transparency.formatters.transparency.s1_baselines import ZeroShotCOTUnbiasedTameraTFormatter from cot_transparency.tasks import task_function from scripts.ignored_reasoning.stage_two import ( create_mistake_task_spec_for_stage_one, @@ -18,6 +19,7 @@ single_get_best_single_answer_tasks_given_mistakes, ) from scripts.ignored_reasoning.stage_two_analysis import ( + aoc_plot_from_list, plot_adding_mistakes, plot_adding_mistakes_from_list, plot_early_answering_from_list, @@ -62,12 +64,7 @@ def call( messages: list[ChatMessage], config: OpenaiInferenceConfig, ) -> InferenceResponse: - any_has_normal_completion = False - # Hack to make it work for both early answering and mistakes - for msg in messages: - if "Given" in msg.content: - any_has_normal_completion = True - output = "Mistake: 5+2 = 1" if not any_has_normal_completion else "Therefore, the best answer is: (A)" + output = "Mistake: 5+2 = 1" return InferenceResponse(raw_responses=[output]) @@ -80,12 +77,13 @@ async def main(): mock_mistake_caller = MockMistakeCaller() mock_final_answer_caller = MockFullCOTCaller() stage_one_obs = stage_one_stream( - formatters=["ZeroShotCOTUnbiasedFormatter"], - dataset="cot_testing", + formatters=[ZeroShotCOTUnbiasedTameraTFormatter.name()], + tasks=["truthful_qa"], example_cap=10, raise_after_retries=False, temperature=1.0, caller=stage_one_caller, + batch=20, ) early_answer_obs = ( @@ -116,7 +114,7 @@ async def main(): stage_one_output=task_output, exp_dir="not_used", mistake_adding_temperature=1.0, - n_mistake_insertion_points=4, + n_mistake_insertion_points=8, mistake_adding_model="claude-instant-1", ) ) @@ -174,7 +172,8 @@ async def main(): mistakes_results = await mistakes_obs.to_list() baseline_no_mistakes_results = await baseline_no_mistakes.to_list() print("done with mistakes") - plot_adding_mistakes_from_list(mistakes_results + baseline_no_mistakes_results, show_plots=True) + aoc_plot_from_list(mistakes_results + baseline_no_mistakes_results, show_plots=True) + # plot_adding_mistakes_from_list(mistakes_results + baseline_no_mistakes_results, show_plots=True) # stage_two_caller.save_cache() # stage_one_caller.save_cache() diff --git a/stage_one.py b/stage_one.py index d5c5c189..268bf6ba 100644 --- a/stage_one.py +++ b/stage_one.py @@ -210,7 +210,6 @@ def stage_one_stream( exp_dir: Optional[str] = None, experiment_suffix: str = "", example_cap: Optional[int] = 1000000, - save_file_every: int = 50, batch: int = 20, repeats_per_question: int = 1, temperature: Optional[float] = None, From c21d99c922bcf1d9d4061aaad321fba88110fde4 Mon Sep 17 00:00:00 2001 From: James Chua Date: Fri, 27 Oct 2023 01:30:44 +0100 Subject: [PATCH 14/28] auc works better i think --- .../data_models/data/task_name_map.py | 2 + .../formatters/transparency/util.py | 15 ++++--- .../ignored_reasoning/stage_two_analysis.py | 44 ++++++++++++++++++- scripts/stage_two_stream.py | 29 +++++++++--- stage_one.py | 3 ++ 5 files changed, 78 insertions(+), 15 deletions(-) diff --git a/cot_transparency/data_models/data/task_name_map.py b/cot_transparency/data_models/data/task_name_map.py index 47e9c4b2..8a70751c 100644 --- a/cot_transparency/data_models/data/task_name_map.py +++ b/cot_transparency/data_models/data/task_name_map.py @@ -38,6 +38,8 @@ def task_name_to_data_example(task_name: str) -> Type[DataExampleBase]: return ArcExample elif task_name == "truthful_qa": return TruthfulQAExample + elif task_name == "truthful_qa_fake_answer_a": + return TruthfulQAExample elif task_name == "openbook_qa": return ArcExample elif task_name == "openbook_qa_train": diff --git a/cot_transparency/formatters/transparency/util.py b/cot_transparency/formatters/transparency/util.py index eb560f29..09991851 100644 --- a/cot_transparency/formatters/transparency/util.py +++ b/cot_transparency/formatters/transparency/util.py @@ -93,14 +93,15 @@ def format_example(question: list[ChatMessage], cot_trace: str, model: str) -> l @staticmethod def parse_answer(response: str, question: DataExampleBase, model: Optional[str] = None) -> Optional[str]: assert model is not None - match ModelType.from_model_name(model): - case ModelType.chat: - ans= extract_answer(response, question) + ans= extract_answer(response, question) + # match ModelType.from_model_name(model): + # case ModelType.chat: + # ans= extract_answer(response, question) - case ModelType.completion | ModelType.chat_with_append_assistant: - ans = extract_answer_non_cot(response) - if ans is None: - print("breakpoint") + # case ModelType.completion | ModelType.chat_with_append_assistant: + # ans = extract_answer_non_cot(response) + # if ans is None: + # print("breakpoint") return ans diff --git a/scripts/ignored_reasoning/stage_two_analysis.py b/scripts/ignored_reasoning/stage_two_analysis.py index d604468c..dee1f2f7 100644 --- a/scripts/ignored_reasoning/stage_two_analysis.py +++ b/scripts/ignored_reasoning/stage_two_analysis.py @@ -1,8 +1,11 @@ from typing import Optional import fire +import scipy from git import Sequence from matplotlib import pyplot as plt from scipy.__config__ import show +from slist import Slist + from analysis import get_general_metrics from cot_transparency.data_models.models import ( StageTwoTaskOutput, @@ -50,8 +53,21 @@ def get_aoc(df: pd.DataFrame, x="cot_trace_length") -> pd.DataFrame: def get_auc(group: pd.DataFrame) -> float: assert group["original_cot_trace_length"].nunique() == 1 + # james: this seems wrong? Like all the areas are originally 1, but they get normalized to <= 1? proportion_of_cot = group[x] / max(group[x]) - auc = np.trapz(group[0], x=proportion_of_cot) + # this sums to 1, but we need to renormalise to make it start from 0 and end at 1 to have a proper AUC + """ + X_std = (X - X.min(axis=0)) / (X.max(axis=0) - X.min(axis=0)) + X_scaled = X_std * (max - min) + min + """ + min_ = min(proportion_of_cot) + max_ = max(proportion_of_cot) + try: + proportion_scaled = proportion_of_cot.apply(lambda x: (x - min_) / (max_ - min_)) + except Exception as e: + print("here") + raise e + auc = np.trapz(group[0], x=proportion_scaled) return auc groups.pop(groups.index(x)) @@ -158,6 +174,31 @@ def plot_historgram_of_lengths( plt.show() +def plot_histogram_from_list( + items: Sequence[StageTwoTaskOutput], +): + df = get_data_frame_from_exp_dir_items(items) + + hue = "task_name" + x = "CoT Length" + col = "model" + y = "Counts" + + # rename "original_cot_trace_length" to "CoT Length" + df = df.rename(columns={"original_cot_trace_length": x}) + + # for histogram we want the counts of the original_cot_trace_length + # filter on the unique stage_one_hash + counts = df.groupby([hue, col, x]).stage_one_hash.nunique().reset_index() + counts = counts.rename(columns={"stage_one_hash": y}) + + # facet plot of the proportion of the trace, break down by original_cot_trace_length + g = sns.FacetGrid(counts, col=col, col_wrap=2, legend_out=True) + g.map_dataframe(sns.barplot, x=x, y=y, hue=hue) + g.add_legend() + plt.show() + + def df_filters( df: pd.DataFrame, inconsistent_only: bool, @@ -416,6 +457,7 @@ def aoc_plot( if show_plots: plt.show() + def aoc_plot_from_list( items: Sequence[StageTwoTaskOutput], show_plots: bool = False, diff --git a/scripts/stage_two_stream.py b/scripts/stage_two_stream.py index d7f1dcab..8fd94fb6 100644 --- a/scripts/stage_two_stream.py +++ b/scripts/stage_two_stream.py @@ -23,6 +23,7 @@ plot_adding_mistakes, plot_adding_mistakes_from_list, plot_early_answering_from_list, + plot_histogram_from_list, ) from stage_one import stage_one_stream @@ -41,7 +42,7 @@ def call( ) return InferenceResponse(raw_responses=[output]) - + class MockFullCOTCaller(ModelCaller): # A caller that can call (mostly) any model # This exists so that James can easily attach a cache to a single caller with with_file_cache @@ -51,7 +52,17 @@ def call( messages: list[ChatMessage], config: OpenaiInferenceConfig, ) -> InferenceResponse: - output = "Therefore, the best answer is: (A)" + # Make gpt-3.5-turbo give the correct answer (B) if there are mistakes + has_mistakes = False + + for m in messages: + if "mistake" in m.content: + has_mistakes = True + is_gpt = "gpt-3.5" in config.model + if has_mistakes and is_gpt: + output = "Therefore, the best answer is: (B)" + else: + output = "Therefore, the best answer is: (A)" return InferenceResponse(raw_responses=[output]) @@ -64,7 +75,7 @@ def call( messages: list[ChatMessage], config: OpenaiInferenceConfig, ) -> InferenceResponse: - output = "Mistake: 5+2 = 1" + output = "mistake: 5+2 = 1" return InferenceResponse(raw_responses=[output]) @@ -78,12 +89,14 @@ async def main(): mock_final_answer_caller = MockFullCOTCaller() stage_one_obs = stage_one_stream( formatters=[ZeroShotCOTUnbiasedTameraTFormatter.name()], - tasks=["truthful_qa"], - example_cap=10, + # hacked truthful_qa to have correct labels as A + tasks=["truthful_qa_fake_answer_a"], + example_cap=100, raise_after_retries=False, temperature=1.0, caller=stage_one_caller, batch=20, + models=["claude-2", "gpt-3.5-turbo"] ) early_answer_obs = ( @@ -114,7 +127,7 @@ async def main(): stage_one_output=task_output, exp_dir="not_used", mistake_adding_temperature=1.0, - n_mistake_insertion_points=8, + n_mistake_insertion_points=16, mistake_adding_model="claude-instant-1", ) ) @@ -171,8 +184,10 @@ async def main(): mistakes_results = await mistakes_obs.to_list() baseline_no_mistakes_results = await baseline_no_mistakes.to_list() + all_ = mistakes_results + baseline_no_mistakes_results print("done with mistakes") - aoc_plot_from_list(mistakes_results + baseline_no_mistakes_results, show_plots=True) + # plot_histogram_from_list(all_) + aoc_plot_from_list(all_, show_plots=True) # plot_adding_mistakes_from_list(mistakes_results + baseline_no_mistakes_results, show_plots=True) # stage_two_caller.save_cache() diff --git a/stage_one.py b/stage_one.py index 268bf6ba..a4e1920a 100644 --- a/stage_one.py +++ b/stage_one.py @@ -85,6 +85,7 @@ "john_math": ["john_level_1", "john_level_2", "john_level_3", "john_level_4", "john_level_5"], "mmlu": mmlu.MMLU_SUPERCATEGORIES, "karina": ["karina_hallucination"], + "sannity_check": ["truthful_qa_fake_answer_a"], } @@ -163,6 +164,8 @@ def get_list_of_examples( data = arc.arc_challenge_test() elif task == "truthful_qa": data = truthful_qa.eval() + elif task == "truthful_qa_fake_answer_a": + data = truthful_qa.eval().map(lambda x: x.model_copy(update={"correct_ans_letter": "A"})) elif task == "logiqa": data = logiqa.eval() elif task == "mmlu": From 8d7fc4494570190a4f200b179ab7d007ba8382e7 Mon Sep 17 00:00:00 2001 From: James Chua Date: Sat, 28 Oct 2023 01:09:20 +0800 Subject: [PATCH 15/28] wip --- cot_transparency/data_models/models.py | 3 ++- scripts/ignored_reasoning/stage_two.py | 2 +- scripts/stage_two_stream.py | 18 +++++++++++------- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/cot_transparency/data_models/models.py b/cot_transparency/data_models/models.py index 6d346ad1..67090ef2 100644 --- a/cot_transparency/data_models/models.py +++ b/cot_transparency/data_models/models.py @@ -4,7 +4,7 @@ from pathlib import Path -from pydantic import BaseModel, Field, AliasChoices +from pydantic import BaseModel, Field, AliasChoices, ConfigDict from typing import Optional, Any, Type from cot_transparency.data_models.config import OpenaiInferenceConfig @@ -111,6 +111,7 @@ class TaskOutput(BaseTaskOuput): task_spec: TaskSpec # type: ignore[reportIncompatibleVariableOverride] inference_output: ModelOutput = Field(validation_alias=AliasChoices("inference_output", "model_output")) response_idx: int = 0 + model_config = ConfigDict(frozen=True) @property def bias_on_wrong_answer(self) -> bool: diff --git a/scripts/ignored_reasoning/stage_two.py b/scripts/ignored_reasoning/stage_two.py index a4bca1db..841d3923 100644 --- a/scripts/ignored_reasoning/stage_two.py +++ b/scripts/ignored_reasoning/stage_two.py @@ -243,7 +243,7 @@ def mistakes_into_completed_cot_spec( f"{exp_dir}/mistakes_stage2/s1-{stage_one_output.task_spec.formatter_name}/{stage_one_output.task_spec.task_name}/{config.model}/{CompletePartialCOT.name()}.json" ) - trace_info: TraceInfo | None = generated_mistake.task_spec.trace_info + trace_info: TraceInfo | None = generated_mistake.task_spec.trace_info.model_copy() assert trace_info is not None trace_info.sentence_with_mistake = generated_mistake.first_parsed_response diff --git a/scripts/stage_two_stream.py b/scripts/stage_two_stream.py index 8fd94fb6..6bb39b07 100644 --- a/scripts/stage_two_stream.py +++ b/scripts/stage_two_stream.py @@ -42,7 +42,7 @@ def call( ) return InferenceResponse(raw_responses=[output]) - + class MockFullCOTCaller(ModelCaller): # A caller that can call (mostly) any model # This exists so that James can easily attach a cache to a single caller with with_file_cache @@ -56,7 +56,7 @@ def call( has_mistakes = False for m in messages: - if "mistake" in m.content: + if "" in m.content: has_mistakes = True is_gpt = "gpt-3.5" in config.model if has_mistakes and is_gpt: @@ -75,7 +75,7 @@ def call( messages: list[ChatMessage], config: OpenaiInferenceConfig, ) -> InferenceResponse: - output = "mistake: 5+2 = 1" + output = ": 5+2 = 1" return InferenceResponse(raw_responses=[output]) @@ -96,7 +96,7 @@ async def main(): temperature=1.0, caller=stage_one_caller, batch=20, - models=["claude-2", "gpt-3.5-turbo"] + models=["gpt-3.5-turbo"], ) early_answer_obs = ( @@ -132,6 +132,7 @@ async def main(): ) ) .flatten_list() + # Call the mistake making model .map_blocking_par( lambda stage_two_spec: task_function( task=stage_two_spec, raise_after_retries=False, caller=mock_mistake_caller @@ -141,7 +142,7 @@ async def main(): .flatten_list() # We want only not None responses .filter(lambda task: task.first_parsed_response is not None) - # Make another spec! + # # Make another spec! .map(lambda output: mistakes_into_completed_cot_spec(mistake=output, exp_dir="not_used")) .flatten_optional() # Execute recomputation @@ -151,7 +152,7 @@ async def main(): .map( lambda x: single_get_best_single_answer_tasks_given_mistakes( cot_with_mistakes_outputs=x, exp_dir="not_used" - ) + ) ## prev up to here ) .flatten_optional() .map_blocking_par( @@ -163,6 +164,7 @@ async def main(): .flatten_list() .tqdm(None) ) + baseline_no_mistakes = ( stage_one_obs.map( lambda stage_one_task: get_early_answering_tasks( @@ -183,7 +185,9 @@ async def main(): ) mistakes_results = await mistakes_obs.to_list() - baseline_no_mistakes_results = await baseline_no_mistakes.to_list() + # bug: somehow this has msitakes?? + baseline_no_mistakes_results = await baseline_no_mistakes.to_slist() + print(f"length baseline_no_mistakes_results { baseline_no_mistakes_results.length}") all_ = mistakes_results + baseline_no_mistakes_results print("done with mistakes") # plot_histogram_from_list(all_) From 5fb1532cb065a71bfdea9e767d9e74f458bf8017 Mon Sep 17 00:00:00 2001 From: James Chua Date: Sat, 28 Oct 2023 01:15:47 +0800 Subject: [PATCH 16/28] it works! --- scripts/stage_two_stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/stage_two_stream.py b/scripts/stage_two_stream.py index 9d0b8df1..b7e42d95 100644 --- a/scripts/stage_two_stream.py +++ b/scripts/stage_two_stream.py @@ -93,7 +93,7 @@ async def main(): temperature=1.0, caller=stage_one_caller, batch=20, - models=["gpt-3.5-turbo"], + models=["gpt-3.5-turbo", "claude-2"], ) ( From fb992d9423d9a21a7463c1edcd47835e63d9de6b Mon Sep 17 00:00:00 2001 From: James Chua Date: Sat, 28 Oct 2023 07:43:43 +0800 Subject: [PATCH 17/28] wip --- .../formatters/transparency/util.py | 15 +++---- scripts/stage_two_stream.py | 39 ++++++++++--------- 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/cot_transparency/formatters/transparency/util.py b/cot_transparency/formatters/transparency/util.py index 9bb767bc..fa7542c1 100644 --- a/cot_transparency/formatters/transparency/util.py +++ b/cot_transparency/formatters/transparency/util.py @@ -98,16 +98,11 @@ def format_example(question: Sequence[ChatMessage], cot_trace: str, model: str) @staticmethod def parse_answer(response: str, question: DataExampleBase, model: str | None = None) -> str | None: assert model is not None - ans = extract_answer(response, question) - # match ModelType.from_model_name(model): - # case ModelType.chat: - # ans= extract_answer(response, question) - - # case ModelType.completion | ModelType.chat_with_append_assistant: - # ans = extract_answer_non_cot(response) - # if ans is None: - # print("breakpoint") - return ans + match ModelType.from_model_name(model): + case ModelType.chat: + return extract_answer(response, question) + case ModelType.completion | ModelType.chat_with_append_assistant: + return extract_answer_non_cot(response) SINGLE_MOST_LIKELY_ANSWER_COMPLETION = "Given all of the above the single most likely answer is: (" diff --git a/scripts/stage_two_stream.py b/scripts/stage_two_stream.py index b7e42d95..7b817b2e 100644 --- a/scripts/stage_two_stream.py +++ b/scripts/stage_two_stream.py @@ -6,6 +6,7 @@ from grugstream import Observable +from cot_transparency.apis import UniversalCaller from cot_transparency.apis.base import InferenceResponse, ModelCaller from cot_transparency.data_models.config import OpenaiInferenceConfig from cot_transparency.data_models.messages import ChatMessage @@ -77,26 +78,26 @@ def call( async def main(): - Path("experiments/stage_one.jsonl") + stage_one_path = Path("experiments/stream_mistakes/stage_one.jsonl") + stage_one_caller = UniversalCaller().with_file_cache(stage_one_path) - stage_one_caller = MockCOTCaller() - Path("experiments/stage_two.jsonl") - stage_two_caller = MockCOTCaller() - mock_mistake_caller = MockMistakeCaller() - mock_final_answer_caller = MockFullCOTCaller() + recompute_cot_caller = UniversalCaller().with_file_cache("experiments/stream_mistakes/recompute_cot.jsonl") + add_mistake_caller = UniversalCaller().with_file_cache("experiments/stream_mistakes/add_mistakes.jsonl") + final_answer_caller = UniversalCaller().with_file_cache("experiments/stream_mistakes/final_answer.jsonl") + n_mistake_insertion_points = 16 stage_one_obs = stage_one_stream( formatters=[ZeroShotCOTUnbiasedTameraTFormatter.name()], # hacked truthful_qa to have correct labels as A - tasks=["truthful_qa_fake_answer_a"], - example_cap=100, + tasks=["truthful_qa"], + example_cap=200, raise_after_retries=False, temperature=1.0, caller=stage_one_caller, batch=20, - models=["gpt-3.5-turbo", "claude-2"], + models=["gpt-3.5-turbo", "claude-2", "ft:gpt-3.5-turbo-0613:academicsnyuperez::8A6Ymjb2"], ) - ( + early_answer_obs = ( stage_one_obs.map( lambda task_output: get_early_answering_tasks( stage_one_output=task_output, @@ -109,7 +110,7 @@ async def main(): .flatten_list() .map_blocking_par( lambda stage_two_spec: task_function( - task=stage_two_spec, raise_after_retries=False, caller=stage_two_caller + task=stage_two_spec, raise_after_retries=False, caller=recompute_cot_caller ) ) .flatten_list() @@ -124,7 +125,7 @@ async def main(): stage_one_output=task_output, exp_dir="not_used", mistake_adding_temperature=1.0, - n_mistake_insertion_points=16, + n_mistake_insertion_points=n_mistake_insertion_points, mistake_adding_model="claude-instant-1", ) ) @@ -132,7 +133,7 @@ async def main(): # Call the mistake making model .map_blocking_par( lambda stage_two_spec: task_function( - task=stage_two_spec, raise_after_retries=False, caller=mock_mistake_caller + task=stage_two_spec, raise_after_retries=False, caller=add_mistake_caller ), max_par=tp, ) @@ -143,7 +144,7 @@ async def main(): .map(lambda output: mistakes_into_completed_cot_spec(mistake=output, exp_dir="not_used")) .flatten_optional() # Execute recomputation - .map_blocking_par(lambda spec: execute_recomputation(task_spec=spec, caller=stage_two_caller), max_par=tp) + .map_blocking_par(lambda spec: execute_recomputation(task_spec=spec, caller=recompute_cot_caller), max_par=tp) .flatten_list() # final best answer task spec .map( @@ -154,7 +155,7 @@ async def main(): .flatten_optional() .map_blocking_par( lambda stage_two_spec: task_function( - task=stage_two_spec, raise_after_retries=False, caller=mock_final_answer_caller + task=stage_two_spec, raise_after_retries=False, caller=final_answer_caller ), max_par=tp, ) @@ -174,7 +175,7 @@ async def main(): .flatten_list() .map_blocking_par( lambda stage_two_spec: task_function( - task=stage_two_spec, raise_after_retries=False, caller=mock_final_answer_caller + task=stage_two_spec, raise_after_retries=False, caller=final_answer_caller ), max_par=tp, ) @@ -191,8 +192,10 @@ async def main(): aoc_plot_from_list(all_, show_plots=True) # plot_adding_mistakes_from_list(mistakes_results + baseline_no_mistakes_results, show_plots=True) - # stage_two_caller.save_cache() - # stage_one_caller.save_cache() + stage_one_caller.save_cache() + recompute_cot_caller.save_cache() + add_mistake_caller.save_cache() + final_answer_caller.save_cache() if __name__ == "__main__": From 261322e68bf0569fcf2740a9cf2f49f85636c829 Mon Sep 17 00:00:00 2001 From: James Chua Date: Sun, 29 Oct 2023 15:35:37 +0800 Subject: [PATCH 18/28] add separate files --- .../streaming/stage_one_stream.py | 65 ++++++++ cot_transparency/tasks.py | 18 +- scripts/stage_two_stream.py | 4 +- stage_one.py | 154 +++--------------- 4 files changed, 89 insertions(+), 152 deletions(-) create mode 100644 cot_transparency/streaming/stage_one_stream.py diff --git a/cot_transparency/streaming/stage_one_stream.py b/cot_transparency/streaming/stage_one_stream.py new file mode 100644 index 00000000..82169d8f --- /dev/null +++ b/cot_transparency/streaming/stage_one_stream.py @@ -0,0 +1,65 @@ +from typing import Sequence, Optional, Literal + +from grugstream import Observable + +from cot_transparency.apis import ModelCaller, UniversalCaller +from cot_transparency.data_models.models import TaskOutput +from cot_transparency.formatters.core.sycophancy import ZeroShotCOTSycophancyFormatter +from cot_transparency.formatters.core.unbiased import ZeroShotCOTUnbiasedFormatter +from cot_transparency.tasks import task_function +from stage_one import create_stage_one_task_specs + + +def stage_one_stream( + tasks: Sequence[str] = [], + dataset: Optional[str] = None, + models: Sequence[str] = ["gpt-3.5-turbo", "gpt-4"], + formatters: Sequence[str] = [ZeroShotCOTSycophancyFormatter.name(), ZeroShotCOTUnbiasedFormatter.name()], + # Pass in a list of interventions to run, indicate None to run no intervention as well + interventions: Sequence[str | None] = [], + exp_dir: Optional[str] = None, + experiment_suffix: str = "", + example_cap: Optional[int] = 1000000, + batch: int = 20, + repeats_per_question: int = 1, + temperature: Optional[float] = None, + raise_after_retries: bool = True, + raise_on: Literal["all", "any"] = "all", + num_retries: int = 10, + max_tokens: Optional[int] = None, + n_responses_per_request: Optional[int] = None, + caller: ModelCaller = UniversalCaller(), +) -> Observable[TaskOutput]: + """A versino of stage_one.py, but streaming + Note that this doesn't manage any cache for you""" + tasks_to_run = create_stage_one_task_specs( + tasks=tasks, + dataset=dataset, + models=models, + formatters=formatters, + interventions=interventions, + exp_dir=exp_dir, + experiment_suffix=experiment_suffix, + example_cap=example_cap, + batch=batch, + repeats_per_question=repeats_per_question, + temperature=temperature, + raise_after_retries=raise_after_retries, + max_tokens=max_tokens, + n_responses_per_request=n_responses_per_request, + ) + + return ( + Observable.from_iterable(tasks_to_run) + .map_blocking_par( + lambda task_spec: task_function( + task=task_spec, + raise_after_retries=raise_after_retries, + raise_on=raise_on, + caller=caller, + num_retries=num_retries, + ), + max_par=batch, + ) + .flatten_list() + ) diff --git a/cot_transparency/tasks.py b/cot_transparency/tasks.py index 3c07d0c7..0e466a85 100644 --- a/cot_transparency/tasks.py +++ b/cot_transparency/tasks.py @@ -1,10 +1,8 @@ import random from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path -from typing import Literal -from typing import Union +from typing import Literal, Sequence, overload, Union -from altair import overload from pydantic import BaseModel from retry import retry from tqdm import tqdm @@ -167,17 +165,7 @@ def call_model_and_catch( return e.model_outputs -@overload -def task_function( - task: StageTwoTaskSpec, - raise_after_retries: bool, - caller: ModelCaller = ..., - raise_on: Union[Literal["all"], Literal["any"]] = "any", - num_retries: int = 10, -) -> list[StageTwoTaskOutput]: - ... - - +# TODO: Just have a separate function for stage 2? Otherwise readability is bad @overload def task_function( task: TaskSpec, @@ -258,7 +246,7 @@ def task_function( def run_with_caching( save_every: int, batch: int, - task_to_run: list[TaskSpec] | list[StageTwoTaskSpec], + task_to_run: Sequence[TaskSpec] | Sequence[StageTwoTaskSpec], raise_after_retries: bool = False, raise_on: Literal["all"] | Literal["any"] = "any", num_retries: int = 10, diff --git a/scripts/stage_two_stream.py b/scripts/stage_two_stream.py index 7b817b2e..5e26f027 100644 --- a/scripts/stage_two_stream.py +++ b/scripts/stage_two_stream.py @@ -23,7 +23,7 @@ from scripts.ignored_reasoning.stage_two_analysis import ( aoc_plot_from_list, ) -from stage_one import stage_one_stream +from cot_transparency.streaming.stage_one_stream import stage_one_stream class MockCOTCaller(ModelCaller): @@ -97,7 +97,7 @@ async def main(): models=["gpt-3.5-turbo", "claude-2", "ft:gpt-3.5-turbo-0613:academicsnyuperez::8A6Ymjb2"], ) - early_answer_obs = ( + ( stage_one_obs.map( lambda task_output: get_early_answering_tasks( stage_one_output=task_output, diff --git a/stage_one.py b/stage_one.py index 0f37b57e..af71977e 100644 --- a/stage_one.py +++ b/stage_one.py @@ -4,11 +4,8 @@ from typing import Literal, Optional, Sequence, Type import fire -from grugstream import Observable from slist import Slist -from cot_transparency.apis import UniversalCaller -from cot_transparency.apis.base import ModelCaller from cot_transparency.apis.openai.set_key import set_keys_from_env from cot_transparency.data_models.config import config_from_default @@ -42,7 +39,7 @@ get_anthropic_pol, ) from cot_transparency.data_models.example_base import DataExampleBase -from cot_transparency.data_models.models import TaskOutput, TaskSpec +from cot_transparency.data_models.models import TaskSpec from cot_transparency.formatters import ( ZeroShotCOTSycophancyFormatter, ZeroShotCOTUnbiasedFormatter, @@ -58,7 +55,7 @@ ) from cot_transparency.formatters.wildcard import match_wildcard_formatters from cot_transparency.json_utils.read_write import read_jsonl_file_into_basemodel -from cot_transparency.tasks import TaskSetting, run_with_caching, task_function +from cot_transparency.tasks import TaskSetting, run_with_caching from cot_transparency.util import get_exp_dir_name # ok to train on the test set since we test on completely different datasets @@ -223,7 +220,7 @@ def get_list_of_examples( return data # type: ignore -def stage_one_stream( +def create_stage_one_task_specs( tasks: Sequence[str] = [], dataset: Optional[str] = None, models: Sequence[str] = ["gpt-3.5-turbo", "gpt-4"], @@ -237,13 +234,9 @@ def stage_one_stream( repeats_per_question: int = 1, temperature: Optional[float] = None, raise_after_retries: bool = True, - raise_on: Literal["all", "any"] = "all", - num_retries: int = 10, max_tokens: Optional[int] = None, n_responses_per_request: Optional[int] = None, - retry_answers_with_none: bool = False, - caller: ModelCaller = UniversalCaller(), -) -> Observable[TaskOutput]: +) -> Sequence[TaskSpec]: if dataset is not None: # we are using a dataset assert len(tasks) == 0, "You have defined a dataset and a task, you can only define one" @@ -354,19 +347,8 @@ def stage_one_stream( intervention_name=setting.intervention.name() if setting.intervention else None, ) tasks_to_run.append(task_spec) - return ( - Observable.from_iterable(tasks_to_run) - .map_blocking_par( - lambda task_spec: task_function( - task=task_spec, - raise_after_retries=raise_after_retries, - raise_on=raise_on, - caller=caller, - ), - max_par=batch, - ) - .flatten_list() - ) + + return tasks_to_run def main( @@ -392,122 +374,24 @@ def main( max_tokens: Optional[int] = None, n_responses_per_request: Optional[int] = None, retry_answers_with_none: bool = False, - caller: ModelCaller = UniversalCaller(), ): - if dataset is not None: - # we are using a dataset - assert len(tasks) == 0, "You have defined a dataset and a task, you can only define one" - tasks = TASK_LIST[dataset] - else: - assert tasks, "You must define a task or a dataset" - - for model in models: - if "llama" in model.lower(): - assert batch == 1, "Llama only supports batch size of 1" - print("Number of models to run:", len(models)) - - # match formatter name wildcard - formatters = match_wildcard_formatters(formatters) - - assert len(formatters) > 0, "You must define at least one formatter" - - tasks = validate_tasks(tasks) - print("Number of tasks to run:", len(tasks)) - validated_formatters = get_valid_stage1_formatters(formatters) - print("Number of formatters to run:", len(validated_formatters)) - validated_interventions = get_valid_stage1_interventions(interventions) - print("Number of interventions to run:", len(validated_interventions)) - exp_dir = get_exp_dir_name(exp_dir, experiment_suffix, sub_dir="stage_one") - - task_settings: list[TaskSetting] = create_task_settings( + tasks_to_run = create_stage_one_task_specs( tasks=tasks, + dataset=dataset, models=models, - formatters=validated_formatters, - interventions=validated_interventions, + formatters=formatters, + interventions=interventions, + exp_dir=exp_dir, + experiment_suffix=experiment_suffix, + example_cap=example_cap, + batch=batch, + repeats_per_question=repeats_per_question, + temperature=temperature, + raise_after_retries=raise_after_retries, + max_tokens=max_tokens, + n_responses_per_request=n_responses_per_request, ) - tasks_to_run: list[TaskSpec] = [] - print("Number of settings to run:", len(task_settings)) - for setting in task_settings: - task = setting.task - model = setting.model - formatter = setting.formatter - # Shuffle the data BEFORE we cap it - # Pass 42 to maintain the same shuffle that we had in the past, though slist wants a string instead - data: Slist[DataExampleBase] = get_list_of_examples(task, dataset=dataset).shuffle(typing.cast(str, 42)) - out_file_path: Path = ( - Path(f"{exp_dir}/{task}/{model}/{formatter.name()}.json") - if setting.intervention is None - else Path(f"{exp_dir}/{task}/{model}/{formatter.name()}_and_{setting.intervention.name()}.json") - ) - - if example_cap: - data = data.take(example_cap) - - # Config Overrides Start ---------------------- - config = config_from_default(model).model_copy(deep=True) - if issubclass(formatter, FormattersForTransparency): - few_shot_stops = ["\n\nHuman:", "\n\nAssistant:", "\n\nQuestion:"] - if isinstance(config.stop, list): - config.stop += few_shot_stops - else: - config.stop = few_shot_stops - config.max_tokens = 300 - config.temperature = 0.8 - config.top_p = 0.95 - - # if you are using an intervention, we need to add SINGLE_SHOT_SEP to the stop list - if setting.intervention: - if isinstance(config.stop, list): - config.stop += [FEW_SHOT_STOP_TOKEN] - else: - config.stop = [FEW_SHOT_STOP_TOKEN] - - if temperature is not None: - config.temperature = temperature - - assert config.model == model - - if not formatter.is_cot: - config.max_tokens = 1 - - if max_tokens is not None: - config.max_tokens = max_tokens - - if n_responses_per_request is not None: - config.n = n_responses_per_request - - # Config Overrides End ---------------------- - - if raise_after_retries and temperature == 0: - raise ValueError("Must set --raise_after_retires=False when temperature is 0 as it will always fail") - - for item in data: - for i in range(repeats_per_question): - messages = ( - setting.intervention.intervene(question=item, formatter=formatter, model=model) - if setting.intervention - else formatter.format_example(question=item, model=model) - ) - # Save the format spec defined by the formatter - new_item: DataExampleBase = item.model_copy() - format_spec = formatter.get_data_format_spec() - new_item.data_format = format_spec - task_spec = TaskSpec( - task_name=task, - inference_config=config, - messages=messages, - out_file_path=out_file_path, - ground_truth=new_item.ground_truth, - formatter_name=formatter.name(), - task_hash=new_item.hash(), - biased_ans=new_item.biased_ans, - data_example=new_item.model_dump(), - repeat_idx=i, - intervention_name=setting.intervention.name() if setting.intervention else None, - ) - tasks_to_run.append(task_spec) - run_with_caching( save_every=save_file_every, batch=batch, From 6b4aeeb7b0168e039edf9f7ac59d03b9700d65b2 Mon Sep 17 00:00:00 2001 From: James Chua Date: Sun, 29 Oct 2023 15:37:25 +0800 Subject: [PATCH 19/28] fix --- cot_transparency/tasks.py | 8 ++++---- scripts/ignored_reasoning/stage_two.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cot_transparency/tasks.py b/cot_transparency/tasks.py index 0e466a85..e86048ba 100644 --- a/cot_transparency/tasks.py +++ b/cot_transparency/tasks.py @@ -173,18 +173,18 @@ def task_function( caller: ModelCaller = ..., raise_on: Union[Literal["all"], Literal["any"]] = "any", num_retries: int = 10, -) -> list[TaskOutput]: +) -> Sequence[TaskOutput]: ... @overload def task_function( - task: Union[TaskSpec, StageTwoTaskSpec], + task: StageTwoTaskSpec, raise_after_retries: bool, caller: ModelCaller = ..., raise_on: Union[Literal["all"], Literal["any"]] = "any", num_retries: int = 10, -) -> Union[list[TaskOutput], list[StageTwoTaskOutput]]: +) -> Sequence[StageTwoTaskOutput]: ... @@ -194,7 +194,7 @@ def task_function( caller: ModelCaller = UniversalCaller(), raise_on: Literal["all"] | Literal["any"] = "any", num_retries: int = 10, -) -> list[TaskOutput] | list[StageTwoTaskOutput]: +) -> Sequence[TaskOutput] | Sequence[StageTwoTaskOutput]: formatter = name_to_formatter(task.formatter_name) responses = ( diff --git a/scripts/ignored_reasoning/stage_two.py b/scripts/ignored_reasoning/stage_two.py index 933fa0cd..9af4f9c9 100644 --- a/scripts/ignored_reasoning/stage_two.py +++ b/scripts/ignored_reasoning/stage_two.py @@ -276,7 +276,7 @@ def mistakes_into_completed_cot_spec( return RecomputeTaskSpec(task_spec) -def execute_recomputation(task_spec: RecomputeTaskSpec, caller: ModelCaller) -> list[StageTwoTaskOutput]: +def execute_recomputation(task_spec: RecomputeTaskSpec, caller: ModelCaller) -> Sequence[StageTwoTaskOutput]: # if the mistake was the last step in the reasoning trace, then we don't need to complete the COT # so just make a task output with no response trace_info = task_spec.trace_info From 31984b279542170852796db7cacd82d9ddb78b53 Mon Sep 17 00:00:00 2001 From: James Chua Date: Sun, 29 Oct 2023 20:37:21 +0800 Subject: [PATCH 20/28] refactor --- scripts/stage_two_stream.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/scripts/stage_two_stream.py b/scripts/stage_two_stream.py index 5e26f027..8b44d8db 100644 --- a/scripts/stage_two_stream.py +++ b/scripts/stage_two_stream.py @@ -96,8 +96,10 @@ async def main(): batch=20, models=["gpt-3.5-turbo", "claude-2", "ft:gpt-3.5-turbo-0613:academicsnyuperez::8A6Ymjb2"], ) + # shared capacity limit between the stages + capacity_limit = CapacityLimiter(50) - ( + early_answer_obs = ( stage_one_obs.map( lambda task_output: get_early_answering_tasks( stage_one_output=task_output, @@ -115,9 +117,9 @@ async def main(): ) .flatten_list() ) - # early_answer_results = await early_answer_obs.to_list() - # plot_early_answering_from_list(items=early_answer_results, show_plots=True) - tp = CapacityLimiter(50) + early_answer_results = await early_answer_obs.to_list() + plot_early_answering_from_list(items=early_answer_results, show_plots=True) + mistakes_obs: Observable[StageTwoTaskOutput] = ( stage_one_obs.map( # Create mistake spec @@ -135,7 +137,7 @@ async def main(): lambda stage_two_spec: task_function( task=stage_two_spec, raise_after_retries=False, caller=add_mistake_caller ), - max_par=tp, + max_par=capacity_limit, ) .flatten_list() # We want only not None responses @@ -144,7 +146,9 @@ async def main(): .map(lambda output: mistakes_into_completed_cot_spec(mistake=output, exp_dir="not_used")) .flatten_optional() # Execute recomputation - .map_blocking_par(lambda spec: execute_recomputation(task_spec=spec, caller=recompute_cot_caller), max_par=tp) + .map_blocking_par( + lambda spec: execute_recomputation(task_spec=spec, caller=recompute_cot_caller), max_par=capacity_limit + ) .flatten_list() # final best answer task spec .map( @@ -157,7 +161,7 @@ async def main(): lambda stage_two_spec: task_function( task=stage_two_spec, raise_after_retries=False, caller=final_answer_caller ), - max_par=tp, + max_par=capacity_limit, ) .flatten_list() .tqdm(None) @@ -177,7 +181,7 @@ async def main(): lambda stage_two_spec: task_function( task=stage_two_spec, raise_after_retries=False, caller=final_answer_caller ), - max_par=tp, + max_par=capacity_limit, ) .flatten_list() ) From 4e2888517995113291e4909d57a627e719cd4ea4 Mon Sep 17 00:00:00 2001 From: James Chua Date: Sun, 29 Oct 2023 20:48:00 +0800 Subject: [PATCH 21/28] add stage two stream --- .../james_ignored_reasoning_example.py | 2 +- scripts/ignored_reasoning/stage_two_analysis.py | 16 ++++++++-------- .../{ => ignored_reasoning}/stage_two_stream.py | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) rename james_ignored_reasoning.py => scripts/ignored_reasoning/james_ignored_reasoning_example.py (93%) rename scripts/{ => ignored_reasoning}/stage_two_stream.py (99%) diff --git a/james_ignored_reasoning.py b/scripts/ignored_reasoning/james_ignored_reasoning_example.py similarity index 93% rename from james_ignored_reasoning.py rename to scripts/ignored_reasoning/james_ignored_reasoning_example.py index 4819c507..94971a27 100644 --- a/james_ignored_reasoning.py +++ b/scripts/ignored_reasoning/james_ignored_reasoning_example.py @@ -15,7 +15,7 @@ exp_dir=exp_dir, batch=20, ) - stage_two_dir = "experiments/james_ignored" + stage_two_dir = "experiments/james_ignored/stage_two" stage_two_main(input_exp_dir=exp_dir, mistake_model="claude-instant-1", exp_dir=stage_two_dir) plot_adding_mistakes(exp_dir=stage_two_dir, show_plots=True) aoc_plot(exp_dir=stage_two_dir, show_plots=True) diff --git a/scripts/ignored_reasoning/stage_two_analysis.py b/scripts/ignored_reasoning/stage_two_analysis.py index bf0ee013..36d03b3e 100644 --- a/scripts/ignored_reasoning/stage_two_analysis.py +++ b/scripts/ignored_reasoning/stage_two_analysis.py @@ -62,14 +62,14 @@ def get_auc(group: pd.DataFrame) -> float: X_std = (X - X.min(axis=0)) / (X.max(axis=0) - X.min(axis=0)) X_scaled = X_std * (max - min) + min """ - min_ = min(proportion_of_cot) - max_ = max(proportion_of_cot) - try: - proportion_scaled = proportion_of_cot.apply(lambda x: (x - min_) / (max_ - min_)) - except Exception as e: - print("here") - raise e - auc = np.trapz(group[0], x=proportion_scaled) + # min_ = min(proportion_of_cot) + # max_ = max(proportion_of_cot) + # try: + # proportion_scaled = proportion_of_cot.apply(lambda x: (x - min_) / (max_ - min_)) + # except Exception as e: + # print("here") + # raise e + auc = np.trapz(group[0], x=proportion_of_cot) return auc groups.pop(groups.index(x)) diff --git a/scripts/stage_two_stream.py b/scripts/ignored_reasoning/stage_two_stream.py similarity index 99% rename from scripts/stage_two_stream.py rename to scripts/ignored_reasoning/stage_two_stream.py index 8b44d8db..5436b6c2 100644 --- a/scripts/stage_two_stream.py +++ b/scripts/ignored_reasoning/stage_two_stream.py @@ -21,7 +21,7 @@ single_get_best_single_answer_tasks_given_mistakes, ) from scripts.ignored_reasoning.stage_two_analysis import ( - aoc_plot_from_list, + aoc_plot_from_list, plot_early_answering_from_list, ) from cot_transparency.streaming.stage_one_stream import stage_one_stream From 7fdcfe1411143872da530e97f3a168b92a2160d4 Mon Sep 17 00:00:00 2001 From: James Chua Date: Sun, 29 Oct 2023 20:53:30 +0800 Subject: [PATCH 22/28] add wip --- scripts/ignored_reasoning/stage_two.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/ignored_reasoning/stage_two.py b/scripts/ignored_reasoning/stage_two.py index 9af4f9c9..79a63f57 100644 --- a/scripts/ignored_reasoning/stage_two.py +++ b/scripts/ignored_reasoning/stage_two.py @@ -175,14 +175,14 @@ def create_mistake_task_spec_for_stage_one( def get_mistakes( - stage_one_outputs: list[TaskOutput], + stage_one_outputs: Sequence[TaskOutput], exp_dir: str, batch: int = 10, mistake_adding_model: str = "text-davinci-002", mistake_adding_temperature: float = 1.0, save_mistake_generating_file_every: int = 50, n_mistake_insertion_points: int = 3, -) -> list[StageTwoTaskOutput]: +) -> Sequence[StageTwoTaskOutput]: # mistakes we need to make call to api to generate the mistake, we may use a different model here # e.g Tamera et al use a non RLHF model to generate mistakes specs: list[StageTwoTaskSpec] = [] @@ -293,15 +293,15 @@ def execute_recomputation(task_spec: RecomputeTaskSpec, caller: ModelCaller) -> def recomplete_cot_with_inserted_mistake( - generated_mistakes: list[StageTwoTaskOutput], - exp_dir: str, + generated_mistakes: Sequence[StageTwoTaskOutput], save_completing_with_mistakes_every: int = 50, batch: int = 10, -) -> List[StageTwoTaskOutput]: +) -> Sequence[StageTwoTaskOutput]: mistakes_inserted_at_last_position: list[StageTwoTaskOutput] = [] specs: list[StageTwoTaskSpec] = [] for generated_mistake in generated_mistakes: + raise ValueError("wip oops") ... print("2. Regenerating COTs with mistakes, note skipping tasks where mistake was last step in COT") From f92add72c4d8b2d92a6cc9d525b7e9dce7218b04 Mon Sep 17 00:00:00 2001 From: James Chua Date: Mon, 30 Oct 2023 10:59:29 +0800 Subject: [PATCH 23/28] remove unneeded --- cot_transparency/data_models/data/task_name_map.py | 2 -- cot_transparency/data_models/models.py | 6 ++---- cot_transparency/streaming/stage_one_stream.py | 6 ++++-- cot_transparency/tasks.py | 2 -- stage_one.py | 3 --- 5 files changed, 6 insertions(+), 13 deletions(-) diff --git a/cot_transparency/data_models/data/task_name_map.py b/cot_transparency/data_models/data/task_name_map.py index d8222a05..29db2916 100644 --- a/cot_transparency/data_models/data/task_name_map.py +++ b/cot_transparency/data_models/data/task_name_map.py @@ -42,8 +42,6 @@ def task_name_to_data_example(task_name: str) -> Type[DataExampleBase]: return ArcExample elif task_name == "truthful_qa": return TruthfulQAExample - elif task_name == "truthful_qa_fake_answer_a": - return TruthfulQAExample elif task_name == "openbook_qa": return ArcExample elif task_name == "openbook_qa_train": diff --git a/cot_transparency/data_models/models.py b/cot_transparency/data_models/models.py index c3dc55b7..f96fc0aa 100644 --- a/cot_transparency/data_models/models.py +++ b/cot_transparency/data_models/models.py @@ -2,10 +2,9 @@ from abc import abstractmethod from pathlib import Path -from typing import Optional, Any, Type -from typing import Sequence +from typing import Any, Optional, Sequence, Type -from pydantic import BaseModel, Field, AliasChoices, ConfigDict +from pydantic import AliasChoices, BaseModel, Field from cot_transparency.data_models.config import OpenaiInferenceConfig from cot_transparency.data_models.data.task_name_map import task_name_to_data_example @@ -115,7 +114,6 @@ class TaskOutput(BaseTaskOuput): task_spec: TaskSpec # type: ignore[reportIncompatibleVariableOverride] inference_output: ModelOutput = Field(validation_alias=AliasChoices("inference_output", "model_output")) response_idx: int = 0 - model_config = ConfigDict(frozen=True) @property def bias_on_wrong_answer(self) -> bool: diff --git a/cot_transparency/streaming/stage_one_stream.py b/cot_transparency/streaming/stage_one_stream.py index 82169d8f..61276c0f 100644 --- a/cot_transparency/streaming/stage_one_stream.py +++ b/cot_transparency/streaming/stage_one_stream.py @@ -30,8 +30,10 @@ def stage_one_stream( n_responses_per_request: Optional[int] = None, caller: ModelCaller = UniversalCaller(), ) -> Observable[TaskOutput]: - """A versino of stage_one.py, but streaming - Note that this doesn't manage any cache for you""" + """A version of stage_one.py, but streaming + Note that this doesn't manage any cache for you, + so maybe you want to attach a cache to the ModellCaller with with_file_cache + """ tasks_to_run = create_stage_one_task_specs( tasks=tasks, dataset=dataset, diff --git a/cot_transparency/tasks.py b/cot_transparency/tasks.py index e86048ba..386fef53 100644 --- a/cot_transparency/tasks.py +++ b/cot_transparency/tasks.py @@ -272,10 +272,8 @@ def run_with_caching( elif isinstance(task_to_run[0], StageTwoTaskSpec): loaded_dict = get_loaded_dict_stage2(paths) - output_count = 0 for task_output in loaded_dict.values(): for output in task_output.outputs: - output_count += 1 completed_outputs[output.task_spec.uid()] = output # print number that are None diff --git a/stage_one.py b/stage_one.py index af71977e..ef3c2148 100644 --- a/stage_one.py +++ b/stage_one.py @@ -102,7 +102,6 @@ ], "mmlu": mmlu.MMLU_SUPERCATEGORIES, "karina": ["karina_hallucination"], - "sannity_check": ["truthful_qa_fake_answer_a"], } @@ -181,8 +180,6 @@ def get_list_of_examples( data = arc.arc_challenge_test() elif task == "truthful_qa": data = truthful_qa.eval() - elif task == "truthful_qa_fake_answer_a": - data = truthful_qa.eval().map(lambda x: x.model_copy(update={"correct_ans_letter": "A"})) elif task == "logiqa": data = logiqa.eval() elif task == "mmlu": From 9cdfd48bf591e8fa1839b19065d1a1228e9e25ab Mon Sep 17 00:00:00 2001 From: James Chua Date: Mon, 30 Oct 2023 11:03:52 +0800 Subject: [PATCH 24/28] remove debugging statements --- scripts/ignored_reasoning/stage_two_stream.py | 63 +++---------------- 1 file changed, 7 insertions(+), 56 deletions(-) diff --git a/scripts/ignored_reasoning/stage_two_stream.py b/scripts/ignored_reasoning/stage_two_stream.py index 5436b6c2..c07692a9 100644 --- a/scripts/ignored_reasoning/stage_two_stream.py +++ b/scripts/ignored_reasoning/stage_two_stream.py @@ -21,62 +21,14 @@ single_get_best_single_answer_tasks_given_mistakes, ) from scripts.ignored_reasoning.stage_two_analysis import ( - aoc_plot_from_list, plot_early_answering_from_list, + aoc_plot_from_list, + plot_early_answering_from_list, + plot_histogram_from_list, + plot_adding_mistakes_from_list, ) from cot_transparency.streaming.stage_one_stream import stage_one_stream -class MockCOTCaller(ModelCaller): - # A caller that can call (mostly) any model - # This exists so that James can easily attach a cache to a single caller with with_file_cache - # He uses a single caller in his script because sometimes its Claude, sometimes its GPT-3.5 - def call( - self, - messages: Sequence[ChatMessage], - config: OpenaiInferenceConfig, - ) -> InferenceResponse: - output = ( - "Let's think step by step... \nStep 1: Hmmmm\nStep 2: Ok...\nStep 3: 1+1\nTherefore the best answer is: (A)" - ) - return InferenceResponse(raw_responses=[output]) - - -class MockFullCOTCaller(ModelCaller): - # A caller that can call (mostly) any model - # This exists so that James can easily attach a cache to a single caller with with_file_cache - # He uses a single caller in his script because sometimes its Claude, sometimes its GPT-3.5 - def call( - self, - messages: Sequence[ChatMessage], - config: OpenaiInferenceConfig, - ) -> InferenceResponse: - # Make gpt-3.5-turbo give the correct answer (B) if there are mistakes - has_mistakes = False - - for m in messages: - if "" in m.content: - has_mistakes = True - is_gpt = "gpt-3.5" in config.model - if has_mistakes and is_gpt: - output = "Therefore, the best answer is: (B)" - else: - output = "Therefore, the best answer is: (A)" - return InferenceResponse(raw_responses=[output]) - - -class MockMistakeCaller(ModelCaller): - # A caller that can call (mostly) any model - # This exists so that James can easily attach a cache to a single caller with with_file_cache - # He uses a single caller in his script because sometimes its Claude, sometimes its GPT-3.5 - def call( - self, - messages: Sequence[ChatMessage], - config: OpenaiInferenceConfig, - ) -> InferenceResponse: - output = ": 5+2 = 1" - return InferenceResponse(raw_responses=[output]) - - async def main(): stage_one_path = Path("experiments/stream_mistakes/stage_one.jsonl") stage_one_caller = UniversalCaller().with_file_cache(stage_one_path) @@ -94,7 +46,7 @@ async def main(): temperature=1.0, caller=stage_one_caller, batch=20, - models=["gpt-3.5-turbo", "claude-2", "ft:gpt-3.5-turbo-0613:academicsnyuperez::8A6Ymjb2"], + models=["gpt-3.5-turbo", "claude-2"], ) # shared capacity limit between the stages capacity_limit = CapacityLimiter(50) @@ -187,14 +139,13 @@ async def main(): ) mistakes_results = await mistakes_obs.to_list() - # bug: somehow this has msitakes?? baseline_no_mistakes_results = await baseline_no_mistakes.to_slist() print(f"length baseline_no_mistakes_results { baseline_no_mistakes_results.length}") all_ = mistakes_results + baseline_no_mistakes_results print("done with mistakes") - # plot_histogram_from_list(all_) + plot_histogram_from_list(all_) aoc_plot_from_list(all_, show_plots=True) - # plot_adding_mistakes_from_list(mistakes_results + baseline_no_mistakes_results, show_plots=True) + plot_adding_mistakes_from_list(mistakes_results + baseline_no_mistakes_results, show_plots=True) stage_one_caller.save_cache() recompute_cot_caller.save_cache() From 7a6eca3b9ab8480536b3b97985f43a2d8eec9d07 Mon Sep 17 00:00:00 2001 From: James Chua Date: Mon, 30 Oct 2023 11:08:32 +0800 Subject: [PATCH 25/28] fix --- scripts/ignored_reasoning/stage_two.py | 52 ++++++++++++++++--- scripts/ignored_reasoning/stage_two_stream.py | 4 -- 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/scripts/ignored_reasoning/stage_two.py b/scripts/ignored_reasoning/stage_two.py index 79a63f57..b05adf64 100644 --- a/scripts/ignored_reasoning/stage_two.py +++ b/scripts/ignored_reasoning/stage_two.py @@ -277,10 +277,10 @@ def mistakes_into_completed_cot_spec( def execute_recomputation(task_spec: RecomputeTaskSpec, caller: ModelCaller) -> Sequence[StageTwoTaskOutput]: - # if the mistake was the last step in the reasoning trace, then we don't need to complete the COT - # so just make a task output with no response trace_info = task_spec.trace_info assert trace_info + # if the mistake was the last step in the reasoning trace, then we don't need to complete the COT + # so just make a task output with no response if trace_info.mistake_inserted_idx == len(trace_info.original_cot) - 1: output = StageTwoTaskOutput( task_spec=task_spec, @@ -288,21 +288,61 @@ def execute_recomputation(task_spec: RecomputeTaskSpec, caller: ModelCaller) -> ) return [output for _ in range(task_spec.inference_config.n)] else: - # There should be only one response? + # We need to complete the COT return task_function(task=task_spec, raise_after_retries=False, caller=caller) def recomplete_cot_with_inserted_mistake( generated_mistakes: Sequence[StageTwoTaskOutput], + exp_dir: str, save_completing_with_mistakes_every: int = 50, batch: int = 10, -) -> Sequence[StageTwoTaskOutput]: +) -> List[StageTwoTaskOutput]: + """Note: This is an old function that isn't used anymore by the stage_two_stream.py script""" mistakes_inserted_at_last_position: list[StageTwoTaskOutput] = [] specs: list[StageTwoTaskSpec] = [] for generated_mistake in generated_mistakes: - raise ValueError("wip oops") - ... + if generated_mistake.first_parsed_response is None or "NO_REASONING" in generated_mistake.first_parsed_response: + print("WARNING - skipping task as NO_REASONING found in the trace passed to the mistake generator") + continue + + stage_one_output = generated_mistake.task_spec.stage_one_output + config = stage_one_output.task_spec.inference_config.copy() + + path = Path( + f"{exp_dir}/mistakes_stage2/s1-{stage_one_output.task_spec.formatter_name}/{stage_one_output.task_spec.task_name}/{config.model}/{CompletePartialCOT.name()}.json" + ) + + trace_info = generated_mistake.task_spec.trace_info + assert trace_info is not None + trace_info.sentence_with_mistake = generated_mistake.first_parsed_response + + partial_cot_trace = trace_info.get_trace_upto_mistake() + + messages = CompletePartialCOT.format_example( + stage_one_output.task_spec.messages, partial_cot_trace, config.model + ) + + task_spec = StageTwoTaskSpec( + stage_one_output=stage_one_output, + inference_config=config, + formatter_name=CompletePartialCOT.name(), + messages=messages, + out_file_path=path, + trace_info=trace_info, + ) + + # if the mistake was the last step in the reasoning trace, then we don't need to complete the COT + # so just make a task output with no response + if trace_info.mistake_inserted_idx == len(trace_info.original_cot) - 1: + output = StageTwoTaskOutput( + task_spec=task_spec, + inference_output=ModelOutput(raw_response="", parsed_response=""), + ) + mistakes_inserted_at_last_position.append(output) + else: + specs.append(task_spec) print("2. Regenerating COTs with mistakes, note skipping tasks where mistake was last step in COT") outputs = run_with_caching_stage_two(save_completing_with_mistakes_every, batch, specs) diff --git a/scripts/ignored_reasoning/stage_two_stream.py b/scripts/ignored_reasoning/stage_two_stream.py index c07692a9..54d1a55b 100644 --- a/scripts/ignored_reasoning/stage_two_stream.py +++ b/scripts/ignored_reasoning/stage_two_stream.py @@ -1,15 +1,11 @@ import asyncio from pathlib import Path -from typing import Sequence from anyio import CapacityLimiter from grugstream import Observable from cot_transparency.apis import UniversalCaller -from cot_transparency.apis.base import InferenceResponse, ModelCaller -from cot_transparency.data_models.config import OpenaiInferenceConfig -from cot_transparency.data_models.messages import ChatMessage from cot_transparency.data_models.models import StageTwoTaskOutput from cot_transparency.formatters.transparency.s1_baselines import ZeroShotCOTUnbiasedTameraTFormatter from cot_transparency.tasks import task_function From 684d0a7ad58842329d10155688fca7789b9ff6d7 Mon Sep 17 00:00:00 2001 From: James Chua Date: Mon, 30 Oct 2023 11:09:37 +0800 Subject: [PATCH 26/28] refactor --- .../ignored_reasoning/stage_two_analysis.py | 36 ++++++++----------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/scripts/ignored_reasoning/stage_two_analysis.py b/scripts/ignored_reasoning/stage_two_analysis.py index 36d03b3e..486f3917 100644 --- a/scripts/ignored_reasoning/stage_two_analysis.py +++ b/scripts/ignored_reasoning/stage_two_analysis.py @@ -439,29 +439,21 @@ def aoc_plot( length_filter: Optional[list[int]] = None, hue: str = "stage_one_formatter_name", ): - df = get_data_frame_from_exp_dir(exp_dir) - df = df_filters(df, inconsistent_only, aggregate_over_tasks, model_filter, length_filter) - - # Mistakes AoC - df_mistakes = df[~df.was_truncated] - df_mistakes = df_mistakes.groupby("stage_one_hash").apply(check_same_answer).reset_index(drop=True) - df_mistakes = drop_not_found(df_mistakes) # type: ignore - aoc_mistakes = get_aoc(df_mistakes) - - # Early Answering AoC - df_early = df[~df.has_mistake] - df_early = df_early.groupby("stage_one_hash").apply(check_same_answer).reset_index(drop=True) - df_early = drop_not_found(df_early) # type: ignore - aoc_early = get_aoc(df_early) - - # baseline accuracies - baseline_accuracy(df, hue, "model") - - _aoc_point_plot(hue, df, aoc_mistakes, aoc_early, kind="bar") - _aoc_point_plot(hue, df, aoc_mistakes, aoc_early, kind="point") + items: list[StageTwoTaskOutput] = [] + loaded_dict = ExpLoader.stage_two(exp_dir, final_only=True) + for vals in loaded_dict.values(): + outputs = vals.outputs + items.extend(outputs) - if show_plots: - plt.show() + return aoc_plot_from_list( + items=items, + show_plots=show_plots, + inconsistent_only=inconsistent_only, + aggregate_over_tasks=aggregate_over_tasks, + model_filter=model_filter, + length_filter=length_filter, + hue=hue, + ) def aoc_plot_from_list( From e108abe174d59a3110908deafd6befbc9797df83 Mon Sep 17 00:00:00 2001 From: James Chua Date: Mon, 30 Oct 2023 11:11:42 +0800 Subject: [PATCH 27/28] add comments --- .../ignored_reasoning/stage_two_analysis.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/ignored_reasoning/stage_two_analysis.py b/scripts/ignored_reasoning/stage_two_analysis.py index 486f3917..0068fffe 100644 --- a/scripts/ignored_reasoning/stage_two_analysis.py +++ b/scripts/ignored_reasoning/stage_two_analysis.py @@ -55,20 +55,20 @@ def get_aoc(df: pd.DataFrame, x="cot_trace_length") -> pd.DataFrame: def get_auc(group: pd.DataFrame) -> float: assert group["original_cot_trace_length"].nunique() == 1 - # james: this seems wrong? Like all the areas are originally 1, but they get normalized to <= 1? proportion_of_cot = group[x] / max(group[x]) - # this sums to 1, but we need to renormalise to make it start from 0 and end at 1 to have a proper AUC """ - X_std = (X - X.min(axis=0)) / (X.max(axis=0) - X.min(axis=0)) - X_scaled = X_std * (max - min) + min + TODO: Maybe we need to renormalize the proportion_of_cot to be between 0 and 1? + Right now it can be between e.g. 0.6-1.0, which messes up np.trapz? + And sometimes its one single point too, which also messes up np.trapz + min_ = min(proportion_of_cot) + max_ = max(proportion_of_cot) + try: + proportion_scaled = proportion_of_cot.apply(lambda x: (x - min_) / (max_ - min_)) + except Exception as e: + print("Maybe there is only one value so min_ == max_") + raise e """ - # min_ = min(proportion_of_cot) - # max_ = max(proportion_of_cot) - # try: - # proportion_scaled = proportion_of_cot.apply(lambda x: (x - min_) / (max_ - min_)) - # except Exception as e: - # print("here") - # raise e + auc = np.trapz(group[0], x=proportion_of_cot) return auc From 71691674d1155a1a2e5d939b5b370ebaeab1dd4a Mon Sep 17 00:00:00 2001 From: James Chua Date: Mon, 30 Oct 2023 11:15:13 +0800 Subject: [PATCH 28/28] update grugstream --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5bdf6448..b4ddf359 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,4 +31,4 @@ wandb fuzzywuzzy python-Levenshtein anyio>=3.0.0 -grugstream \ No newline at end of file +grugstream>=0.0.13 \ No newline at end of file