diff --git a/dspy/teleprompt/copro_optimizer.py b/dspy/teleprompt/copro_optimizer.py index 5b62d9283..f9064443c 100644 --- a/dspy/teleprompt/copro_optimizer.py +++ b/dspy/teleprompt/copro_optimizer.py @@ -29,6 +29,7 @@ * results_latest: The min,max,avg,stddev of newest prompt scores for each predictor at each depth. * total_calls: The total number of calls to the task metric. These statistics will be returned as attributes of the best program. +* additional_instructions: Instructions appended to the generation signatures. Can be used to provide explicit details on the optimization metric. """ @@ -63,6 +64,7 @@ def __init__( depth=3, init_temperature=1.4, track_stats=False, + additional_instructions=None, **_kwargs, ): if breadth <= 1: @@ -73,9 +75,33 @@ def __init__( self.init_temperature = init_temperature self.prompt_model = prompt_model self.track_stats = track_stats + self._additional_instructions = additional_instructions if "verbose" in _kwargs: - dspy.logger.warning("DeprecationWarning: 'verbose' has been deprecated. To see all information for debugging, use 'dspy.set_log_level('debug')'. In the future this will raise an error.") + dspy.logger.warning( + "DeprecationWarning: 'verbose' has been deprecated. To see all information for debugging, use 'dspy.set_log_level('debug')'. In the future this will raise an error.", + ) + + def append_instructions(self, base_signature, additional_instructions): + return self._get_signature(base_signature).with_instructions( + " ".join([base_signature.instructions, additional_instructions]), + ) + + @property + def basic_generate_instruction(self): + return ( + self.append_instructions(BasicGenerateInstruction, self._additional_instructions) + if self._additional_instructions + else BasicGenerateInstruction + ) + + @property + def generate_instruction_given_attempts(self): + return ( + self.append_instructions(GenerateInstructionGivenAttempts, self._additional_instructions) + if self._additional_instructions + else GenerateInstructionGivenAttempts + ) def _check_candidates_equal(self, candidate1, candidate2): for p1, p2 in zip(candidate1["program"].predictors(), candidate2["program"].predictors()): @@ -153,13 +179,13 @@ def compile(self, student, *, trainset, eval_kwargs): if self.prompt_model: with dspy.settings.context(lm=self.prompt_model): instruct = dspy.Predict( - BasicGenerateInstruction, + self.basic_generate_instruction, n=self.breadth - 1, temperature=self.init_temperature, )(basic_instruction=basic_instruction) else: instruct = dspy.Predict( - BasicGenerateInstruction, + self.basic_generate_instruction, n=self.breadth - 1, temperature=self.init_temperature, )(basic_instruction=basic_instruction) @@ -299,19 +325,21 @@ def compile(self, student, *, trainset, eval_kwargs): if self.prompt_model: with dspy.settings.context(lm=self.prompt_model): instr = dspy.Predict( - GenerateInstructionGivenAttempts, + self.generate_instruction_given_attempts, n=self.breadth, temperature=self.init_temperature, )(attempted_instructions=attempts) else: instr = dspy.Predict( - GenerateInstructionGivenAttempts, + self.generate_instruction_given_attempts, n=self.breadth, temperature=self.init_temperature, )(attempted_instructions=attempts) if self.prompt_model: - dspy.logger.debug(f"(self.prompt_model.inspect_history(n=1)) {self.prompt_model.inspect_history(n=1)}") + dspy.logger.debug( + f"(self.prompt_model.inspect_history(n=1)) {self.prompt_model.inspect_history(n=1)}", + ) # Get candidates for each predictor new_candidates[id(p_base)] = instr.completions all_candidates[id(p_base)].proposed_instruction.extend(instr.completions.proposed_instruction) diff --git a/dspy/teleprompt/mipro_optimizer.py b/dspy/teleprompt/mipro_optimizer.py index c9e34189f..d244fd3ce 100644 --- a/dspy/teleprompt/mipro_optimizer.py +++ b/dspy/teleprompt/mipro_optimizer.py @@ -36,12 +36,13 @@ * init_temperature: The temperature used to generate new prompts. Higher roughly equals more creative. Default=1.0. * verbose: Tells the method whether or not to print intermediate steps. * track_stats: Tells the method whether or not to track statistics about the optimization process. - If True, the method will track a dictionary with a key corresponding to the trial number, + If True, the method will track a dictionary with a key corresponding to the trial number, and a value containing a dict with the following keys: * program: the program being evaluated at a given trial * score: the last average evaluated score for the program * pruned: whether or not this program was pruned This information will be returned as attributes of the best program. +* additional_instructions: Instructions appended to the generation signatures. Can be used to provide explicit details on the optimization metric. """ @@ -142,6 +143,7 @@ def __init__( verbose=False, track_stats=True, view_data_batch_size=10, + additional_instructions=None, ): self.num_candidates = num_candidates self.metric = metric @@ -152,6 +154,46 @@ def __init__( self.track_stats = track_stats self.teacher_settings = teacher_settings self.view_data_batch_size = view_data_batch_size + self._additional_instructions = additional_instructions + + def append_instructions(self, base_signature, additional_instructions): + return self._get_signature(base_signature).with_instructions( + " ".join([base_signature.instructions, additional_instructions]), + ) + + @property + def basic_generate_instruction(self): + return ( + self.append_instructions(BasicGenerateInstruction, self._additional_instructions) + if self._additional_instructions + else BasicGenerateInstruction + ) + + @property + def generate_instruction_with_data_observations(self): + return ( + self.append_instructions(BasicGenerateInstructionWithDataObservations, self._additional_instructions) + if self._additional_instructions + else BasicGenerateInstructionWithDataObservations + ) + + @property + def generate_instruction_with_examples(self): + return ( + self.append_instructions(BasicGenerateInstructionWithExamples, self._additional_instructions) + if self._additional_instructions + else BasicGenerateInstructionWithExamples + ) + + @property + def generate_instruction_with_examples_and_observations(self): + return ( + self.append_instructions( + BasicGenerateInstructionWithExamplesAndDataObservations, self._additional_instructions, + ) + if self._additional_instructions + else BasicGenerateInstructionWithExamplesAndDataObservations + ) def _print_full_program(self, program): for i, predictor in enumerate(program.predictors()): @@ -282,7 +324,7 @@ def _generate_first_N_candidates( # noqa: N802 instruct = None for i in range(1, self.num_candidates): new_instruct = dspy.Predict( - BasicGenerateInstructionWithExamplesAndDataObservations, + self.generate_instruction_with_examples_and_observations, n=1, temperature=self.init_temperature, )( @@ -302,7 +344,7 @@ def _generate_first_N_candidates( # noqa: N802 # Just data elif view_data: instruct = dspy.Predict( - BasicGenerateInstructionWithDataObservations, + self.generate_instruction_with_data_observations, n=N - 1, temperature=self.init_temperature, )(basic_instruction=basic_instruction, observations=self.observations) @@ -311,7 +353,7 @@ def _generate_first_N_candidates( # noqa: N802 instruct = None for i in range(1, self.num_candidates): # Note: skip over the first example set which is empty new_instruct = dspy.Predict( - BasicGenerateInstructionWithExamples, + self.generate_instruction_with_examples, n=1, temperature=self.init_temperature, )( @@ -329,7 +371,11 @@ def _generate_first_N_candidates( # noqa: N802 ) # Neither else: - instruct = dspy.Predict(BasicGenerateInstruction, n=N - 1, temperature=self.init_temperature)( + instruct = dspy.Predict( + self.basic_generate_instruction, + n=N - 1, + temperature=self.init_temperature, + )( basic_instruction=basic_instruction, ) @@ -371,7 +417,8 @@ def compile( student.predictors(), ) # num data summary calls + N * P - user_message = textwrap.dedent(f"""\ + user_message = textwrap.dedent( + f"""\ {YELLOW}{BOLD}WARNING: Projected Language Model (LM) Calls{ENDC} Please be advised that based on the parameters you have set, the maximum number of LM calls is projected as follows: @@ -381,22 +428,25 @@ def compile( {YELLOW}{BOLD}Estimated Cost Calculation:{ENDC} - {YELLOW}Total Cost = (Number of calls to task model * (Avg Input Token Length per Call * Task Model Price per Input Token + Avg Output Token Length per Call * Task Model Price per Output Token) + {YELLOW}Total Cost = (Number of calls to task model * (Avg Input Token Length per Call * Task Model Price per Input Token + Avg Output Token Length per Call * Task Model Price per Output Token) + (Number of calls to prompt model * (Avg Input Token Length per Call * Task Prompt Price per Input Token + Avg Output Token Length per Call * Prompt Model Price per Output Token).{ENDC} For a preliminary estimate of potential costs, we recommend you perform your own calculations based on the task and prompt models you intend to use. If the projected costs exceed your budget or expectations, you may consider: {YELLOW}- Reducing the number of trials (`num_trials`), the size of the trainset, or the number of LM calls in your program.{ENDC} - {YELLOW}- Using a cheaper task model to optimize the prompt.{ENDC}""") + {YELLOW}- Using a cheaper task model to optimize the prompt.{ENDC}""", + ) - user_confirmation_message = textwrap.dedent(f"""\ + user_confirmation_message = textwrap.dedent( + f"""\ To proceed with the execution of this program, please confirm by typing {BLUE}'y'{ENDC} for yes or {BLUE}'n'{ENDC} for no. If you would like to bypass this confirmation step in future executions, set the {YELLOW}`requires_permission_to_run`{ENDC} flag to {YELLOW}`False`.{ENDC} {YELLOW}Awaiting your input...{ENDC} - """) + """, + ) print(user_message)