Skip to content

Commit

Permalink
fix: add missing models and tools support to prompt optimizer
Browse files Browse the repository at this point in the history
  • Loading branch information
@hootan-na
hootan-na committed Nov 14, 2024
1 parent 3c3cc3e commit 15fe42b
Showing 1 changed file with 37 additions and 33 deletions.
70 changes: 37 additions & 33 deletions gemini/prompts/prompt_optimizer/vertex_ai_prompt_optimizer_ui.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,28 +30,28 @@
},
"source": [
"# Vertex Prompt Optimizer Notebook UI (Preview)\n",
"<table align=\"left\">\n",
" <td style=\"text-align: center\">\n",
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/generative-ai/blob/main/gemini/prompts/prompt_optimizer/vertex_ai_prompt_optimizer_ui.ipynb\">\n",
" <img width=\"32px\" src=\"https://www.gstatic.com/pantheon/images/bigquery/welcome_page/colab-logo.svg\" alt=\"Google Colaboratory logo\"><br> Open in Colab\n",
" </a>\n",
" </td>\n",
" <td style=\"text-align: center\">\n",
" <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fgenerative-ai%2Fmain%2Fgemini%2Fprompts%2Fprompt_optimizer%2Fvertex_ai_prompt_optimizer_ui.ipynb\">\n",
" <img width=\"32px\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" alt=\"Google Cloud Colab Enterprise logo\"><br> Open in Colab Enterprise\n",
" </a>\n",
" </td>\n",
" <td style=\"text-align: center\">\n",
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/generative-ai/main/gemini/prompts/prompt_optimizer/vertex_ai_prompt_optimizer_ui.ipynb\">\n",
" <img src=\"https://www.gstatic.com/images/branding/gcpiconscolors/vertexai/v1/32px.svg\" alt=\"Vertex AI logo\"><br> Open in Vertex AI Workbench\n",
" </a>\n",
" </td>\n",
" <td style=\"text-align: center\">\n",
" <a href=\"https://github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/prompts/prompt_optimizer/vertex_ai_prompt_optimizer_ui.ipynb\">\n",
" <img width=\"32px\" src=\"https://upload.wikimedia.org/wikipedia/commons/9/91/Octicons-mark-github.svg\" alt=\"GitHub logo\"><br> View on GitHub\n",
" </a>\n",
" </td>\n",
"</table>"
"\u003ctable align=\"left\"\u003e\n",
" \u003ctd style=\"text-align: center\"\u003e\n",
" \u003ca href=\"https://colab.research.google.com/github/GoogleCloudPlatform/generative-ai/blob/main/gemini/prompts/prompt_optimizer/vertex_ai_prompt_optimizer_ui.ipynb\"\u003e\n",
" \u003cimg width=\"32px\" src=\"https://www.gstatic.com/pantheon/images/bigquery/welcome_page/colab-logo.svg\" alt=\"Google Colaboratory logo\"\u003e\u003cbr\u003e Open in Colab\n",
" \u003c/a\u003e\n",
" \u003c/td\u003e\n",
" \u003ctd style=\"text-align: center\"\u003e\n",
" \u003ca href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fgenerative-ai%2Fmain%2Fgemini%2Fprompts%2Fprompt_optimizer%2Fvertex_ai_prompt_optimizer_ui.ipynb\"\u003e\n",
" \u003cimg width=\"32px\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" alt=\"Google Cloud Colab Enterprise logo\"\u003e\u003cbr\u003e Open in Colab Enterprise\n",
" \u003c/a\u003e\n",
" \u003c/td\u003e\n",
" \u003ctd style=\"text-align: center\"\u003e\n",
" \u003ca href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/generative-ai/main/gemini/prompts/prompt_optimizer/vertex_ai_prompt_optimizer_ui.ipynb\"\u003e\n",
" \u003cimg src=\"https://www.gstatic.com/images/branding/gcpiconscolors/vertexai/v1/32px.svg\" alt=\"Vertex AI logo\"\u003e\u003cbr\u003e Open in Vertex AI Workbench\n",
" \u003c/a\u003e\n",
" \u003c/td\u003e\n",
" \u003ctd style=\"text-align: center\"\u003e\n",
" \u003ca href=\"https://github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/prompts/prompt_optimizer/vertex_ai_prompt_optimizer_ui.ipynb\"\u003e\n",
" \u003cimg width=\"32px\" src=\"https://upload.wikimedia.org/wikipedia/commons/9/91/Octicons-mark-github.svg\" alt=\"GitHub logo\"\u003e\u003cbr\u003e View on GitHub\n",
" \u003c/a\u003e\n",
" \u003c/td\u003e\n",
"\u003c/table\u003e"
]
},
{
Expand Down Expand Up @@ -184,7 +184,7 @@
},
"outputs": [],
"source": [
"# @markdown **Project setup**: <br/>\n",
"# @markdown **Project setup**: \u003cbr/\u003e\n",
"PROJECT_ID = \"[YOUR_PROJECT]\" # @param {type:\"string\"}\n",
"LOCATION = \"us-central1\" # @param {type:\"string\"}\n",
"OUTPUT_PATH = \"[OUTPUT_PATH]\" # @param {type:\"string\"}\n",
Expand Down Expand Up @@ -216,8 +216,8 @@
},
"outputs": [],
"source": [
"SOURCE_MODEL = \"\" # @param [\"\", \"gemini-1.0-pro-001\", \"gemini-1.0-pro-002\", \"gemini-1.5-flash-001\", \"gemini-1.5-pro-001\", \"gemini-1.5-pro-002\", \"gemini-1.0-ultra-001\", \"text-bison@001\", \"text-bison@002\", \"text-bison32k@002\", \"text-unicorn@001\"]\n",
"TARGET_MODEL = \"gemini-1.5-flash-001\" # @param [\"gemini-1.0-pro-001\", \"gemini-1.0-pro-002\", \"gemini-1.5-flash-001\", \"gemini-1.5-pro-001\", \"gemini-1.5-pro-002\", \"gemini-1.0-ultra-001\"]\n",
"SOURCE_MODEL = \"\" # @param [\"\", \"gemini-1.0-pro-001\", \"gemini-1.0-pro-002\", \"gemini-1.5-flash-001\", \"gemini-1.5-flash-002\", \"gemini-1.5-pro-001\", \"gemini-1.5-pro-002\", \"gemini-1.0-ultra-001\", \"gemini-experimental\", \"gemini-flash-experimental\", \"gemini-pro-experimental\", \"text-bison@001\", \"text-bison@002\", \"text-bison32k@002\", \"text-unicorn@001\"]\n",
"TARGET_MODEL = \"gemini-1.5-flash-001\" # @param [\"gemini-1.0-pro-001\", \"gemini-1.0-pro-002\", \"gemini-1.5-flash-001\", \"gemini-1.5-flash-002\", \"gemini-1.5-pro-001\", \"gemini-1.5-pro-002\", \"gemini-1.0-ultra-001\", \"gemini-experimental\", \"gemini-flash-experimental\", \"gemini-pro-experimental\"]\n",
"OPTIMIZATION_MODE = \"instruction_and_demo\" # @param [\"instruction\", \"demonstration\", \"instruction_and_demo\"]\n",
"EVAL_METRIC = \"question_answering_correctness\" # @param [\"bleu\", \"coherence\", \"exact_match\", \"fluency\", \"groundedness\", \"text_quality\", \"verbosity\", \"rouge_1\", \"rouge_2\", \"rouge_l\", \"rouge_l_sum\", \"safety\", \"question_answering_correctness\", \"question_answering_quality\", \"summarization_quality\", \"tool_name_match\", \"tool_parameter_key_match\", \"tool_parameter_kv_match\", \"tool_call_valid\"] {type:\"string\"}"
]
Expand All @@ -240,20 +240,20 @@
},
"outputs": [],
"source": [
"# @markdown **Instruction Optimization Configs**: <br/>\n",
"# @markdown **Instruction Optimization Configs**: \u003cbr/\u003e\n",
"NUM_INST_OPTIMIZATION_STEPS = 10 # @param {type:\"integer\"}\n",
"NUM_TEMPLATES_PER_STEP = 2 # @param {type:\"integer\"}\n",
"\n",
"# @markdown **Demonstration Optimization Configs**: <br/>\n",
"# @markdown **Demonstration Optimization Configs**: \u003cbr/\u003e\n",
"NUM_DEMO_OPTIMIZATION_STEPS = 10 # @param {type:\"integer\"}\n",
"NUM_DEMO_PER_PROMPT = 3 # @param {type:\"integer\"}\n",
"\n",
"# @markdown **Model Configs**: <br/>\n",
"# @markdown **Model Configs**: \u003cbr/\u003e\n",
"TARGET_MODEL_QPS = 3.0 # @param {type:\"number\"}\n",
"SOURCE_MODEL_QPS = 3.0 # @param {type:\"number\"}\n",
"EVAL_QPS = 3.0 # @param {type:\"number\"}\n",
"\n",
"# @markdown **Multi-metric Configs**: <br/>\n",
"# @markdown **Multi-metric Configs**: \u003cbr/\u003e\n",
"# @markdown Use this section only if you need more than one metric for optimization. This will override the metric you picked above.\n",
"EVAL_METRIC_1 = \"NA\" # @param [\"NA\", \"bleu\", \"coherence\", \"exact_match\", \"fluency\", \"groundedness\", \"text_quality\", \"verbosity\", \"rouge_1\", \"rouge_2\", \"rouge_l\", \"rouge_l_sum\", \"safety\", \"question_answering_correctness\", \"question_answering_quality\", \"summarization_quality\", \"tool_name_match\", \"tool_parameter_key_match\", \"tool_parameter_kv_match\", \"tool_call_valid\"] {type:\"string\"}\n",
"EVAL_METRIC_1_WEIGHT = 0.0 # @param {type:\"number\"}\n",
Expand All @@ -263,11 +263,13 @@
"EVAL_METRIC_3_WEIGHT = 0.0 # @param {type:\"number\"}\n",
"METRIC_AGGREGATION_TYPE = \"weighted_sum\" # @param [\"weighted_sum\", \"weighted_average\"]\n",
"\n",
"# @markdown **Misc Configs**: <br/>\n",
"# @markdown **Misc Configs**: \u003cbr/\u003e\n",
"PLACEHOLDER_TO_VALUE = \"{}\" # @param\n",
"RESPONSE_MIME_TYPE = \"application/json\" # @param [\"text/plain\", \"application/json\", \"text/x.enum\"]\n",
"RESPONSE_SCHEMA = \"\"\n",
"TARGET_LANGUAGE = \"English\" # @param [\"English\", \"French\", \"German\", \"Hebrew\", \"Hindi\", \"Japanese\", \"Korean\", \"Portuguese\", \"Simplified Chinese\", \"Spanish\", \"Traditional Chinese\"]"
"TARGET_LANGUAGE = \"English\" # @param [\"English\", \"French\", \"German\", \"Hebrew\", \"Hindi\", \"Japanese\", \"Korean\", \"Portuguese\", \"Simplified Chinese\", \"Spanish\", \"Traditional Chinese\"]\n",
"TOOLS = \"\" # @param\n",
"TOOL_CONFIG = \"\" # @param"
]
},
{
Expand Down Expand Up @@ -339,6 +341,8 @@
" \"response_schema\": RESPONSE_SCHEMA,\n",
" \"language\": TARGET_LANGUAGE,\n",
" \"placeholder_to_content\": json.loads(PLACEHOLDER_TO_VALUE),\n",
" \"tools\": TOOLS,\n",
" \"tool_config\": TOOL_CONFIG,\n",
"}\n",
"\n",
"if EVAL_METRIC_1 == \"NA\":\n",
Expand Down Expand Up @@ -390,7 +394,7 @@
"results_ui = vapo_lib.ResultsUI(RESULT_PATH)\n",
"\n",
"results_df_html = \"\"\"\n",
"<style>\n",
"\u003cstyle\u003e\n",
" .scrollable {\n",
" width: 100%;\n",
" height: 80px;\n",
Expand All @@ -406,7 +410,7 @@
" th {\n",
" background-color: var(--colab-highlighted-surface-color);\n",
" }\n",
"</style>\n",
"\u003c/style\u003e\n",
"\"\"\"\n",
"\n",
"display(HTML(results_df_html))\n",
Expand Down

0 comments on commit 15fe42b

Please sign in to comment.