From 58b7c5a6a0321621acc8ee9c33b6fea2eb08d77a Mon Sep 17 00:00:00 2001 From: J2-D2-3PO <188380414+J2-D2-3PO@users.noreply.github.com> Date: Fri, 6 Dec 2024 12:18:58 -0700 Subject: [PATCH 1/4] docs(weave): Upload and document Service API notebook --- .../gen_notebooks/weave_via_service_api.md | 488 ++++++++++++++ docs/notebooks/weave_via_service_api.ipynb | 628 ++++++++++++++++++ 2 files changed, 1116 insertions(+) create mode 100644 docs/docs/reference/gen_notebooks/weave_via_service_api.md create mode 100644 docs/notebooks/weave_via_service_api.ipynb diff --git a/docs/docs/reference/gen_notebooks/weave_via_service_api.md b/docs/docs/reference/gen_notebooks/weave_via_service_api.md new file mode 100644 index 000000000000..c2bdc350c61f --- /dev/null +++ b/docs/docs/reference/gen_notebooks/weave_via_service_api.md @@ -0,0 +1,488 @@ +--- +title: Use the Service API to Log and Query Traces +--- + + +:::note[This is a notebook] + +
Open In Colab
Open in Colab
 + +
View in Github
View in Github
 + +::: + + + +In the following notebook, you will learn how to use the Weave Service API to log traces. Specifically, you will use the Service API to: + +1. [Create a mock of a simple LLM call and response, and log it to Weave.](#simple-trace) +2. [Create a mock of a more complex LLM call and response, and log it to Weave.](#complex-trace) +3. [Run a sample lookup query on the logged traces.](#run-a-lookup-query) + +:::tip[View logged traces] +You can view all of the Weave traces created when you run the code in this guide by going to the **Traces** tab in your Weave project (specified by `team_id\project_id`), and selecting the name of the trace. +::: + +Before beginning, complete the [prerequisites](#prerequisites-set-variables-and-endpoints) + +## Prerequisites: Set variables and endpoints + +The following code sets the URL endpoints that will be used to access the Service API: + +- [`https://trace.wandb.ai/call/start`](https://weave-docs.wandb.ai/reference/service-api/call-start-call-start-post) +- [`https://trace.wandb.ai/call/end`](https://weave-docs.wandb.ai/reference/service-api/call-end-call-end-post) +- [`https://trace.wandb.ai/calls/stream_query`](https://weave-docs.wandb.ai/reference/service-api/calls-query-stream-calls-stream-query-post) + +Additionally, you must set the following variables: + +- `project_id`: The name of the W&B project that you want to log your traces to. +- `team_id`: Your W&B team name. +- `wandb_token`: Your [W&B authorization token](https://wandb.ai/authorize). + + +```python +import datetime +import json +import requests + +# Headers and URLs +headers = {"Content-Type": "application/json"} +url_start = "https://trace.wandb.ai/call/start" +url_end = "https://trace.wandb.ai/call/end" +url_stream_query = "https://trace.wandb.ai/calls/stream_query" + +# W&B variables +team_id="" +project_id="" +wandb_token="" +``` + +## Simple trace +The following sections walk you through creating a simple trace. + +1. [Start a complex trace](#start-a-complex-trace) +2. [End a simple trace](#end-a-simple-trace) + +### Start a simple trace + +The following code creates a sample LLM call `payload_start` and logs it to Weave using the `url_start` endpoint. The `payload_start` object mimics a call to OpenAI's `gpt-4o` with the query `Why is the sky blue?`. + +On success, this code will output a message indicating that the trace was started: + +``` +Call started. ID: 01939cdc-38d2-7d61-940d-dcca0a56c575, Trace ID: 01939cdc-38d2-7d61-940d-dcd0e76c5f34 +``` + + +```python +## ------------ +## Start trace +## ------------ +payload_start = { + "start": { + "project_id": f"{team_id}/{project_id}", + "op_name": "simple_trace", + "started_at": datetime.datetime.now().isoformat(), + "inputs": { + # Use this "messages" style to generate the chat UI in the expanded trace. + "messages": [{"role":"user","content":"Why is the sky blue?"}], + "model": "gpt-4o" + }, + "attributes": {}, + } +} +response = requests.post( + url_start, headers=headers, json=payload_start, auth=("api", wandb_token) +) +if response.status_code == 200: + data = response.json() + call_id = data.get("id") + trace_id = data.get("trace_id") + print(f"Call started. ID: {call_id}, Trace ID: {trace_id}") +else: + print("Start request failed with status:", response.status_code) + print(response.text) + exit() +``` + +### End a simple trace + +To complete the simple trace, the following code creates a sample LLM call `payload_end` and logs it to Weave using the `url_end` endpoint. The `payload_end` object mimics the response from OpenAI's `gpt-4o` given the query `Why is the sky blue?`. The object is formatted so that pricing summary information and the chat completion are generated in trace view in the Weave Dashboard. + +On success, this code will output a message indicating that the trace completed: + +``` +Call ended. +``` + + +```python +## ------------ +## End trace +## ------------ +payload_end = { + "end": { + "project_id": f"{team_id}/{project_id}", + "id": call_id, + "ended_at": datetime.datetime.now().isoformat(), + "output": { + # Use this "choices" style to add the completion to the chat UI in the expanded trace. + "choices": [ + {"message": {"content": "It’s due to Rayleigh scattering, where shorter blue wavelengths of sunlight scatter in all directions."}}, + ] + }, + # Format the summary like this to generate the pricing summary information in the traces table. + "summary": { + "usage": { + "gpt-4o": { + "prompt_tokens": 10, + "completion_tokens": 20, + "total_tokens": 30, + "requests": 1, + } + } + }, + } +} +response = requests.post( + url_end, headers=headers, json=payload_end, auth=("api", wandb_token) +) +if response.status_code == 200: + print("Call ended.") +else: + print("End request failed with status:", response.status_code) + print(response.text) +``` + +## Complex trace +The following sections walk you through creating a more complex trace with child spans, similar to a mult-operation RAG lookup. + +1. [Start a complex trace](#complex-trace) +2. [Add a child span: RAG document lookup](#add-a-child-span-to-a-complex-trace-rag-document-lookup) +3. [Add a child span: LLM completion call](#add-a-child-span-to-a-complex-trace-llm-completion-call) +4. [End a complex trace](#end-a-complex-trace) + +### Start a complex trace + +The following code demonstrates how to create a more complex trace with multiple spans. An example of this would be a Retrieval-Augmented Generation (RAG) lookup followed by an LLM call. The first part initializes a parent trace(`payload_parent_start`) that represents the overall operation. In this case, the operation is processing the user query `Can you summarize the key points of this document?`. + +The `payload_parent_start` object mimics the initial step in a multi-step workflow, logging the the operation in Weave using the `url_start` endpoint. + +On success, this code will output a message indicating that the parent call was logged: + +``` +Parent call started. ID: 01939d26-0844-7c43-94bb-cdc471b6d65f, Trace ID: 01939d26-0844-7c43-94bb-cdd97dc296c8 +``` + + +```python +## ------------ +## Start trace (parent) +## ------------ + +# Parent call: Start +payload_parent_start = { + "start": { + "project_id": f"{team_id}/{project_id}", + "op_name": "complex_trace", + "started_at": datetime.datetime.now().isoformat(), + "inputs": { + "question": "Can you summarize the key points of this document?" + }, + "attributes": {}, + } +} +response = requests.post( + url_start, headers=headers, json=payload_parent_start, auth=("api", wandb_token) +) +if response.status_code == 200: + data = response.json() + parent_call_id = data.get("id") + trace_id = data.get("trace_id") + print(f"Parent call started. ID: {parent_call_id}, Trace ID: {trace_id}") +else: + print("Parent start request failed with status:", response.status_code) + print(response.text) + exit() +``` + +### Add a child span to a complex trace: RAG document lookup + +The following code demonstrates how to add a child span to the parent trace started in the previous step. This step models a the RAG document lookup sub-operation in the overarching workflow. + +The child trace is initiated with the `payload_child_start` object, which includes: +- `trace_id`: Links this child span to the parent trace. +- `parent_id`: Associates the child span with the parent operation. +- `inputs`: Logs the search query, e.g., + `"This is a search query of the documents I'm looking for."` + +On a successful call to the `url_start` endpoint, the code outputs a message indicating that the child call was started and completed: + +``` +Child call started. ID: 01939d32-23d6-75f2-9128-36a4a806f179 +Child call ended. +``` + + +```python +## ------------ +## Child span: +## Ex. RAG Document lookup +## ------------ + +# Child call: Start +payload_child_start = { + "start": { + "project_id": f"{team_id}/{project_id}", + "op_name": "rag_document_lookup", + "trace_id": trace_id, + "parent_id": parent_call_id, + "started_at": datetime.datetime.now().isoformat(), + "inputs": { + "document_search": "This is a search query of the documents I'm looking for." + }, + "attributes": {}, + } +} +response = requests.post( + url_start, headers=headers, json=payload_child_start, auth=("api", wandb_token) +) +if response.status_code == 200: + data = response.json() + child_call_id = data.get("id") + print(f"Child call started. ID: {child_call_id}") +else: + print("Child start request failed with status:", response.status_code) + print(response.text) + exit() + +# Child call: End +payload_child_end = { + "end": { + "project_id": f"{team_id}/{project_id}", + "id": child_call_id, + "ended_at": datetime.datetime.now().isoformat(), + "output": { + "document_results": "This will be the RAG'd document text which will be returned from the search query." + }, + "summary": {}, + } +} +response = requests.post( + url_end, headers=headers, json=payload_child_end, auth=("api", wandb_token) +) +if response.status_code == 200: + print("Child call ended.") +else: + print("Child end request failed with status:", response.status_code) + print(response.text) +``` + +### Add a child span to a complex trace: LLM completion call + +The following code demonstrates how to add another child span to the parent trace, representing an LLM completion call. This step models the AI's response generation based on document context retrieved in the previous RAG operation. + +The LLM completion trace is initiated with the `payload_child_start` object, which includes: +- `trace_id`: Links this child span to the parent trace. +- `parent_id`: Associates the child span with the overarching workflow. +- `inputs`: Logs the input messages for the LLM, including the user query and the appended document context. +- `model`: Specifies the model used for the operation (`gpt-4o`). + +On success, the code outputs a message indicating the LLM child span trace has started and ended: + +``` +Child call started. ID: 0245acdf-83a9-4c90-90df-dcb2b89f234a +``` + +Once the operation completes, the `payload_child_end` object ends the trace by logging the LLM-generated response in the `output` field. Usage summary information is also logged. + +On success, the code outputs a message indicating the LLM child span trace has started and ended: + +``` +Child call started. ID: 0245acdf-83a9-4c90-90df-dcb2b89f234a +Child call ended. +``` + + +```python +## ------------ +## Child span: +## Create an LLM completion call +## ------------ + +# Child call: Start +payload_child_start = { + "start": { + "project_id": f"{team_id}/{project_id}", + "op_name": "llm_completion", + "trace_id": trace_id, + "parent_id": parent_call_id, + "started_at": datetime.datetime.now().isoformat(), + "inputs": { + "messages": [{"role":"user","content":"With the following document context, could you help me answer:\n Can you summarize the key points of this document?\n [+ appended document context]"}], + "model": "gpt-4o" + }, + "attributes": {}, + } +} +response = requests.post( + url_start, headers=headers, json=payload_child_start, auth=("api", wandb_token) +) +if response.status_code == 200: + data = response.json() + child_call_id = data.get("id") + print(f"Child call started. ID: {child_call_id}") +else: + print("Child start request failed with status:", response.status_code) + print(response.text) + exit() + +# Child call: End +payload_child_end = { + "end": { + "project_id": f"{team_id}/{project_id}", + "id": child_call_id, + "ended_at": datetime.datetime.now().isoformat(), + "output": { + "choices": [ + {"message": {"content": "This is the response generated by the LLM."}}, + ] + }, + "summary": { + "usage": { + "gpt-4o": { + "prompt_tokens": 10, + "completion_tokens": 20, + "total_tokens": 30, + "requests": 1, + } + } + }, + } +} +response = requests.post( + url_end, headers=headers, json=payload_child_end, auth=("api", wandb_token) +) +if response.status_code == 200: + print("Child call ended.") +else: + print("Child end request failed with status:", response.status_code) + print(response.text) +``` + +### End a complex trace + +The following code demonstrates how to finalize the parent trace, marking the completion of the entire workflow. This step aggregates the results of all child spans (e.g., RAG lookup and LLM completion) and logs the final output and metadata. + +The trace is finalized using the `payload_parent_end` object, which includes: +- `id`: The `parent_call_id` from the initial parent trace start. +- `output`: Represents the final output of the entire workflow. +- `summary`: Consolidates usage data for the entire workflow. +- `prompt_tokens`: Total tokens used for all prompts. +- `completion_tokens`: Total tokens generated in all responses. +- `total_tokens`: Combined token count for the workflow. +- `requests`: Total number of requests made (in this case, `1`). + +On success, the code outputs: + +``` +Parent call ended. +``` + + +```python +## ------------ +## End trace +## ------------ + +# Parent call: End +payload_parent_end = { + "end": { + "project_id": f"{team_id}/{project_id}", + "id": parent_call_id, + "ended_at": datetime.datetime.now().isoformat(), + "output": { + "choices": [ + {"message": {"content": "This is the response generated by the LLM."}}, + ] + }, + "summary": { + "usage": { + "gpt-4o": { + "prompt_tokens": 10, + "completion_tokens": 20, + "total_tokens": 30, + "requests": 1, + } + } + }, + } +} +response = requests.post( + url_end, headers=headers, json=payload_parent_end, auth=("api", wandb_token) +) +if response.status_code == 200: + print("Parent call ended.") +else: + print("Parent end request failed with status:", response.status_code) + print(response.text) +``` + +## Run a lookup query +The following code demonstrates how to query the traces created in previous examples, filtering only for traces where the `inputs.model` field is equal to `gpt-4o`. + +The `query_payload` object includes: +- `project_id`: Identifies the team and project to query. +- `filter`: Ensures the query returns only **trace roots** (top-level traces). +- `query`: Defines the filter logic using the `$expr` operator: + - `$getField`: Retrieves the `inputs.model` field. + - `$literal`: Matches traces where `inputs.model` equals `"gpt-4o"`. +- `limit`: Limits the query to 10,000 results. +- `offset`: Starts the query at the first result. +- `sort_by`: Orders results by the `started_at` timestamp in descending order. +- `include_feedback`: Excludes feedback data from the results. + +On a successful query, the response will include trace data matching the query parameters: + +``` +{'id': '01939cf3-541f-76d3-ade3-50cfae068b39', 'project_id': 'cool-new-team/uncategorized', 'op_name': 'simple_trace', 'display_name': None, 'trace_id': '01939cf3-541f-76d3-ade3-50d5cfabe2db', 'parent_id': None, 'started_at': '2024-12-06T17:10:12.590000Z', 'attributes': {}, 'inputs': {'messages': [{'role': 'user', 'content': 'Why is the sky blue?'}], 'model': 'gpt-4o'}, 'ended_at': '2024-12-06T17:47:08.553000Z', 'exception': None, 'output': {'choices': [{'message': {'content': 'It’s due to Rayleigh scattering, where shorter blue wavelengths of sunlight scatter in all directions.'}}]}, 'summary': {'usage': {'gpt-4o': {'prompt_tokens': 10, 'completion_tokens': 20, 'requests': 1, 'total_tokens': 30}}, 'weave': {'status': 'success', 'trace_name': 'simple_trace', 'latency_ms': 2215963}}, 'wb_user_id': 'VXNlcjoyMDk5Njc0', 'wb_run_id': None, 'deleted_at': None} +``` + + + + + +```python +query_payload = { + "project_id": f"{team_id}/{project_id}", + "filter": {"trace_roots_only": True}, + "query": { + "$expr": { + "$eq": [ + {"$getField": "inputs.model"}, + {"$literal": "gpt-4o"} + ] + } + }, + "limit": 10000, + "offset": 0, + "sort_by": [{"field": "started_at", "direction": "desc"}], + "include_feedback": False +} +response = requests.post( + url_stream_query, headers=headers, json=query_payload, auth=("api", wandb_token) +) +if response.status_code == 200: + print("Query successful!") + try: + data = response.json() + print(data) + except json.JSONDecodeError as e: + # Alternate decode + json_objects = response.text.strip().split("\n") + parsed_data = [json.loads(obj) for obj in json_objects] + print(parsed_data) +else: + print(f"Query failed with status code: {response.status_code}") + print(response.text) +``` diff --git a/docs/notebooks/weave_via_service_api.ipynb b/docs/notebooks/weave_via_service_api.ipynb new file mode 100644 index 000000000000..f6431cc1517d --- /dev/null +++ b/docs/notebooks/weave_via_service_api.ipynb @@ -0,0 +1,628 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "xSqO08zH3qRH" + }, + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Use the Service API to Log and Query Traces" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the following guide, you will learn how to use the Weave Service API to log traces. Specifically, you will use the Service API to:\n", + "\n", + "1. [Create a mock of a simple LLM call and response, and log it to Weave.](#simple-trace)\n", + "2. [Create a mock of a more complex LLM call and response, and log it to Weave.](#complex-trace)\n", + "3. [Run a sample lookup query on the logged traces.](#run-a-lookup-query)\n", + "\n", + "> **View logged traces**\n", + ">\n", + "> You can view all of the Weave traces created when you run the code in this guide by going to the **Traces** tab in your Weave project (specified by `team_id\\project_id`), and selecting the name of the trace.\n", + "\n", + "Before beginning, complete the [prerequisites](#prerequisites-set-variables-and-endpoints)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prerequisites: Set variables and endpoints\n", + "\n", + "The following code sets the URL endpoints that will be used to access the Service API:\n", + "\n", + "- [`https://trace.wandb.ai/call/start`](https://weave-docs.wandb.ai/reference/service-api/call-start-call-start-post)\n", + "- [`https://trace.wandb.ai/call/end`](https://weave-docs.wandb.ai/reference/service-api/call-end-call-end-post)\n", + "- [`https://trace.wandb.ai/calls/stream_query`](https://weave-docs.wandb.ai/reference/service-api/calls-query-stream-calls-stream-query-post)\n", + "\n", + "Additionally, you must set the following variables:\n", + "\n", + "- `project_id`: The name of the W&B project that you want to log your traces to.\n", + "- `team_id`: Your W&B team name.\n", + "- `wandb_token`: Your [W&B authorization token](https://wandb.ai/authorize)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "fkUXz72LCLlA" + }, + "outputs": [], + "source": [ + "import datetime\n", + "import json\n", + "import requests\n", + "\n", + "# Headers and URLs\n", + "headers = {\"Content-Type\": \"application/json\"}\n", + "url_start = \"https://trace.wandb.ai/call/start\"\n", + "url_end = \"https://trace.wandb.ai/call/end\"\n", + "url_stream_query = \"https://trace.wandb.ai/calls/stream_query\"\n", + "\n", + "# W&B variables\n", + "team_id=\"\"\n", + "project_id=\"\"\n", + "wandb_token=\"\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Simple trace\n", + "The following sections walk you through creating a simple trace.\n", + "\n", + "1. [Start a complex trace](#start-a-complex-trace)\n", + "2. [End a simple trace](#end-a-simple-trace)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1bnDRiDN-dtc" + }, + "source": [ + "### Start a simple trace \n", + "\n", + "The following code creates a sample LLM call `payload_start` and logs it to Weave using the `url_start` endpoint. The `payload_start` object mimics a call to OpenAI's `gpt-4o` with the query `Why is the sky blue?`.\n", + "\n", + "On success, this code will output a message indicating that the trace was started:\n", + "\n", + "```\n", + "Call started. ID: 01939cdc-38d2-7d61-940d-dcca0a56c575, Trace ID: 01939cdc-38d2-7d61-940d-dcd0e76c5f34\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "sGq8p8Le-pc2", + "outputId": "dd123b02-9478-408d-9755-29fc5612e505" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Call started. ID: 01939cdc-38d2-7d61-940d-dcca0a56c575, Trace ID: 01939cdc-38d2-7d61-940d-dcd0e76c5f34\n" + ] + } + ], + "source": [ + "## ------------\n", + "## Start trace\n", + "## ------------\n", + "payload_start = {\n", + " \"start\": {\n", + " \"project_id\": f\"{team_id}/{project_id}\",\n", + " \"op_name\": \"simple_trace\",\n", + " \"started_at\": datetime.datetime.now().isoformat(),\n", + " \"inputs\": {\n", + " # Use this \"messages\" style to generate the chat UI in the expanded trace.\n", + " \"messages\": [{\"role\":\"user\",\"content\":\"Why is the sky blue?\"}],\n", + " \"model\": \"gpt-4o\"\n", + " },\n", + " \"attributes\": {},\n", + " }\n", + "}\n", + "response = requests.post(\n", + " url_start, headers=headers, json=payload_start, auth=(\"api\", wandb_token)\n", + ")\n", + "if response.status_code == 200:\n", + " data = response.json()\n", + " call_id = data.get(\"id\")\n", + " trace_id = data.get(\"trace_id\")\n", + " print(f\"Call started. ID: {call_id}, Trace ID: {trace_id}\")\n", + "else:\n", + " print(\"Start request failed with status:\", response.status_code)\n", + " print(response.text)\n", + " exit()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "G36zXvAM3jPa" + }, + "source": [ + "### End a simple trace\n", + "\n", + "To complete the simple trace, the following code creates a sample LLM call `payload_end` and logs it to Weave using the `url_end` endpoint. The `payload_end` object mimics the response from OpenAI's `gpt-4o` given the query `Why is the sky blue?`. The object is formatted so that pricing summary information and the chat completion are generated in trace view in the Weave Dashboard.\n", + "\n", + "On success, this code will output a message indicating that the trace completed:\n", + "\n", + "```\n", + "Call ended.\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "5xthEk_4-0iZ" + }, + "outputs": [], + "source": [ + "## ------------\n", + "## End trace\n", + "## ------------\n", + "payload_end = {\n", + " \"end\": {\n", + " \"project_id\": f\"{team_id}/{project_id}\",\n", + " \"id\": call_id,\n", + " \"ended_at\": datetime.datetime.now().isoformat(),\n", + " \"output\": {\n", + " # Use this \"choices\" style to add the completion to the chat UI in the expanded trace.\n", + " \"choices\": [\n", + " {\"message\": {\"content\": \"It’s due to Rayleigh scattering, where shorter blue wavelengths of sunlight scatter in all directions.\"}},\n", + " ]\n", + " },\n", + " # Format the summary like this to generate the pricing summary information in the traces table.\n", + " \"summary\": {\n", + " \"usage\": {\n", + " \"gpt-4o\": {\n", + " \"prompt_tokens\": 10,\n", + " \"completion_tokens\": 20,\n", + " \"total_tokens\": 30,\n", + " \"requests\": 1,\n", + " }\n", + " }\n", + " },\n", + " }\n", + "}\n", + "response = requests.post(\n", + " url_end, headers=headers, json=payload_end, auth=(\"api\", wandb_token)\n", + ")\n", + "if response.status_code == 200:\n", + " print(\"Call ended.\")\n", + "else:\n", + " print(\"End request failed with status:\", response.status_code)\n", + " print(response.text)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Complex trace\n", + "The following sections walk you through creating a more complex trace with child spans, similar to a mult-operation RAG lookup.\n", + "\n", + "1. [Start a complex trace](#complex-trace)\n", + "2. [Add a child span: RAG document lookup](#add-a-child-span-to-a-complex-trace-rag-document-lookup)\n", + "3. [Add a child span: LLM completion call](#add-a-child-span-to-a-complex-trace-llm-completion-call)\n", + "4. [End a complex trace](#end-a-complex-trace)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3nsK302HCFdu" + }, + "source": [ + "### Start a complex trace \n", + "\n", + "The following code demonstrates how to create a more complex trace with multiple spans. An example of this would be a Retrieval-Augmented Generation (RAG) lookup followed by an LLM call. The first part initializes a parent trace(`payload_parent_start`) that represents the overall operation. In this case, the operation is processing the user query `Can you summarize the key points of this document?`.\n", + "\n", + "The `payload_parent_start` object mimics the initial step in a multi-step workflow, logging the the operation in Weave using the `url_start` endpoint.\n", + "\n", + "On success, this code will output a message indicating that the parent call was logged:\n", + "\n", + "```\n", + "Parent call started. ID: 01939d26-0844-7c43-94bb-cdc471b6d65f, Trace ID: 01939d26-0844-7c43-94bb-cdd97dc296c8\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "V_1r4gPsy4Yj" + }, + "outputs": [], + "source": [ + "## ------------\n", + "## Start trace (parent)\n", + "## ------------\n", + "\n", + "# Parent call: Start\n", + "payload_parent_start = {\n", + " \"start\": {\n", + " \"project_id\": f\"{team_id}/{project_id}\",\n", + " \"op_name\": \"complex_trace\",\n", + " \"started_at\": datetime.datetime.now().isoformat(),\n", + " \"inputs\": {\n", + " \"question\": \"Can you summarize the key points of this document?\"\n", + " },\n", + " \"attributes\": {},\n", + " }\n", + "}\n", + "response = requests.post(\n", + " url_start, headers=headers, json=payload_parent_start, auth=(\"api\", wandb_token)\n", + ")\n", + "if response.status_code == 200:\n", + " data = response.json()\n", + " parent_call_id = data.get(\"id\")\n", + " trace_id = data.get(\"trace_id\")\n", + " print(f\"Parent call started. ID: {parent_call_id}, Trace ID: {trace_id}\")\n", + "else:\n", + " print(\"Parent start request failed with status:\", response.status_code)\n", + " print(response.text)\n", + " exit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Add a child span to a complex trace: RAG document lookup\n", + "\n", + "The following code demonstrates how to add a child span to the parent trace started in the previous step. This step models a the RAG document lookup sub-operation in the overarching workflow.\n", + "\n", + "The child trace is initiated with the `payload_child_start` object, which includes:\n", + "- `trace_id`: Links this child span to the parent trace.\n", + "- `parent_id`: Associates the child span with the parent operation.\n", + "- `inputs`: Logs the search query, e.g., \n", + " `\"This is a search query of the documents I'm looking for.\"`\n", + "\n", + "On a successful call to the `url_start` endpoint, the code outputs a message indicating that the child call was started and completed:\n", + "\n", + "```\n", + "Child call started. ID: 01939d32-23d6-75f2-9128-36a4a806f179\n", + "Child call ended.\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Bj_7uOF5zNT6" + }, + "outputs": [], + "source": [ + "## ------------\n", + "## Child span:\n", + "## Ex. RAG Document lookup\n", + "## ------------\n", + "\n", + "# Child call: Start\n", + "payload_child_start = {\n", + " \"start\": {\n", + " \"project_id\": f\"{team_id}/{project_id}\",\n", + " \"op_name\": \"rag_document_lookup\",\n", + " \"trace_id\": trace_id,\n", + " \"parent_id\": parent_call_id,\n", + " \"started_at\": datetime.datetime.now().isoformat(),\n", + " \"inputs\": {\n", + " \"document_search\": \"This is a search query of the documents I'm looking for.\"\n", + " },\n", + " \"attributes\": {},\n", + " }\n", + "}\n", + "response = requests.post(\n", + " url_start, headers=headers, json=payload_child_start, auth=(\"api\", wandb_token)\n", + ")\n", + "if response.status_code == 200:\n", + " data = response.json()\n", + " child_call_id = data.get(\"id\")\n", + " print(f\"Child call started. ID: {child_call_id}\")\n", + "else:\n", + " print(\"Child start request failed with status:\", response.status_code)\n", + " print(response.text)\n", + " exit()\n", + "\n", + "# Child call: End\n", + "payload_child_end = {\n", + " \"end\": {\n", + " \"project_id\": f\"{team_id}/{project_id}\",\n", + " \"id\": child_call_id,\n", + " \"ended_at\": datetime.datetime.now().isoformat(),\n", + " \"output\": {\n", + " \"document_results\": \"This will be the RAG'd document text which will be returned from the search query.\"\n", + " },\n", + " \"summary\": {},\n", + " }\n", + "}\n", + "response = requests.post(\n", + " url_end, headers=headers, json=payload_child_end, auth=(\"api\", wandb_token)\n", + ")\n", + "if response.status_code == 200:\n", + " print(\"Child call ended.\")\n", + "else:\n", + " print(\"Child end request failed with status:\", response.status_code)\n", + " print(response.text)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Add a child span to a complex trace: LLM completion call\n", + "\n", + "The following code demonstrates how to add another child span to the parent trace, representing an LLM completion call. This step models the AI's response generation based on document context retrieved in the previous RAG operation.\n", + "\n", + "The LLM completion trace is initiated with the `payload_child_start` object, which includes:\n", + "- `trace_id`: Links this child span to the parent trace.\n", + "- `parent_id`: Associates the child span with the overarching workflow.\n", + "- `inputs`: Logs the input messages for the LLM, including the user query and the appended document context.\n", + "- `model`: Specifies the model used for the operation (`gpt-4o`).\n", + "\n", + "On success, the code outputs a message indicating the LLM child span trace has started and ended:\n", + "\n", + "```\n", + "Child call started. ID: 0245acdf-83a9-4c90-90df-dcb2b89f234a\n", + "```\n", + "\n", + "Once the operation completes, the `payload_child_end` object ends the trace by logging the LLM-generated response in the `output` field. Usage summary information is also logged.\n", + "\n", + "On success, the code outputs a message indicating the LLM child span trace has started and ended:\n", + "\n", + "```\n", + "Child call started. ID: 0245acdf-83a9-4c90-90df-dcb2b89f234a\n", + "Child call ended.\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "7-IpjOPp1amg" + }, + "outputs": [], + "source": [ + "## ------------\n", + "## Child span:\n", + "## Create an LLM completion call\n", + "## ------------\n", + "\n", + "# Child call: Start\n", + "payload_child_start = {\n", + " \"start\": {\n", + " \"project_id\": f\"{team_id}/{project_id}\",\n", + " \"op_name\": \"llm_completion\",\n", + " \"trace_id\": trace_id,\n", + " \"parent_id\": parent_call_id,\n", + " \"started_at\": datetime.datetime.now().isoformat(),\n", + " \"inputs\": {\n", + " \"messages\": [{\"role\":\"user\",\"content\":\"With the following document context, could you help me answer:\\n Can you summarize the key points of this document?\\n [+ appended document context]\"}],\n", + " \"model\": \"gpt-4o\"\n", + " },\n", + " \"attributes\": {},\n", + " }\n", + "}\n", + "response = requests.post(\n", + " url_start, headers=headers, json=payload_child_start, auth=(\"api\", wandb_token)\n", + ")\n", + "if response.status_code == 200:\n", + " data = response.json()\n", + " child_call_id = data.get(\"id\")\n", + " print(f\"Child call started. ID: {child_call_id}\")\n", + "else:\n", + " print(\"Child start request failed with status:\", response.status_code)\n", + " print(response.text)\n", + " exit()\n", + "\n", + "# Child call: End\n", + "payload_child_end = {\n", + " \"end\": {\n", + " \"project_id\": f\"{team_id}/{project_id}\",\n", + " \"id\": child_call_id,\n", + " \"ended_at\": datetime.datetime.now().isoformat(),\n", + " \"output\": {\n", + " \"choices\": [\n", + " {\"message\": {\"content\": \"This is the response generated by the LLM.\"}},\n", + " ]\n", + " },\n", + " \"summary\": {\n", + " \"usage\": {\n", + " \"gpt-4o\": {\n", + " \"prompt_tokens\": 10,\n", + " \"completion_tokens\": 20,\n", + " \"total_tokens\": 30,\n", + " \"requests\": 1,\n", + " }\n", + " }\n", + " },\n", + " }\n", + "}\n", + "response = requests.post(\n", + " url_end, headers=headers, json=payload_child_end, auth=(\"api\", wandb_token)\n", + ")\n", + "if response.status_code == 200:\n", + " print(\"Child call ended.\")\n", + "else:\n", + " print(\"Child end request failed with status:\", response.status_code)\n", + " print(response.text)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### End a complex trace\n", + "\n", + "The following code demonstrates how to finalize the parent trace, marking the completion of the entire workflow. This step aggregates the results of all child spans (e.g., RAG lookup and LLM completion) and logs the final output and metadata.\n", + "\n", + "The trace is finalized using the `payload_parent_end` object, which includes:\n", + "- `id`: The `parent_call_id` from the initial parent trace start.\n", + "- `output`: Represents the final output of the entire workflow. \n", + "- `summary`: Consolidates usage data for the entire workflow.\n", + "- `prompt_tokens`: Total tokens used for all prompts.\n", + "- `completion_tokens`: Total tokens generated in all responses.\n", + "- `total_tokens`: Combined token count for the workflow.\n", + "- `requests`: Total number of requests made (in this case, `1`).\n", + "\n", + "On success, the code outputs:\n", + "\n", + "```\n", + "Parent call ended.\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "kNXNwHO8zdJn" + }, + "outputs": [], + "source": [ + "## ------------\n", + "## End trace\n", + "## ------------\n", + "\n", + "# Parent call: End\n", + "payload_parent_end = {\n", + " \"end\": {\n", + " \"project_id\": f\"{team_id}/{project_id}\",\n", + " \"id\": parent_call_id,\n", + " \"ended_at\": datetime.datetime.now().isoformat(),\n", + " \"output\": {\n", + " \"choices\": [\n", + " {\"message\": {\"content\": \"This is the response generated by the LLM.\"}},\n", + " ]\n", + " },\n", + " \"summary\": {\n", + " \"usage\": {\n", + " \"gpt-4o\": {\n", + " \"prompt_tokens\": 10,\n", + " \"completion_tokens\": 20,\n", + " \"total_tokens\": 30,\n", + " \"requests\": 1,\n", + " }\n", + " }\n", + " },\n", + " }\n", + "}\n", + "response = requests.post(\n", + " url_end, headers=headers, json=payload_parent_end, auth=(\"api\", wandb_token)\n", + ")\n", + "if response.status_code == 200:\n", + " print(\"Parent call ended.\")\n", + "else:\n", + " print(\"Parent end request failed with status:\", response.status_code)\n", + " print(response.text)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XU9QNlMrCx-j" + }, + "source": [ + "## Run a lookup query\n", + "The following code demonstrates how to query the traces created in previous examples, filtering only for traces where the `inputs.model` field is equal to `gpt-4o`.\n", + "\n", + "The `query_payload` object includes:\n", + "- `project_id`: Identifies the team and project to query.\n", + "- `filter`: Ensures the query returns only **trace roots** (top-level traces).\n", + "- `query`: Defines the filter logic using the `$expr` operator:\n", + " - `$getField`: Retrieves the `inputs.model` field.\n", + " - `$literal`: Matches traces where `inputs.model` equals `\"gpt-4o\"`.\n", + "- `limit`: Limits the query to 10,000 results.\n", + "- `offset`: Starts the query at the first result.\n", + "- `sort_by`: Orders results by the `started_at` timestamp in descending order.\n", + "- `include_feedback`: Excludes feedback data from the results.\n", + "\n", + "On a successful query, the response will include trace data matching the query parameters:\n", + "\n", + "```\n", + "{'id': '01939cf3-541f-76d3-ade3-50cfae068b39', 'project_id': 'cool-new-team/uncategorized', 'op_name': 'simple_trace', 'display_name': None, 'trace_id': '01939cf3-541f-76d3-ade3-50d5cfabe2db', 'parent_id': None, 'started_at': '2024-12-06T17:10:12.590000Z', 'attributes': {}, 'inputs': {'messages': [{'role': 'user', 'content': 'Why is the sky blue?'}], 'model': 'gpt-4o'}, 'ended_at': '2024-12-06T17:47:08.553000Z', 'exception': None, 'output': {'choices': [{'message': {'content': 'It’s due to Rayleigh scattering, where shorter blue wavelengths of sunlight scatter in all directions.'}}]}, 'summary': {'usage': {'gpt-4o': {'prompt_tokens': 10, 'completion_tokens': 20, 'requests': 1, 'total_tokens': 30}}, 'weave': {'status': 'success', 'trace_name': 'simple_trace', 'latency_ms': 2215963}}, 'wb_user_id': 'VXNlcjoyMDk5Njc0', 'wb_run_id': None, 'deleted_at': None}\n", + "```\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "9AxhF3aIC1S5" + }, + "outputs": [], + "source": [ + "query_payload = {\n", + " \"project_id\": f\"{team_id}/{project_id}\",\n", + " \"filter\": {\"trace_roots_only\": True},\n", + " \"query\": {\n", + " \"$expr\": {\n", + " \"$eq\": [\n", + " {\"$getField\": \"inputs.model\"},\n", + " {\"$literal\": \"gpt-4o\"}\n", + " ]\n", + " }\n", + " },\n", + " \"limit\": 10000,\n", + " \"offset\": 0,\n", + " \"sort_by\": [{\"field\": \"started_at\", \"direction\": \"desc\"}],\n", + " \"include_feedback\": False\n", + "}\n", + "response = requests.post(\n", + " url_stream_query, headers=headers, json=query_payload, auth=(\"api\", wandb_token)\n", + ")\n", + "if response.status_code == 200:\n", + " print(\"Query successful!\")\n", + " try:\n", + " data = response.json()\n", + " print(data)\n", + " except json.JSONDecodeError as e:\n", + " # Alternate decode\n", + " json_objects = response.text.strip().split(\"\\n\")\n", + " parsed_data = [json.loads(obj) for obj in json_objects]\n", + " print(parsed_data)\n", + "else:\n", + " print(f\"Query failed with status code: {response.status_code}\")\n", + " print(response.text)" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} From fe6fe589bbae1d3e31826b602e8172eee0a26fb0 Mon Sep 17 00:00:00 2001 From: J2-D2-3PO <188380414+J2-D2-3PO@users.noreply.github.com> Date: Fri, 6 Dec 2024 12:35:09 -0700 Subject: [PATCH 2/4] Fix nb formatting with ruff? --- docs/notebooks/weave_via_service_api.ipynb | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/notebooks/weave_via_service_api.ipynb b/docs/notebooks/weave_via_service_api.ipynb index f6431cc1517d..fe80cc4a4ee3 100644 --- a/docs/notebooks/weave_via_service_api.ipynb +++ b/docs/notebooks/weave_via_service_api.ipynb @@ -60,6 +60,7 @@ "source": [ "import datetime\n", "import json\n", + "\n", "import requests\n", "\n", "# Headers and URLs\n", From fb6b0682da4223efab88c622773f0add9983f32d Mon Sep 17 00:00:00 2001 From: J2-D2-3PO <188380414+J2-D2-3PO@users.noreply.github.com> Date: Thu, 12 Dec 2024 18:33:24 -0700 Subject: [PATCH 3/4] Typos, lint fix? --- .../gen_notebooks/weave_via_service_api.md | 2 +- docs/notebooks/weave_via_service_api.ipynb | 378 +++++++++--------- 2 files changed, 191 insertions(+), 189 deletions(-) diff --git a/docs/docs/reference/gen_notebooks/weave_via_service_api.md b/docs/docs/reference/gen_notebooks/weave_via_service_api.md index c2bdc350c61f..6b878f35c51a 100644 --- a/docs/docs/reference/gen_notebooks/weave_via_service_api.md +++ b/docs/docs/reference/gen_notebooks/weave_via_service_api.md @@ -60,7 +60,7 @@ wandb_token="" ## Simple trace The following sections walk you through creating a simple trace. -1. [Start a complex trace](#start-a-complex-trace) +1. [Start a complex trace](#start-a-simple-trace) 2. [End a simple trace](#end-a-simple-trace) ### Start a simple trace diff --git a/docs/notebooks/weave_via_service_api.ipynb b/docs/notebooks/weave_via_service_api.ipynb index fe80cc4a4ee3..0122e431ea15 100644 --- a/docs/notebooks/weave_via_service_api.ipynb +++ b/docs/notebooks/weave_via_service_api.ipynb @@ -70,9 +70,9 @@ "url_stream_query = \"https://trace.wandb.ai/calls/stream_query\"\n", "\n", "# W&B variables\n", - "team_id=\"\"\n", - "project_id=\"\"\n", - "wandb_token=\"\"" + "team_id = \"\"\n", + "project_id = \"\"\n", + "wandb_token = \"\"" ] }, { @@ -82,7 +82,7 @@ "## Simple trace\n", "The following sections walk you through creating a simple trace.\n", "\n", - "1. [Start a complex trace](#start-a-complex-trace)\n", + "1. [Start a simple trace](#start-a-simple-trace)\n", "2. [End a simple trace](#end-a-simple-trace)" ] }, @@ -127,30 +127,30 @@ "## Start trace\n", "## ------------\n", "payload_start = {\n", - " \"start\": {\n", - " \"project_id\": f\"{team_id}/{project_id}\",\n", - " \"op_name\": \"simple_trace\",\n", - " \"started_at\": datetime.datetime.now().isoformat(),\n", - " \"inputs\": {\n", - " # Use this \"messages\" style to generate the chat UI in the expanded trace.\n", - " \"messages\": [{\"role\":\"user\",\"content\":\"Why is the sky blue?\"}],\n", - " \"model\": \"gpt-4o\"\n", - " },\n", - " \"attributes\": {},\n", - " }\n", + " \"start\": {\n", + " \"project_id\": f\"{team_id}/{project_id}\",\n", + " \"op_name\": \"simple_trace\",\n", + " \"started_at\": datetime.datetime.now().isoformat(),\n", + " \"inputs\": {\n", + " # Use this \"messages\" style to generate the chat UI in the expanded trace.\n", + " \"messages\": [{\"role\": \"user\", \"content\": \"Why is the sky blue?\"}],\n", + " \"model\": \"gpt-4o\",\n", + " },\n", + " \"attributes\": {},\n", + " }\n", "}\n", "response = requests.post(\n", - " url_start, headers=headers, json=payload_start, auth=(\"api\", wandb_token)\n", + " url_start, headers=headers, json=payload_start, auth=(\"api\", wandb_token)\n", ")\n", "if response.status_code == 200:\n", - " data = response.json()\n", - " call_id = data.get(\"id\")\n", - " trace_id = data.get(\"trace_id\")\n", - " print(f\"Call started. ID: {call_id}, Trace ID: {trace_id}\")\n", + " data = response.json()\n", + " call_id = data.get(\"id\")\n", + " trace_id = data.get(\"trace_id\")\n", + " print(f\"Call started. ID: {call_id}, Trace ID: {trace_id}\")\n", "else:\n", - " print(\"Start request failed with status:\", response.status_code)\n", - " print(response.text)\n", - " exit()" + " print(\"Start request failed with status:\", response.status_code)\n", + " print(response.text)\n", + " exit()" ] }, { @@ -182,37 +182,41 @@ "## End trace\n", "## ------------\n", "payload_end = {\n", - " \"end\": {\n", - " \"project_id\": f\"{team_id}/{project_id}\",\n", - " \"id\": call_id,\n", - " \"ended_at\": datetime.datetime.now().isoformat(),\n", - " \"output\": {\n", - " # Use this \"choices\" style to add the completion to the chat UI in the expanded trace.\n", - " \"choices\": [\n", - " {\"message\": {\"content\": \"It’s due to Rayleigh scattering, where shorter blue wavelengths of sunlight scatter in all directions.\"}},\n", - " ]\n", - " },\n", - " # Format the summary like this to generate the pricing summary information in the traces table.\n", - " \"summary\": {\n", - " \"usage\": {\n", - " \"gpt-4o\": {\n", - " \"prompt_tokens\": 10,\n", - " \"completion_tokens\": 20,\n", - " \"total_tokens\": 30,\n", - " \"requests\": 1,\n", - " }\n", - " }\n", - " },\n", - " }\n", + " \"end\": {\n", + " \"project_id\": f\"{team_id}/{project_id}\",\n", + " \"id\": call_id,\n", + " \"ended_at\": datetime.datetime.now().isoformat(),\n", + " \"output\": {\n", + " # Use this \"choices\" style to add the completion to the chat UI in the expanded trace.\n", + " \"choices\": [\n", + " {\n", + " \"message\": {\n", + " \"content\": \"It’s due to Rayleigh scattering, where shorter blue wavelengths of sunlight scatter in all directions.\"\n", + " }\n", + " },\n", + " ]\n", + " },\n", + " # Format the summary like this to generate the pricing summary information in the traces table.\n", + " \"summary\": {\n", + " \"usage\": {\n", + " \"gpt-4o\": {\n", + " \"prompt_tokens\": 10,\n", + " \"completion_tokens\": 20,\n", + " \"total_tokens\": 30,\n", + " \"requests\": 1,\n", + " }\n", + " }\n", + " },\n", + " }\n", "}\n", "response = requests.post(\n", - " url_end, headers=headers, json=payload_end, auth=(\"api\", wandb_token)\n", + " url_end, headers=headers, json=payload_end, auth=(\"api\", wandb_token)\n", ")\n", "if response.status_code == 200:\n", - " print(\"Call ended.\")\n", + " print(\"Call ended.\")\n", "else:\n", - " print(\"End request failed with status:\", response.status_code)\n", - " print(response.text)" + " print(\"End request failed with status:\", response.status_code)\n", + " print(response.text)" ] }, { @@ -261,28 +265,26 @@ "\n", "# Parent call: Start\n", "payload_parent_start = {\n", - " \"start\": {\n", - " \"project_id\": f\"{team_id}/{project_id}\",\n", - " \"op_name\": \"complex_trace\",\n", - " \"started_at\": datetime.datetime.now().isoformat(),\n", - " \"inputs\": {\n", - " \"question\": \"Can you summarize the key points of this document?\"\n", - " },\n", - " \"attributes\": {},\n", - " }\n", + " \"start\": {\n", + " \"project_id\": f\"{team_id}/{project_id}\",\n", + " \"op_name\": \"complex_trace\",\n", + " \"started_at\": datetime.datetime.now().isoformat(),\n", + " \"inputs\": {\"question\": \"Can you summarize the key points of this document?\"},\n", + " \"attributes\": {},\n", + " }\n", "}\n", "response = requests.post(\n", - " url_start, headers=headers, json=payload_parent_start, auth=(\"api\", wandb_token)\n", + " url_start, headers=headers, json=payload_parent_start, auth=(\"api\", wandb_token)\n", ")\n", "if response.status_code == 200:\n", - " data = response.json()\n", - " parent_call_id = data.get(\"id\")\n", - " trace_id = data.get(\"trace_id\")\n", - " print(f\"Parent call started. ID: {parent_call_id}, Trace ID: {trace_id}\")\n", + " data = response.json()\n", + " parent_call_id = data.get(\"id\")\n", + " trace_id = data.get(\"trace_id\")\n", + " print(f\"Parent call started. ID: {parent_call_id}, Trace ID: {trace_id}\")\n", "else:\n", - " print(\"Parent start request failed with status:\", response.status_code)\n", - " print(response.text)\n", - " exit()" + " print(\"Parent start request failed with status:\", response.status_code)\n", + " print(response.text)\n", + " exit()" ] }, { @@ -322,50 +324,50 @@ "\n", "# Child call: Start\n", "payload_child_start = {\n", - " \"start\": {\n", - " \"project_id\": f\"{team_id}/{project_id}\",\n", - " \"op_name\": \"rag_document_lookup\",\n", - " \"trace_id\": trace_id,\n", - " \"parent_id\": parent_call_id,\n", - " \"started_at\": datetime.datetime.now().isoformat(),\n", - " \"inputs\": {\n", - " \"document_search\": \"This is a search query of the documents I'm looking for.\"\n", - " },\n", - " \"attributes\": {},\n", - " }\n", + " \"start\": {\n", + " \"project_id\": f\"{team_id}/{project_id}\",\n", + " \"op_name\": \"rag_document_lookup\",\n", + " \"trace_id\": trace_id,\n", + " \"parent_id\": parent_call_id,\n", + " \"started_at\": datetime.datetime.now().isoformat(),\n", + " \"inputs\": {\n", + " \"document_search\": \"This is a search query of the documents I'm looking for.\"\n", + " },\n", + " \"attributes\": {},\n", + " }\n", "}\n", "response = requests.post(\n", - " url_start, headers=headers, json=payload_child_start, auth=(\"api\", wandb_token)\n", + " url_start, headers=headers, json=payload_child_start, auth=(\"api\", wandb_token)\n", ")\n", "if response.status_code == 200:\n", - " data = response.json()\n", - " child_call_id = data.get(\"id\")\n", - " print(f\"Child call started. ID: {child_call_id}\")\n", + " data = response.json()\n", + " child_call_id = data.get(\"id\")\n", + " print(f\"Child call started. ID: {child_call_id}\")\n", "else:\n", - " print(\"Child start request failed with status:\", response.status_code)\n", - " print(response.text)\n", - " exit()\n", + " print(\"Child start request failed with status:\", response.status_code)\n", + " print(response.text)\n", + " exit()\n", "\n", "# Child call: End\n", "payload_child_end = {\n", - " \"end\": {\n", - " \"project_id\": f\"{team_id}/{project_id}\",\n", - " \"id\": child_call_id,\n", - " \"ended_at\": datetime.datetime.now().isoformat(),\n", - " \"output\": {\n", - " \"document_results\": \"This will be the RAG'd document text which will be returned from the search query.\"\n", - " },\n", - " \"summary\": {},\n", - " }\n", + " \"end\": {\n", + " \"project_id\": f\"{team_id}/{project_id}\",\n", + " \"id\": child_call_id,\n", + " \"ended_at\": datetime.datetime.now().isoformat(),\n", + " \"output\": {\n", + " \"document_results\": \"This will be the RAG'd document text which will be returned from the search query.\"\n", + " },\n", + " \"summary\": {},\n", + " }\n", "}\n", "response = requests.post(\n", - " url_end, headers=headers, json=payload_child_end, auth=(\"api\", wandb_token)\n", + " url_end, headers=headers, json=payload_child_end, auth=(\"api\", wandb_token)\n", ")\n", "if response.status_code == 200:\n", - " print(\"Child call ended.\")\n", + " print(\"Child call ended.\")\n", "else:\n", - " print(\"Child end request failed with status:\", response.status_code)\n", - " print(response.text)" + " print(\"Child end request failed with status:\", response.status_code)\n", + " print(response.text)" ] }, { @@ -413,62 +415,67 @@ "\n", "# Child call: Start\n", "payload_child_start = {\n", - " \"start\": {\n", - " \"project_id\": f\"{team_id}/{project_id}\",\n", - " \"op_name\": \"llm_completion\",\n", - " \"trace_id\": trace_id,\n", - " \"parent_id\": parent_call_id,\n", - " \"started_at\": datetime.datetime.now().isoformat(),\n", - " \"inputs\": {\n", - " \"messages\": [{\"role\":\"user\",\"content\":\"With the following document context, could you help me answer:\\n Can you summarize the key points of this document?\\n [+ appended document context]\"}],\n", - " \"model\": \"gpt-4o\"\n", - " },\n", - " \"attributes\": {},\n", - " }\n", + " \"start\": {\n", + " \"project_id\": f\"{team_id}/{project_id}\",\n", + " \"op_name\": \"llm_completion\",\n", + " \"trace_id\": trace_id,\n", + " \"parent_id\": parent_call_id,\n", + " \"started_at\": datetime.datetime.now().isoformat(),\n", + " \"inputs\": {\n", + " \"messages\": [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"With the following document context, could you help me answer:\\n Can you summarize the key points of this document?\\n [+ appended document context]\",\n", + " }\n", + " ],\n", + " \"model\": \"gpt-4o\",\n", + " },\n", + " \"attributes\": {},\n", + " }\n", "}\n", "response = requests.post(\n", - " url_start, headers=headers, json=payload_child_start, auth=(\"api\", wandb_token)\n", + " url_start, headers=headers, json=payload_child_start, auth=(\"api\", wandb_token)\n", ")\n", "if response.status_code == 200:\n", - " data = response.json()\n", - " child_call_id = data.get(\"id\")\n", - " print(f\"Child call started. ID: {child_call_id}\")\n", + " data = response.json()\n", + " child_call_id = data.get(\"id\")\n", + " print(f\"Child call started. ID: {child_call_id}\")\n", "else:\n", - " print(\"Child start request failed with status:\", response.status_code)\n", - " print(response.text)\n", - " exit()\n", + " print(\"Child start request failed with status:\", response.status_code)\n", + " print(response.text)\n", + " exit()\n", "\n", "# Child call: End\n", "payload_child_end = {\n", - " \"end\": {\n", - " \"project_id\": f\"{team_id}/{project_id}\",\n", - " \"id\": child_call_id,\n", - " \"ended_at\": datetime.datetime.now().isoformat(),\n", - " \"output\": {\n", - " \"choices\": [\n", - " {\"message\": {\"content\": \"This is the response generated by the LLM.\"}},\n", - " ]\n", - " },\n", - " \"summary\": {\n", - " \"usage\": {\n", - " \"gpt-4o\": {\n", - " \"prompt_tokens\": 10,\n", - " \"completion_tokens\": 20,\n", - " \"total_tokens\": 30,\n", - " \"requests\": 1,\n", - " }\n", - " }\n", - " },\n", - " }\n", + " \"end\": {\n", + " \"project_id\": f\"{team_id}/{project_id}\",\n", + " \"id\": child_call_id,\n", + " \"ended_at\": datetime.datetime.now().isoformat(),\n", + " \"output\": {\n", + " \"choices\": [\n", + " {\"message\": {\"content\": \"This is the response generated by the LLM.\"}},\n", + " ]\n", + " },\n", + " \"summary\": {\n", + " \"usage\": {\n", + " \"gpt-4o\": {\n", + " \"prompt_tokens\": 10,\n", + " \"completion_tokens\": 20,\n", + " \"total_tokens\": 30,\n", + " \"requests\": 1,\n", + " }\n", + " }\n", + " },\n", + " }\n", "}\n", "response = requests.post(\n", - " url_end, headers=headers, json=payload_child_end, auth=(\"api\", wandb_token)\n", + " url_end, headers=headers, json=payload_child_end, auth=(\"api\", wandb_token)\n", ")\n", "if response.status_code == 200:\n", - " print(\"Child call ended.\")\n", + " print(\"Child call ended.\")\n", "else:\n", - " print(\"Child end request failed with status:\", response.status_code)\n", - " print(response.text)" + " print(\"Child end request failed with status:\", response.status_code)\n", + " print(response.text)" ] }, { @@ -509,35 +516,35 @@ "\n", "# Parent call: End\n", "payload_parent_end = {\n", - " \"end\": {\n", - " \"project_id\": f\"{team_id}/{project_id}\",\n", - " \"id\": parent_call_id,\n", - " \"ended_at\": datetime.datetime.now().isoformat(),\n", - " \"output\": {\n", - " \"choices\": [\n", - " {\"message\": {\"content\": \"This is the response generated by the LLM.\"}},\n", - " ]\n", - " },\n", - " \"summary\": {\n", - " \"usage\": {\n", - " \"gpt-4o\": {\n", - " \"prompt_tokens\": 10,\n", - " \"completion_tokens\": 20,\n", - " \"total_tokens\": 30,\n", - " \"requests\": 1,\n", - " }\n", - " }\n", - " },\n", - " }\n", + " \"end\": {\n", + " \"project_id\": f\"{team_id}/{project_id}\",\n", + " \"id\": parent_call_id,\n", + " \"ended_at\": datetime.datetime.now().isoformat(),\n", + " \"output\": {\n", + " \"choices\": [\n", + " {\"message\": {\"content\": \"This is the response generated by the LLM.\"}},\n", + " ]\n", + " },\n", + " \"summary\": {\n", + " \"usage\": {\n", + " \"gpt-4o\": {\n", + " \"prompt_tokens\": 10,\n", + " \"completion_tokens\": 20,\n", + " \"total_tokens\": 30,\n", + " \"requests\": 1,\n", + " }\n", + " }\n", + " },\n", + " }\n", "}\n", "response = requests.post(\n", - " url_end, headers=headers, json=payload_parent_end, auth=(\"api\", wandb_token)\n", + " url_end, headers=headers, json=payload_parent_end, auth=(\"api\", wandb_token)\n", ")\n", "if response.status_code == 200:\n", - " print(\"Parent call ended.\")\n", + " print(\"Parent call ended.\")\n", "else:\n", - " print(\"Parent end request failed with status:\", response.status_code)\n", - " print(response.text)" + " print(\"Parent end request failed with status:\", response.status_code)\n", + " print(response.text)" ] }, { @@ -578,34 +585,29 @@ "outputs": [], "source": [ "query_payload = {\n", - " \"project_id\": f\"{team_id}/{project_id}\",\n", - " \"filter\": {\"trace_roots_only\": True},\n", - " \"query\": {\n", - " \"$expr\": {\n", - " \"$eq\": [\n", - " {\"$getField\": \"inputs.model\"},\n", - " {\"$literal\": \"gpt-4o\"}\n", - " ]\n", - " }\n", - " },\n", - " \"limit\": 10000,\n", - " \"offset\": 0,\n", - " \"sort_by\": [{\"field\": \"started_at\", \"direction\": \"desc\"}],\n", - " \"include_feedback\": False\n", + " \"project_id\": f\"{team_id}/{project_id}\",\n", + " \"filter\": {\"trace_roots_only\": True},\n", + " \"query\": {\n", + " \"$expr\": {\"$eq\": [{\"$getField\": \"inputs.model\"}, {\"$literal\": \"gpt-4o\"}]}\n", + " },\n", + " \"limit\": 10000,\n", + " \"offset\": 0,\n", + " \"sort_by\": [{\"field\": \"started_at\", \"direction\": \"desc\"}],\n", + " \"include_feedback\": False,\n", "}\n", "response = requests.post(\n", - " url_stream_query, headers=headers, json=query_payload, auth=(\"api\", wandb_token)\n", + " url_stream_query, headers=headers, json=query_payload, auth=(\"api\", wandb_token)\n", ")\n", "if response.status_code == 200:\n", - " print(\"Query successful!\")\n", - " try:\n", - " data = response.json()\n", - " print(data)\n", - " except json.JSONDecodeError as e:\n", - " # Alternate decode\n", - " json_objects = response.text.strip().split(\"\\n\")\n", - " parsed_data = [json.loads(obj) for obj in json_objects]\n", - " print(parsed_data)\n", + " print(\"Query successful!\")\n", + " try:\n", + " data = response.json()\n", + " print(data)\n", + " except json.JSONDecodeError as e:\n", + " # Alternate decode\n", + " json_objects = response.text.strip().split(\"\\n\")\n", + " parsed_data = [json.loads(obj) for obj in json_objects]\n", + " print(parsed_data)\n", "else:\n", " print(f\"Query failed with status code: {response.status_code}\")\n", " print(response.text)" From 1481232c9a6645a8e50c6de5f7e2fb3028cc8b5c Mon Sep 17 00:00:00 2001 From: J2-D2-3PO <188380414+J2-D2-3PO@users.noreply.github.com> Date: Thu, 12 Dec 2024 18:35:26 -0700 Subject: [PATCH 4/4] typo --- docs/docs/reference/gen_notebooks/weave_via_service_api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/reference/gen_notebooks/weave_via_service_api.md b/docs/docs/reference/gen_notebooks/weave_via_service_api.md index 6b878f35c51a..5081e8923b46 100644 --- a/docs/docs/reference/gen_notebooks/weave_via_service_api.md +++ b/docs/docs/reference/gen_notebooks/weave_via_service_api.md @@ -60,7 +60,7 @@ wandb_token="" ## Simple trace The following sections walk you through creating a simple trace. -1. [Start a complex trace](#start-a-simple-trace) +1. [Start a simple trace](#start-a-simple-trace) 2. [End a simple trace](#end-a-simple-trace) ### Start a simple trace