From e5935f633569f013d406e91864eacb0d3737b2b8 Mon Sep 17 00:00:00 2001 From: C9luster <138663536+C9luster@users.noreply.github.com> Date: Wed, 4 Sep 2024 10:41:21 +0800 Subject: [PATCH] =?UTF-8?q?Appbuilder-SDK=E7=BB=84=E4=BB=B6=EF=BC=88compon?= =?UTF-8?q?ents=EF=BC=89=E9=83=A8=E5=88=86=E6=B7=BB=E5=8A=A0=E5=8D=95?= =?UTF-8?q?=E6=B5=8B=E4=BB=A5=E5=8F=8A=E5=8D=95=E6=B5=8B=E6=A1=86=E6=9E=B6?= =?UTF-8?q?=20(#457)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test-for-tool-eval * update * update * update * update * update * update * update * update * update * update * update * update * update * update-test * Add vscode setting to simplify local vscode develop (#484) * Add vscode setting to simplify local vscode develop * Add golang test discovery * 更新Appbuilder-SDK代码库CookBook (#485) * 更行Appbuilder-SDK代码库CookBook * 更新Appbuilder-SDK代码库CookBook * 更新Appbuilder-SDK代码库CookBook * update * update * update --------- Co-authored-by: yinjiaqi * update base code (#486) * 添加tool_call的cook_book (#487) * 更行Appbuilder-SDK代码库CookBook * 更新Appbuilder-SDK代码库CookBook * 更新Appbuilder-SDK代码库CookBook * update * update * update * tool_call_cook_book * update * update --------- Co-authored-by: yinjiaqi * 更新CookBook (#488) Co-authored-by: yinjiaqi * 更新SDK Cookbook (#490) * update * update --------- Co-authored-by: yinjiaqi * 更新cookbook图片链接 (#492) Co-authored-by: yinjiaqi * update * update * update * update * update * update * update * update * update * update * 新增流水线打印组件检测结果 * update --------- Co-authored-by: yinjiaqi Co-authored-by: wolvever Co-authored-by: Chengmo --- .github/workflows/python-package.yml | 3 +- appbuilder/__init__.py | 5 + appbuilder/core/_exception.py | 6 + appbuilder/tests/component_test.py | 296 ++++++++++++++++++ .../tests/print_components_error_info.py | 53 ++++ appbuilder/tests/run_python_test.sh | 7 + .../tests/test_appbuilder_assistant_trace.py | 2 +- .../tests/test_appbuilder_client_app_list.py | 2 +- ...der_client_toolcall_event_handler_error.py | 2 +- .../tests/test_core_commponents_tool_eval.py | 184 +++++++++++ appbuilder/tests/test_core_componentstest.py | 104 ++++++ appbuilder/tests/test_qa_aicape_animal_rec.py | 3 +- .../tests/test_qa_aicape_doc_crop_enhance.py | 4 +- .../tests/test_qa_aicape_handwriting_ocr.py | 3 +- .../tests/test_qa_aicape_image_understand.py | 3 +- .../tests/test_qa_aicape_mixcard_ocr.py | 4 +- appbuilder/tests/test_qa_aicape_plant_rec.py | 3 +- appbuilder/tests/test_qa_aicape_qrcode_orc.py | 3 +- appbuilder/tests/test_qa_aicape_table_ocr.py | 3 +- ...st_qa_doc_parser_extract_table_from_doc.py | 1 + .../tests/test_qa_llm_dialog_summary.py | 3 +- .../test_qa_llm_get_qianfan_model_list.py | 1 + .../tests/test_qa_llm_is_complex_query.py | 3 +- appbuilder/tests/test_qa_llm_matching.py | 3 +- .../test_qa_llm_oral_query_generation.py | 3 +- .../tests/test_qa_llm_paddle_speech_tts.py | 4 +- appbuilder/tests/test_qa_llm_pandas.py | 3 +- .../tests/test_qa_llm_query_decomposition.py | 4 +- appbuilder/tests/test_qa_llm_style_rewrite.py | 3 +- appbuilder/tests/test_qa_pair_mining.py | 1 + appbuilder/tests/whitelist_components.txt | 13 + 31 files changed, 710 insertions(+), 22 deletions(-) create mode 100644 appbuilder/tests/component_test.py create mode 100644 appbuilder/tests/print_components_error_info.py create mode 100644 appbuilder/tests/test_core_commponents_tool_eval.py create mode 100644 appbuilder/tests/test_core_componentstest.py create mode 100644 appbuilder/tests/whitelist_components.txt diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 64efc78ef..01924f5b4 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -44,7 +44,7 @@ jobs: APPBUILDER_TOKEN_V2: bce-v3/ALTAK-zX2OwTWGE9JxXSKxcBYQp/7dd073d9129c01c617ef76d8b7220a74835eb2f4 BAIDU_VDB_API_KEY: apaasTest1 INSTANCE_ID: vdb-bj-vuzmppgqrnhv - DATASET_ID: 2626a842-132f-45ce-977f-f701d18fd104 + DATASET_ID: 56e82915-9642-4a03-bb02-74744c17863e APPBUILDER_TOKEN_DOC_FORMAT: bce-v3/ALTAK-bcKsgHd39g0Aaq3nCYUUQ/b06384229df1462c6fb011383d09230346a20ac4 strategy: fail-fast: false @@ -92,6 +92,7 @@ jobs: python3 -m pip install SQLAlchemy==2.0.31 python3 -m pip install chainlit~=1.0.200 flask~=2.3.2 flask-restful==0.3.9 python3 -m pip install opentelemetry-exporter-otlp==1.23.0 opentelemetry-instrumentation==0.44b0 opentelemetry-sdk==1.23.0 opentelemetry-api==1.23.0 + python3 -m pip install pandas==2.2.2 - name: Build whl run: | cd cicd/app-builder diff --git a/appbuilder/__init__.py b/appbuilder/__init__.py index 857011c99..68619946b 100644 --- a/appbuilder/__init__.py +++ b/appbuilder/__init__.py @@ -113,6 +113,8 @@ def get_default_header(): from .core.components.image_understand.component import ImageUnderstand from .core.components.mix_card_ocr.component import MixCardOCR +from .tests.component_test import AppbuilderTestToolEval, AutomaticTestToolEval + from appbuilder.core.message import Message from appbuilder.core.agent import AgentRuntime from appbuilder.core.user_session import UserSession @@ -205,6 +207,9 @@ def get_default_header(): 'PPTGenerationFromPaper', 'PPTGenerationFromFile', + 'AppbuilderTestToolEval', + 'AutomaticTestToolEval', + "get_model_list", "AppBuilderClient", diff --git a/appbuilder/core/_exception.py b/appbuilder/core/_exception.py index 3fc5d155b..1eb0fccff 100644 --- a/appbuilder/core/_exception.py +++ b/appbuilder/core/_exception.py @@ -110,3 +110,9 @@ class RiskInputException(BaseRPCException): r"""RiskInputException """ pass + + +class AppbuilderBuildexException(BaseRPCException): + r"""AppbuilderBuildxException + """ + pass diff --git a/appbuilder/tests/component_test.py b/appbuilder/tests/component_test.py new file mode 100644 index 000000000..7f4a141e0 --- /dev/null +++ b/appbuilder/tests/component_test.py @@ -0,0 +1,296 @@ +import requests +import types +import re +import inspect + +from typing import TypeVar, Generic, Union, Type +from appbuilder.core._exception import * +from unittest.mock import Mock +from appbuilder.core import components +from appbuilder.core._session import InnerSession + + +Data_Type = { + 'string': str, + 'integer': int, + 'object': int, + 'array': list, + 'boolean': bool, + 'null': None, +} + +class AppbuilderTestToolEval: + """ + 功能:Components组件模拟post本地运行。 + + 使用方法： + + ```python + # 实例化一个 + image_understand = appbuilder.ImageUnderstand() + + # 设计一个符合规范的tool_eval input(dict数据类型) + tool_eval_input = { + 'streaming': True, + 'traceid': 'traceid', + 'name':"image_understand", + 'img_url':'img_url_str', + 'origin_query':"" + } + + # 设计一个组件API接口预期的response + mock_response_data = { + 'result': {'task_id': '1821485837570181996'}, + 'log_id': 1821485837570181996, + } + mock_response = Mock() + mock_response.status_code = 200 + mock_response.headers = {'Content-Type': 'application/json'} + def mock_json(): + return mock_response_data + mock_response.json = mock_json + + # 实例化一个AppbuilderTestToolEval对象,实现组件本地的自动化测试 + appbuilder.AppbuilderTestToolEval(appbuilder_components=image_understand, + tool_eval_input=tool_eval_input, + response=mock_response) + ``` + """ + def __init__(self, appbuilder_components:components, tool_eval_input:dict, response:requests.Response): + """ + 初始化函数。 + + Args: + appbuilder_components (components): 应用构建器组件对象。 + tool_eval_input (dict): tool_eval的传入参数。 + response (dict): api预期的response返回值。 + + Returns: + None + """ + self.component = appbuilder_components + self.tool_eval_input = tool_eval_input + self.response = response + self.test_manifests() + self.test_tool_eval_input() + self.test_tool_eval_generator() + if hasattr(self.component, '__module__'): + module_name = self.component.__module__ + if re.match(r'appbuilder\.', module_name): + self.test_tool_eval_reponse_raise() + self.test_tool_eval_text_str() + + def test_manifests(self): + """ + 校验组件成员变量manifests是否符合规范。 + + Args: + 无参数。 + + Returns: + 无返回值。 + Raises: + AppbuilderBuildexException: 校验不通过时抛出异常。 + """ + manifests = self.component.manifests + try: + assert isinstance(manifests, list) + assert len(manifests) > 0 + assert isinstance(manifests[0],dict) + assert isinstance(manifests[0]['name'], str) + assert isinstance(manifests[0]['description'], str) + assert isinstance(manifests[0]['parameters'], dict) + except Exception as e: + raise AppbuilderBuildexException(f'请检查{self.component}组件是否存在成员变量manifests或manifests成员变量定义规范, 错误信息：{e}') + + def test_tool_eval_input(self): + """ + 校验tool_eval的传入参数是否合法。 + + Args: + 无参数。 + + Returns: + 无返回值。 + + Raises: + AppbuilderBuildexException: 校验不通过时抛出异常。 + + """ + if not self.tool_eval_input.get('streaming',None): + raise AppbuilderBuildexException(f'请检查{self.component}组件tool_eval的传入参数是否定义streaming') + if hasattr(self.component, '__module__'): + module_name = self.component.__module__ + if re.match(r'appbuilder\.', module_name): + if not self.tool_eval_input.get('traceid',None): + raise AppbuilderBuildexException(f'请检查{self.component}组件tool_eval的传入参数是否有traceid') + try: + manifests = self.component.manifests[0] + parameters = manifests['parameters'] + properties = parameters['properties'] + except: + raise AppbuilderBuildexException(f'请检查{self.component}组件是否存在成员变量manifests或manifests成员变量定义规范') + anyOf = parameters.get('anyOf',None) + if anyOf: + anyOf_test = False + for anyOf_requried_dict in anyOf: + anyOf_requried = anyOf_requried_dict.get('required',None) + if anyOf_requried: + success_number = 0 + for anyOf_requried_data in anyOf_requried: + try: + input_data = self.tool_eval_input[anyOf_requried_data] + input_data_type = Data_Type[properties[anyOf_requried_data]['type']] + if anyOf_requried_data in self.tool_eval_input and isinstance(input_data, input_data_type): + success_number += 1 + except: + pass + if success_number == len(anyOf_requried): + anyOf_test = True + if not anyOf_test: + raise AppbuilderBuildexException(f'请检查{self.component}组件tool_eval的传入参数是否正确或manifests的参数定义是否正确') + + if not anyOf: + un_anyOf_test = False + requried = parameters.get('required',None) + if requried: + success_number = 0 + for requried_data in requried: + try: + input_data = self.tool_eval_input[requried_data] + input_data_type = Data_Type[properties[requried_data]['type']] + if requried_data in self.tool_eval_input and isinstance(input_data, input_data_type): + success_number += 1 + except: + pass + if success_number == len(requried): + un_anyOf_test = True + if not un_anyOf_test: + raise AppbuilderBuildexException(f'请检查{self.component}组件tool_eval的传入参数是否正确或manifests的参数定义是否正确') + + def test_tool_eval_reponse_raise(self): + """ + Args: + 无参数 + + Returns: + 无返回值 + + Raises: + AppbuilderBuildexException: 如果响应头状态码对应的异常类型与捕获到的异常类型不一致，则抛出此异常。 + + 功能：测试tool_eval方法在不同响应头状态码下的异常抛出情况。 + + 首先，设置响应头状态码为bad_request，并模拟InnerSession.post方法的返回值。 + 然后，定义一个状态码与异常类型的映射字典test_status_code_dict，用于测试不同状态码下抛出的异常类型是否正确。 + 接着，遍历test_status_code_dict字典，将状态码和异常类型分别赋值给self.response.status_code和error变量，并重新模拟InnerSession.post方法的返回值。 + 在每次循环中，调用self.component.tool_eval方法，并捕获可能抛出的异常。 + 如果捕获到的异常类型与test_status_code_dict字典中对应状态码的异常类型一致，则继续下一次循环； + 否则，抛出AppbuilderBuildexException异常，提示用户检查self.component组件tool_eval方法的response返回值是否添加了check_response_header检测。 + """ + # test_response_head_status + self.response.status_code = requests.codes.bad_request + InnerSession.post = Mock(return_value=self.response) + test_status_code_dict = { + requests.codes.bad_request: BadRequestException, + requests.codes.forbidden: ForbiddenException, + requests.codes.not_found: ForbiddenException, + requests.codes.precondition_required: PreconditionFailedException, + requests.codes.internal_server_error: InternalServerErrorException + } + for status_code,error in test_status_code_dict.items(): + self.response.status_code = status_code + InnerSession.post = Mock(return_value=self.response) + try: + self.component.tool_eval(**self.tool_eval_input) + except Exception as e: + if isinstance(e,error): + raise AppbuilderBuildexException(f'请检查{self.component}组件tool_eval的response返回值是否添加check_response_header检测') + + def test_tool_eval_generator(self): + """ + 测试组件tool_eval方法返回是否为生成器 + + Args: + 无 + + Returns: + 无 + + Raises: + AppbuilderBuildexException: 如果组件tool_eval的返回值不为生成器时抛出异常 + """ + self.response.status_code = requests.codes.ok + InnerSession.post = Mock(return_value=self.response) + result_generator = self.component.tool_eval(**self.tool_eval_input) + if not result_generator: + raise AppbuilderBuildexException(f'请检查{self.component}组件tool_eval的返回值是否为生成器') + if not isinstance(result_generator, types.GeneratorType): + raise AppbuilderBuildexException(f'请检查{self.component}组件tool_eval的返回值是否为生成器') + + def test_tool_eval_text_str(self): + """ + 测试tool_eval方法返回值的文本是否为字符串类型 + + Args: + 无 + + Returns: + 无返回值，该函数主要进行断言测试 + + Raises: + AppbuilderBuildexException: 当tool_eval方法返回的文本不是字符串类型时抛出异常 + """ + self.response.status_code = requests.codes.ok + InnerSession.post = Mock(return_value=self.response) + result_generator = self.component.tool_eval(**self.tool_eval_input) + for res in result_generator: + if not isinstance(res.get("text",""),str): + raise AppbuilderBuildexException(f'请检查{self.component}组件tool_eval的返回值是否为字符串') + +class AutomaticTestToolEval: + def __init__(self, appbuilder_components:components): + self.components = appbuilder_components + self.test_input() + + def test_input(self): + manifest = self.components.manifests[0] + properties = manifest['parameters']['properties'] + required_params = [] + anyOf = manifest['parameters'].get('anyOf', None) + if anyOf: + for anyOf_dict in anyOf: + required_params += anyOf_dict['required'] + if not anyOf: + required_params += manifest['parameters']['required'] + required_param_dict = { + 'name':str, + 'streaming':bool + } + + for param in required_params: + required_param_dict[param] = Data_Type[properties[param]['type']] + required_params = [] + for param in required_param_dict.keys(): + required_params.append(param) + + # 交互检查 + tool_eval_input_params = [] + signature = inspect.signature(self.components.tool_eval) + for param_name, param in signature.parameters.items(): + if param_name == 'kwargs': + continue + if param_name in required_params: + if required_param_dict[param_name] == param.annotation: + tool_eval_input_params.append(param_name) + else: + raise AppbuilderBuildexException(f'请检查tool_eval的传入参数{param_name}是否符合成员变量manifest的参数类型要求') + else: + raise AppbuilderBuildexException(f'请检查tool_eval的传入参数{param_name}是否在成员变量manifest要求内') + + for required_param in required_params: + if required_param not in tool_eval_input_params: + raise AppbuilderBuildexException(f'请检查成员变量manifest要求的tool_eval的传入参数{required_param}是否在其中') + + + \ No newline at end of file diff --git a/appbuilder/tests/print_components_error_info.py b/appbuilder/tests/print_components_error_info.py new file mode 100644 index 000000000..2135ca23a --- /dev/null +++ b/appbuilder/tests/print_components_error_info.py @@ -0,0 +1,53 @@ +def pretty_print_dict(kv_dict, header=["Key", "Value"]): + spacing = 2 + max_k = 25 + max_v = 80 + + for k, v in kv_dict.items(): + max_k = max(max_k, len(k)) + + h_format = " " + "{{:^{}s}}{}{{:^{}s}}\n".format(max_k, " " * spacing, + max_v) + l_format = " " + "{{:^{}s}}{{}}{{:<{}s}}\n".format(max_k, max_v) + length = max_k + max_v + spacing + + front_border = " ╔" + "".join(["═"] * length) + "╗" + line = " ╠" + "".join(["═"] * length) + "╣" + back_border = " ╚" + "".join(["═"] * length) + "╝" + + draws = "" + draws += front_border + "\n" + + draws += h_format.format(header[0], header[1]) + + draws += line + "\n" + + for k, v in kv_dict.items(): + if isinstance(v, str) and len(v) >= max_v: + str_v = "... " + v[-46:] + else: + str_v = v + draws += l_format.format(k, " " * spacing, str(str_v)) + + draws += back_border + + _str = "\n{}\n".format(draws) + return _str + + +def read_error_file(filename): + kv_dict = {} + with open(filename, 'r', encoding='utf-8') as file: + lines = file.readlines() + header = lines[0].strip().split('\t') + for line in lines[1:-3]: + components = line.strip().split('\t') + if len(components) == 2: + kv_dict[components[0]] = components[1] + return kv_dict, header + + +if __name__ == "__main__": + filename = 'components_error_info.txt' + kv_dict, header = read_error_file(filename) + print(pretty_print_dict(kv_dict, header=header)) \ No newline at end of file diff --git a/appbuilder/tests/run_python_test.sh b/appbuilder/tests/run_python_test.sh index 25e69e576..cd5cc6e26 100644 --- a/appbuilder/tests/run_python_test.sh +++ b/appbuilder/tests/run_python_test.sh @@ -50,6 +50,8 @@ python3 -m pip install diff-cover python3 setup.py bdist_wheel python3 -m pip install --force-reinstall dist/*.whl +python3 -m pip uninstall numpy -y +python3 -m pip install numpy==1.26.4 cd appbuilder/tests/ @@ -95,3 +97,8 @@ if [ $run_result -ne 0 ]; then echo "单测运行失败，请检查错误日志 if [ $cover_result -ne 0 ]; then echo "增量代码的单元测试覆盖率低于90%，请完善单元测试后重试" && exit 1; fi + +echo "--------------------------" +echo "Components组件检查规范性检测结果: " +python3 print_components_error_info.py +echo "--------------------------" \ No newline at end of file diff --git a/appbuilder/tests/test_appbuilder_assistant_trace.py b/appbuilder/tests/test_appbuilder_assistant_trace.py index 9d286f449..775e26261 100644 --- a/appbuilder/tests/test_appbuilder_assistant_trace.py +++ b/appbuilder/tests/test_appbuilder_assistant_trace.py @@ -75,7 +75,7 @@ def tool_calls(self, status_event): ] ) -# @unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_SERIAL", "") +@unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_SERIAL", "") class TestAppBuilderTrace(unittest.TestCase): def setUp(self): """ diff --git a/appbuilder/tests/test_appbuilder_client_app_list.py b/appbuilder/tests/test_appbuilder_client_app_list.py index c844c0a9e..1a8e2d436 100644 --- a/appbuilder/tests/test_appbuilder_client_app_list.py +++ b/appbuilder/tests/test_appbuilder_client_app_list.py @@ -18,7 +18,7 @@ import os import logging -# @unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_SERIAL","") +@unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_SERIAL","") class TestGetAppList(unittest.TestCase): def test_get_app_list_v1(self): app_list = appbuilder.get_app_list() diff --git a/appbuilder/tests/test_appbuilder_client_toolcall_event_handler_error.py b/appbuilder/tests/test_appbuilder_client_toolcall_event_handler_error.py index ec3eae431..afce553be 100644 --- a/appbuilder/tests/test_appbuilder_client_toolcall_event_handler_error.py +++ b/appbuilder/tests/test_appbuilder_client_toolcall_event_handler_error.py @@ -21,7 +21,7 @@ def success(self, run_context, run_response): print("\n\033[1;31m","-> Agent 非流式回答: ", run_response.answer, "\033[0m") -# @unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_SERIAL","") +@unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_SERIAL","") class TestAgentRuntime(unittest.TestCase): def setUp(self): """ diff --git a/appbuilder/tests/test_core_commponents_tool_eval.py b/appbuilder/tests/test_core_commponents_tool_eval.py new file mode 100644 index 000000000..8605ec7e5 --- /dev/null +++ b/appbuilder/tests/test_core_commponents_tool_eval.py @@ -0,0 +1,184 @@ +# Copyright (c) 2024 Baidu, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import unittest +import inspect +import appbuilder +import importlib.util +import numpy as np +import pandas as pd + +from appbuilder.core.component import Component +from appbuilder.core.components.llms.base import CompletionBaseComponent +from appbuilder import AutomaticTestToolEval +from appbuilder.core._exception import AppbuilderBuildexException + +def check_ancestor(cls): + parent_cls = Component + excluded_classes = ('Component', 'MatchingBaseComponent', 'EmbeddingBaseComponent', 'CompletionBaseComponent') + if cls.__name__ in excluded_classes: + return False + if issubclass(cls, CompletionBaseComponent): + return False + if issubclass(cls, parent_cls): + if parent_cls in excluded_classes: + return False + return True + for base in cls.__bases__: + if check_ancestor(base): + return True + return False + + +def find_tool_eval_components(): + current_file_path = os.path.abspath(__file__) + print(current_file_path) + components = [] + added_components = set() + base_path = current_file_path.split('/') + base_path = base_path[:-2]+['core', 'components'] + base_path = '/'.join(base_path) + print(base_path) + + for root, _, files in os.walk(base_path): + for file in files: + if file.endswith(".py"): + module_path = os.path.join(root, file) + module_name = module_path.replace(base_path, '').replace('/', '.').replace('\\', '.').strip('.') + + # 动态加载模块 + spec = importlib.util.spec_from_file_location(module_name, module_path) + if spec is None: + continue + module = importlib.util.module_from_spec(spec) + try: + spec.loader.exec_module(module) + except Exception as e: + continue + + # 查找继承自 Component 的类 + for name, obj in inspect.getmembers(module, inspect.isclass): + has_tool_eval = 'tool_eval' in obj.__dict__ and callable(getattr(obj, 'tool_eval', None)) + if has_tool_eval and obj.__name__ not in added_components and check_ancestor(obj): + added_components.add(obj.__name__) + components.append((name, obj)) + + return components + + +def read_whitelist_components(): + with open('whitelist_components.txt', 'r') as f: + lines = [line.strip() for line in f] + return lines + + +def write_error_data(error_df,error_stats): + txt_file_path = 'components_error_info.txt' + with open(txt_file_path, 'w') as file: + file.write("Component Name\tError Message\n") + for _, row in error_df.iterrows(): + file.write(f"{row['Component Name']}\t{row['Error Message']}\n") + file.write("\n错误统计信息:\n") + for error, count in error_stats.items(): + file.write(f"错误信息: {error}, 出现次数: {count}\n") + print(f"\n错误信息已写入: {txt_file_path}") + +@unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_PARALLEL", "") +class TestComponentManifestsAndToolEval(unittest.TestCase): + def setUp(self) -> None: + self.tool_eval_components = find_tool_eval_components() + self.whitelist_components = read_whitelist_components() + + def test_manifests(self): + """ + 要求必填，格式: list[dict]，dict字段为 + * "name"：str，要求不重复 + * "description"：str，对于组件tool_eval函数功能的描述 + * "parameters"：json_schema，对于tool_eval函数入参的描述，json_schema格式要求见https://json-schema.org/understanding-json-schema + """ + print("完成manifests测试的组件:") + for name, cls in self.tool_eval_components: + init_signature = inspect.signature(cls.__init__) + params = init_signature.parameters + mock_args = {} + for parameter_name, param in params.items(): + # 跳过 'self' 参数 + if parameter_name == 'self': + continue + if parameter_name == 'model' or name == 'model_name': + mock_args[parameter_name] = appbuilder.get_model_list()[0] + app = cls(**mock_args) + manifests = app.manifests + + assert isinstance(manifests, list) + assert len(manifests) > 0 + assert isinstance(manifests[0],dict) + assert isinstance(manifests[0]['name'], str) + assert isinstance(manifests[0]['description'], str) + assert isinstance(manifests[0]['parameters'], dict) + print("组件名称:{}".format(name)) + + def test_tool_eval(self): + """ + 测试tool_eval组件，收集报错信息，生成并存储报错信息表格，并进行统计和可视化。 + """ + print("完成tool_eval测试的组件:") + error_data = [] + + for name, cls in self.tool_eval_components: + init_signature = inspect.signature(cls.__init__) + params = init_signature.parameters + mock_args = {} + for parameter_name, param in params.items(): + # 跳过 'self' 参数 + if parameter_name == 'self': + continue + if parameter_name == 'model' or name == 'model_name': + mock_args[parameter_name] = appbuilder.get_model_list()[0] + app = cls(**mock_args) + try: + AutomaticTestToolEval(app) + print("组件名称:{} 通过测试".format(name)) + except Exception as e: + error_data.append({"Component Name": name, "Error Message": str(e)}) + print("组件名称:{} 错误信息:{}".format(name, e)) + + # 将错误信息表格存储在本地变量中 + error_df = pd.DataFrame(error_data) if error_data else None + + if error_df is not None: + print("\n错误信息表格:") + print(error_df) + # 使用 NumPy 进行统计 + unique_errors, counts = np.unique(error_df["Error Message"], return_counts=True) + error_stats = dict(zip(unique_errors, counts)) + print("\n错误统计信息:") + for error, count in error_stats.items(): + print(f"错误信息: {error}, 出现次数: {count}") + else: + print("\n所有组件测试通过，无错误信息。") + + # 将报错信息写入文件 + write_error_data(error_df, error_stats) + + # 判断报错组件是否位于白名单中 + component_names = error_df["Component Name"].tolist() + for component_name in component_names: + if component_name in self.whitelist_components: + print("{}zu白名单中，暂时忽略报错。".format(component_name)) + else: + raise AppbuilderBuildexException(f"组件 {component_name} 未在白名单中，请检查是否需要添加到白名单。") + +if __name__ == '__main__': + unittest.main() diff --git a/appbuilder/tests/test_core_componentstest.py b/appbuilder/tests/test_core_componentstest.py new file mode 100644 index 000000000..1426d1286 --- /dev/null +++ b/appbuilder/tests/test_core_componentstest.py @@ -0,0 +1,104 @@ +# Copyright (c) 2024 Baidu, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import unittest +import appbuilder +import requests + +from unittest.mock import Mock +from appbuilder.core._exception import AppbuilderBuildexException + + +@unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_PARALLEL", "") +class TestComponents(unittest.TestCase): + def setUp(self): + self.image_understand = appbuilder.ImageUnderstand() + self.table_ocr = appbuilder.TableOCR() + self.img_url = 'img_url' + + + def test_components_raise(self): + tool_eval_input = {} + response = requests.Response() + # 检查tool_eval必须有streaming参数 + with self.assertRaises(AppbuilderBuildexException) as e: + appbuilder.AppbuilderTestToolEval(appbuilder_components=self.image_understand, + tool_eval_input=tool_eval_input, + response=response) + exception = e.exception + self.assertIn('是否定义streaming', str(exception)) + + # 检查组件tool_eval的传入参数是否有traceid + tool_eval_input = { + 'streaming': True, + } + with self.assertRaises(AppbuilderBuildexException) as e: + appbuilder.AppbuilderTestToolEval(appbuilder_components=self.image_understand, + tool_eval_input=tool_eval_input, + response=response) + exception = e.exception + self.assertIn('传入参数是否有traceid', str(exception)) + + # 检查组件tool_eval的传入参数是否正确或manifests的参数定义是否正确(required为anyOf) + tool_eval_input = { + 'streaming': True, + 'traceid': 'traceid', + } + with self.assertRaises(AppbuilderBuildexException) as e: + appbuilder.AppbuilderTestToolEval(appbuilder_components=self.image_understand, + tool_eval_input=tool_eval_input, + response=response) + exception = e.exception + self.assertIn('组件tool_eval的传入参数是否正确或manifests的参数定义是否正确', str(exception)) + + # 检查组件tool_eval的传入参数是否正确或manifests的参数定义是否正确(required不为anyOf) + tool_eval_input = { + 'streaming': True, + 'traceid': 'traceid', + } + with self.assertRaises(AppbuilderBuildexException) as e: + appbuilder.AppbuilderTestToolEval(appbuilder_components=self.table_ocr, + tool_eval_input=tool_eval_input, + response=response) + exception = e.exception + self.assertIn('组件tool_eval的传入参数是否正确或manifests的参数定义是否正确', str(exception)) + + def test_components_tool_eval_image_understand(self): + mock_response_data = { + 'result': {'task_id': '1821485837570181996'}, + 'log_id': 1821485837570181996, + } + mock_response = Mock() + mock_response.status_code = 200 + mock_response.headers = {'Content-Type': 'application/json'} + def mock_json(): + return mock_response_data + mock_response.json = mock_json + + tool_eval_input = { + 'streaming': True, + 'traceid': 'traceid', + 'name':"image_understand", + 'img_url':self.img_url, + 'origin_query':"" + } + + appbuilder.AppbuilderTestToolEval(appbuilder_components=self.image_understand, + tool_eval_input=tool_eval_input, + response=mock_response) + + +if __name__ == "__main__": + unittest.main() + \ No newline at end of file diff --git a/appbuilder/tests/test_qa_aicape_animal_rec.py b/appbuilder/tests/test_qa_aicape_animal_rec.py index 1d17979ca..f3e4b585a 100644 --- a/appbuilder/tests/test_qa_aicape_animal_rec.py +++ b/appbuilder/tests/test_qa_aicape_animal_rec.py @@ -11,11 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import os import unittest import appbuilder import appbuilder +@unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_PARALLEL", "") class TestAnimalRecognition(unittest.TestCase): def test_run(self): diff --git a/appbuilder/tests/test_qa_aicape_doc_crop_enhance.py b/appbuilder/tests/test_qa_aicape_doc_crop_enhance.py index 3570db375..cf5727add 100644 --- a/appbuilder/tests/test_qa_aicape_doc_crop_enhance.py +++ b/appbuilder/tests/test_qa_aicape_doc_crop_enhance.py @@ -11,10 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import os import unittest import appbuilder + +@unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_PARALLEL", "") class TestDocCropEnhance(unittest.TestCase): def test_run(self): diff --git a/appbuilder/tests/test_qa_aicape_handwriting_ocr.py b/appbuilder/tests/test_qa_aicape_handwriting_ocr.py index 87dd3eaaa..f752e9fb8 100644 --- a/appbuilder/tests/test_qa_aicape_handwriting_ocr.py +++ b/appbuilder/tests/test_qa_aicape_handwriting_ocr.py @@ -11,10 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import os import unittest import appbuilder +@unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_PARALLEL", "") class TestHandwritingOcr(unittest.TestCase): def test_run(self): image_url=("https://bj.bcebos.com/v1/appbuilder/test_handwrite_ocr.jpg?" diff --git a/appbuilder/tests/test_qa_aicape_image_understand.py b/appbuilder/tests/test_qa_aicape_image_understand.py index 0c3bf5099..9aee5bc80 100644 --- a/appbuilder/tests/test_qa_aicape_image_understand.py +++ b/appbuilder/tests/test_qa_aicape_image_understand.py @@ -11,11 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import os import unittest import appbuilder import time +@unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_PARALLEL", "") class TestAnimalRecognition(unittest.TestCase): def test_run(self): diff --git a/appbuilder/tests/test_qa_aicape_mixcard_ocr.py b/appbuilder/tests/test_qa_aicape_mixcard_ocr.py index aa96921ec..a6eba362a 100644 --- a/appbuilder/tests/test_qa_aicape_mixcard_ocr.py +++ b/appbuilder/tests/test_qa_aicape_mixcard_ocr.py @@ -11,10 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import os import unittest import appbuilder + +@unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_PARALLEL", "") class TestMixcardOcr(unittest.TestCase): def test_run(self): diff --git a/appbuilder/tests/test_qa_aicape_plant_rec.py b/appbuilder/tests/test_qa_aicape_plant_rec.py index 8cccd5d01..a95f784c4 100644 --- a/appbuilder/tests/test_qa_aicape_plant_rec.py +++ b/appbuilder/tests/test_qa_aicape_plant_rec.py @@ -11,10 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import os import unittest import appbuilder +@unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_PARALLEL", "") class TestPlantRecognition(unittest.TestCase): def test_run(self): diff --git a/appbuilder/tests/test_qa_aicape_qrcode_orc.py b/appbuilder/tests/test_qa_aicape_qrcode_orc.py index 3946331ba..c5810730b 100644 --- a/appbuilder/tests/test_qa_aicape_qrcode_orc.py +++ b/appbuilder/tests/test_qa_aicape_qrcode_orc.py @@ -11,10 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import os import unittest import appbuilder +@unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_PARALLEL", "") class TestQrcodeOcr(unittest.TestCase): def test_run(self): diff --git a/appbuilder/tests/test_qa_aicape_table_ocr.py b/appbuilder/tests/test_qa_aicape_table_ocr.py index a36f32353..d424dce2a 100644 --- a/appbuilder/tests/test_qa_aicape_table_ocr.py +++ b/appbuilder/tests/test_qa_aicape_table_ocr.py @@ -11,10 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import os import unittest import appbuilder +@unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_PARALLEL", "") class TestTableOcr(unittest.TestCase): def test_run(self): diff --git a/appbuilder/tests/test_qa_doc_parser_extract_table_from_doc.py b/appbuilder/tests/test_qa_doc_parser_extract_table_from_doc.py index b9c378feb..a5fcbdf5c 100644 --- a/appbuilder/tests/test_qa_doc_parser_extract_table_from_doc.py +++ b/appbuilder/tests/test_qa_doc_parser_extract_table_from_doc.py @@ -17,6 +17,7 @@ import appbuilder from appbuilder.core.message import Message +@unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_PARALLEL", "") class TestDocParserExtractTableFromDoc(unittest.TestCase): def test_run(self): diff --git a/appbuilder/tests/test_qa_llm_dialog_summary.py b/appbuilder/tests/test_qa_llm_dialog_summary.py index b142b6284..c22a52625 100644 --- a/appbuilder/tests/test_qa_llm_dialog_summary.py +++ b/appbuilder/tests/test_qa_llm_dialog_summary.py @@ -11,11 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import os import unittest import appbuilder from appbuilder import Message +@unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_PARALLEL", "") class TestDialogSummaryComponent(unittest.TestCase): def test_run(self): diff --git a/appbuilder/tests/test_qa_llm_get_qianfan_model_list.py b/appbuilder/tests/test_qa_llm_get_qianfan_model_list.py index 8e4476029..09b697121 100644 --- a/appbuilder/tests/test_qa_llm_get_qianfan_model_list.py +++ b/appbuilder/tests/test_qa_llm_get_qianfan_model_list.py @@ -16,6 +16,7 @@ import unittest import appbuilder +@unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_PARALLEL", "") class TestGetQianfanModelList(unittest.TestCase): def test_run(self): diff --git a/appbuilder/tests/test_qa_llm_is_complex_query.py b/appbuilder/tests/test_qa_llm_is_complex_query.py index ca2de0029..82ef8c7b8 100644 --- a/appbuilder/tests/test_qa_llm_is_complex_query.py +++ b/appbuilder/tests/test_qa_llm_is_complex_query.py @@ -11,10 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import os import unittest import appbuilder +@unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_PARALLEL", "") class TestIsComplexQuery(unittest.TestCase): def test_run(self): diff --git a/appbuilder/tests/test_qa_llm_matching.py b/appbuilder/tests/test_qa_llm_matching.py index 9d31d4fb9..515d145ad 100644 --- a/appbuilder/tests/test_qa_llm_matching.py +++ b/appbuilder/tests/test_qa_llm_matching.py @@ -11,10 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import os import unittest import appbuilder +@unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_PARALLEL", "") class TestSemanticRankComponent(unittest.TestCase): def test_normal_case(self): query = "合同中展期利率与罚息利率分别是什么？" diff --git a/appbuilder/tests/test_qa_llm_oral_query_generation.py b/appbuilder/tests/test_qa_llm_oral_query_generation.py index d983bb0eb..e45cf04af 100644 --- a/appbuilder/tests/test_qa_llm_oral_query_generation.py +++ b/appbuilder/tests/test_qa_llm_oral_query_generation.py @@ -11,10 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import os import unittest import appbuilder +@unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_PARALLEL", "") class TestOralQueryGeneration(unittest.TestCase): def test_normal_case(self): model_name = "ERNIE-3.5-8K" diff --git a/appbuilder/tests/test_qa_llm_paddle_speech_tts.py b/appbuilder/tests/test_qa_llm_paddle_speech_tts.py index 016e5fb7a..096db358e 100644 --- a/appbuilder/tests/test_qa_llm_paddle_speech_tts.py +++ b/appbuilder/tests/test_qa_llm_paddle_speech_tts.py @@ -11,11 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import os import unittest import appbuilder - +@unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_PARALLEL", "") class TestPaddleSpeechTTS(unittest.TestCase): def test_normal_case(self): text = "吸塑包装盒在工业化生产和物流运输中分别有什么重要性" diff --git a/appbuilder/tests/test_qa_llm_pandas.py b/appbuilder/tests/test_qa_llm_pandas.py index 7dc7abce2..58356efeb 100644 --- a/appbuilder/tests/test_qa_llm_pandas.py +++ b/appbuilder/tests/test_qa_llm_pandas.py @@ -11,11 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import os import unittest import appbuilder from appbuilder import Message +@unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_PARALLEL", "") class TestNl2pandasComponent(unittest.TestCase): def test_normal_case(self): model_name = "ERNIE-3.5-8K" diff --git a/appbuilder/tests/test_qa_llm_query_decomposition.py b/appbuilder/tests/test_qa_llm_query_decomposition.py index 51eae75e4..f7f95e0ad 100644 --- a/appbuilder/tests/test_qa_llm_query_decomposition.py +++ b/appbuilder/tests/test_qa_llm_query_decomposition.py @@ -11,11 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import os import unittest import appbuilder - +@unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_PARALLEL", "") class TestQueryDecomposition(unittest.TestCase): def test_normal_case(self): model_name = "ERNIE-3.5-8K" diff --git a/appbuilder/tests/test_qa_llm_style_rewrite.py b/appbuilder/tests/test_qa_llm_style_rewrite.py index e2168d653..e77d19318 100644 --- a/appbuilder/tests/test_qa_llm_style_rewrite.py +++ b/appbuilder/tests/test_qa_llm_style_rewrite.py @@ -11,10 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import os import unittest import appbuilder +@unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_PARALLEL", "") class TestStyleRewrite(unittest.TestCase): def test_normal_case(self): text = "文心大模型发布新版" diff --git a/appbuilder/tests/test_qa_pair_mining.py b/appbuilder/tests/test_qa_pair_mining.py index 53251dfdd..43994967d 100644 --- a/appbuilder/tests/test_qa_pair_mining.py +++ b/appbuilder/tests/test_qa_pair_mining.py @@ -18,6 +18,7 @@ import appbuilder + @unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_SERIAL", "") class TestQAPairMiningComponent(unittest.TestCase): def setUp(self): diff --git a/appbuilder/tests/whitelist_components.txt b/appbuilder/tests/whitelist_components.txt new file mode 100644 index 000000000..cfabd8879 --- /dev/null +++ b/appbuilder/tests/whitelist_components.txt @@ -0,0 +1,13 @@ +TableOCR +Excel2Figure +HandwriteOCR +GeneralOCR +Translation +ObjectRecognition +DocFormatConverter +MixCardOCR +ASR +PlantRecognition +AnimalRecognition +QRcodeOCR +ImageUnderstand \ No newline at end of file