fix(api/category): fix http api datasource syncer

src/api/bkuser_core/categories/management/commands/make_crontab_sync_task_for_category.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available.
+Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+import logging
+
+from bkuser_core.categories.models import ProfileCategory
+from bkuser_core.categories.plugins.utils import make_periodic_sync_task
+from django.core.management.base import BaseCommand
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = "make a sync task for a given category"
+
+    def add_arguments(self, parser):
+        parser.add_argument("--category_id", type=str, help="目录ID")
+        parser.add_argument("--interval", type=int, help="同步周期(单位秒)")
+        parser.add_argument("--operator", type=str, default="admin", help="操作者")
+
+    def handle(self, *args, **options):
+        category_id = options.get("category_id")
+        interval = options.get("interval")
+        operator = options.get("operator")
+
+        self.stdout.write(f"Will add a sync task for category_id={category_id}, run every {interval} seconds")
+
+        try:
+            category = ProfileCategory.objects.get(id=category_id)
+        except ProfileCategory.DoesNotExist:
+            self.stdout.write(f"Category {category_id} does not exist, exit")
+            return
+
+        self.stdout.write(f"Category {category_id} exists, name={category.display_name} and type={category.type}")
+
+        try:
+            make_periodic_sync_task(int(category_id), operator, interval)
+        except Exception:  # pylint: disable=broad-except
+            self.stdout.write(f"Failed to add sync task for category {category_id}")

src/api/bkuser_core/categories/plugins/custom/models.py

@@ -8,8 +8,8 @@
 an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 specific language governing permissions and limitations under the License.
 """
-from dataclasses import dataclass
-from typing import Any, Dict, List, Type
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Type
 
 from bkuser_core.departments.models import Department
 from bkuser_core.profiles.models import Profile
@@ -48,6 +48,11 @@ def key(self) -> str:
     def display_str(self) -> str:
         return getattr(self, self.__display_field)
 
+    def __post_init__(self):
+        for k in self.__require_fields:
+            if not getattr(self, k):
+                raise ValueError(f"{k} field is required.")
+
 
 @dataclass
 class CustomTypeList:
@@ -81,32 +86,38 @@ class CustomProfile(CustomType):
     """自定义的 Profile 对象"""
 
     __db_class = Profile
-    __require_fields = ("username", "email", "telephone")
+    # NOTE: 多渠道来源登录, email和telephone数据可能没有
+    __require_fields = ("code", "username")
 
-    username: str
-    email: str
-    telephone: str
-    display_name: str
     code: str
+    username: str
 
-    leaders: List[str]
-    departments: List[str]
+    email: Optional[str] = ""
+    telephone: Optional[str] = ""
+    display_name: Optional[str] = ""
 
-    extras: Dict
-    position: str
+    leaders: List[str] = field(default_factory=list)
+    departments: List[str] = field(default_factory=list)
+
+    extras: Optional[Dict] = field(default_factory=dict)
+    position: Optional[str] = "0"
 
 
-@dataclass
 class CustomDepartment(CustomType):
     """自定义的 Department 对象"""
 
     __db_class = Department
     __display_field = "name"
-    __require_fields = ("name", "parent")
+    __require_fields = ("code", "name")
 
-    name: str
-    parent: str
     code: str
+    name: str
+    parent: Optional[str] = ""
+
+    def __init__(self, code, name, parent=None):
+        self.code = code
+        self.name = name
+        self.parent = parent
 
     @property
     def display_str(self) -> str:

src/api/bkuser_core/categories/plugins/plugin.py

@@ -67,9 +67,7 @@ def register(self):
         """注册插件"""
         register_plugin(self)
         if self.settings_path is not None:
-            pass
-            # TODO: 最好只进行一次配置加载，而不是每个进程都执行
-            # self.load_settings_from_yaml()
+            self.load_settings_from_yaml()
 
     def init_settings(self, setting_meta_key: str, meta_info: dict):
         namespace = meta_info.pop("namespace", "general")

src/api/bkuser_core/categories/tasks.py

@@ -83,6 +83,7 @@ def adapter_sync(instance_id: int, operator: str, task_id: Optional[uuid.UUID] =
         try:
             task_id = SyncTask.objects.register_task(category=category, operator=operator, type_=SyncTaskType.AUTO).id
         except ExistsSyncingTaskError as e:
+            logger.exception("register task fail, the task is already exists")
             raise error_codes.LOAD_DATA_FAILED.f(str(e))
 
     try:

src/api/bkuser_core/config/common/system.py

@@ -45,6 +45,8 @@
 # ==============================================================================
 
 # 强制数据返回格式为原生格式 请求头
+#### 1. SaaS调用api
+#### 2. for_sync场景从一个用户管理同步数据到另一个用户管理中使用
 FORCE_RAW_RESPONSE_HEADER = "HTTP_FORCE_RAW_RESPONSE"
 # 强制返回原始用户名（不带登陆域）请求头
 FORCE_RAW_USERNAME_HEADER = "HTTP_RAW_USERNAME"
@@ -75,6 +77,12 @@
 LOGIN_RECORD_COUNT_SECONDS = env.int("LOGIN_RECORD_COUNT_SECONDS", default=60 * 60 * 24 * 30)
 
 DRF_CROWN_DEFAULT_CONFIG = {"remain_request": True}
+
+# sync, 用户管理本身做业务 HTTP API 数据源, 可以被另一个用户管理同步过去
+# 复用 API, 接口参数中存在 SYNC_API_PARAM 时, 以sync的接口协议返回
+SYNC_API_PARAM = "for_sync"
+
+
 # ==============================================================================
 # 开发调试
 # ==============================================================================

src/api/bkuser_core/departments/v2/serializers.py

@@ -43,6 +43,20 @@ class Meta:
         fields = ("id", "name", "order", "full_name")
 
 
+class ForSyncDepartmentSerializer(CustomFieldsModelSerializer):
+    """this serializer is for sync data from one bk-user to another
+    the api protocol:
+    https://github.com/TencentBlueKing/bk-user/blob/development/src/api/bkuser_core/categories/plugins/custom/README.md
+    """
+
+    class Meta:
+        model = Department
+        fields = ("code",)
+
+    def to_representation(self, instance):
+        return instance.code
+
+
 class DepartmentSerializer(CustomFieldsModelSerializer):
     name = serializers.CharField(required=True)
     has_children = serializers.SerializerMethodField()

src/api/bkuser_core/departments/v2/views.py

@@ -14,6 +14,7 @@
 from bkuser_core.apis.v2.viewset import (
     AdvancedBatchOperateViewSet,
     AdvancedListAPIView,
+    AdvancedListSerializer,
     AdvancedModelViewSet,
     AdvancedSearchFilter,
 )
@@ -86,6 +87,15 @@ def get_serializer_context(self):
         origin.update({"extra_defaults": DynamicFieldInfo.objects.get_extras_default_values()})
         return origin
 
+    @method_decorator(cache_page(settings.GLOBAL_CACHES_TIMEOUT))
+    @swagger_auto_schema(query_serializer=AdvancedListSerializer())
+    def list(self, request, *args, **kwargs):
+        # NOTE: 用于两套用户管理之间的数据同步
+        if settings.SYNC_API_PARAM in request.query_params:
+            request.META[settings.FORCE_RAW_RESPONSE_HEADER] = "true"
+
+        return super().list(request, *args, **kwargs)
+
     @method_decorator(cache_page(settings.GLOBAL_CACHES_TIMEOUT))
     @swagger_auto_schema(
         query_serializer=local_serializers.DepartmentRetrieveSerializer(),

src/api/bkuser_core/profiles/v2/serializers.py

@@ -11,10 +11,10 @@
 from typing import Union
 
 from bkuser_core.apis.v2.serializers import AdvancedRetrieveSerialzier, CustomFieldsMixin, CustomFieldsModelSerializer
-from bkuser_core.departments.v2.serializers import SimpleDepartmentSerializer
+from bkuser_core.departments.v2.serializers import ForSyncDepartmentSerializer, SimpleDepartmentSerializer
 from bkuser_core.profiles.constants import TIME_ZONE_CHOICES, LanguageEnum, RoleCodeEnum
 from bkuser_core.profiles.models import DynamicFieldInfo, Profile
-from bkuser_core.profiles.utils import force_use_raw_username, get_username
+from bkuser_core.profiles.utils import force_use_raw_username, get_username, parse_username_domain
 from bkuser_core.profiles.validators import validate_domain, validate_username
 from django.utils.translation import ugettext_lazy as _
 from rest_framework import serializers
@@ -126,6 +126,24 @@ def get_extras(self, obj: "Profile") -> dict:
         return get_extras(obj.extras, self.context.get("extra_defaults", {}).copy())
 
 
+class ForSyncRapidProfileSerializer(RapidProfileSerializer):
+    """this serializer is for sync data from one bk-user to another
+    the api protocol:
+    https://github.com/TencentBlueKing/bk-user/blob/development/src/api/bkuser_core/categories/plugins/custom/README.md
+    """
+
+    code = serializers.CharField(required=True)
+    departments = ForSyncDepartmentSerializer(many=True, required=False)
+
+    username = serializers.SerializerMethodField()
+
+    def get_username(self, obj):
+        """change username with domain to raw username
+        for sync data between bk-user instances
+        """
+        return parse_username_domain(obj.username)[0]
+
+
 class ProfileDepartmentSerializer(AdvancedRetrieveSerialzier):
     with_family = serializers.BooleanField(default=False, help_text="是否返回所有祖先（兼容）")
     with_ancestors = serializers.BooleanField(default=False, help_text="是否返回所有祖先")

src/api/bkuser_core/profiles/v2/views.py

@@ -111,8 +111,7 @@ def get_serializer_class(self):
         """Serializer 路由"""
         if self.action in ("create", "update", "partial_update"):
             return local_serializers.CreateProfileSerializer
-        elif self.action in ("list",):
-            return local_serializers.RapidProfileSerializer
+        # NOTE: "list" 比较特殊, 放到self.list方法中处理
         else:
             return self.serializer_class
 
@@ -171,11 +170,18 @@ def list(self, request, *args, **kwargs):
         _query_slz.is_valid(True)
         query_data = _query_slz.validated_data
 
+        serializer_class = local_serializers.RapidProfileSerializer
+
+        # NOTE: 用于两套用户管理之间的数据同步
+        if settings.SYNC_API_PARAM in request.query_params:
+            serializer_class = local_serializers.ForSyncRapidProfileSerializer
+            request.META[settings.FORCE_RAW_RESPONSE_HEADER] = "true"
+
         fields = query_data.get("fields", [])
         if fields:
             self._check_fields(fields)
         else:
-            fields = self.get_serializer().fields
+            fields = serializer_class().fields
 
         self._ensure_enabled_field(request, fields=fields)
 
@@ -201,7 +207,7 @@ def list(self, request, *args, **kwargs):
         page = self.paginate_queryset(queryset)
         # page may be empty list
         if page is not None:
-            serializer = self.get_serializer(page, fields=fields, many=True)
+            serializer = serializer_class(page, fields=fields, many=True)
             return self.get_paginated_response(serializer.data)
 
         fields = [x for x in fields if x in self._get_model_field_names()]

src/api/bkuser_core/tests/categories/plugins/custom/test_client.py

@@ -91,9 +91,9 @@ def test_fetch_profiles(self, client, fake_profiles, expected):
             (
                 [
                     {
+                        # "code": "code-1",
                         "username": "fake-user",
                         "email": "[email protected]",
-                        "code": "code-1",
                         "extras": {"aaa": "xxxx", "bbb": "qqqq", "uniquetest": "vvvv"},
                         "position": 0,
                     }

src/saas/bkuser_shell/config_center/views.py

@@ -219,7 +219,7 @@ def update(self, request, category_id, namespace_name, validated_data):
                 setting_id = setting_instances[setting_info["key"]].id
             except KeyError:
                 logger.exception(
-                    "找不到 Setting<%s>. [category_id=%s, namesapce_name=%s]",
+                    "找不到 Setting<%s>. [category_id=%s, namespace_name=%s]",
                     setting_info["key"],
                     category_id,
                     namespace_name,