fix(api/celery): change CELERY_RESULT_BACKEND to django-db, instead of MQ or Redis

.github/workflows/unittest.yml

@@ -15,7 +15,7 @@ jobs:
         poetry-version: [1.1.7]
         os: [ubuntu-18.04]
     runs-on: ${{ matrix.os }}
-    
+
     env:
       DB_DATABASE: bk_user_api_test
       DB_USER: root

deploy/helm/bk-user/charts/api/templates/external-storage-configmap.yaml

@@ -6,7 +6,7 @@ metadata:
 data:
   # ---------------
   # 数据库
-  # --------------- 
+  # ---------------
   DB_NAME: "{{ .Values.externalDatabase.default.name | default .Values.preferDBName }}"
   DB_USER: "{{ .Values.externalDatabase.default.user }}"
   DB_PASSWORD: "{{ .Values.externalDatabase.default.password }}"
@@ -15,5 +15,4 @@ data:
   # ---------------
   # Redis Related
   # ---------------
-  CELERY_BROKER_URL: {{ .Values.celeryBrokerUrl | default (include "bk-user.externalRedisBrokerUrl" .) }}
-  CELERY_RESULT_BACKEND: {{ .Values.celeryResultBackend | default (include "bk-user.externalRedisBrokerUrl" .) }}
+  CELERY_BROKER_URL: {{ .Values.celeryBrokerUrl | default (include "bk-user.externalRedisBrokerUrl" .) }}

deploy/helm/bk-user/templates/redis-env-configmap.yaml

@@ -7,8 +7,6 @@ metadata:
 data:
   {{- if .Values.redis.enabled }}
   CELERY_BROKER_URL: {{ include "bk-user.builtinRedisBrokerUrl" . }}
-  CELERY_RESULT_BACKEND: {{ include "bk-user.builtinRedisBrokerUrl" . }}
   {{- else }}
   CELERY_BROKER_URL: {{ .Values.api.celeryBrokerUrl | default (include "bk-user.apiExternalRedisBrokerUrl" .) }}
-  CELERY_RESULT_BACKEND: {{ .Values.api.celeryResultBackend | default (include "bk-user.apiExternalRedisBrokerUrl" .) }}
   {{- end -}}

src/api/bkuser_core/categories/plugins/custom/helpers.py

@@ -99,9 +99,7 @@ def _handle_department(self, dept_code) -> Optional[Department]:
                     error=_("父组织【{department}】不存在").format(department=dept_info.parent),
                 )
                 raise ValueError(
-                    "the given parent<%s> of department<%s> not found",
-                    dept_info.parent,
-                    dept_code,
+                    f"the given parent<{dept_info.parent}> of department<{dept_code}:{dept_info.name}> not found"
                 )
             parent_id = parent_dept.pk
 
@@ -158,7 +156,7 @@ def _load_base_info(self):
                 validate_username(value=info.username)
             except ValidationError as e:
                 self.context.add_record(step=SyncStep.USERS, success=False, username=info.username, error=str(e))
-                logger.warning("username<%s> does not meet format", info.username)
+                logger.warning("username<%s:%s> does not meet format", info.code, info.username)
                 continue
 
             # 1. 先更新 profile 本身
@@ -207,7 +205,7 @@ def _load_base_info(self):
                         department=dep_id,
                         error=_("部门不存在"),
                     )
-                    logger.warning("the department<%s> of profile<%s> is missing", dep_id, info.code)
+                    logger.warning("the department<%s> of profile<%s:%s> is missing", dep_id, info.code, info.username)
                     continue
 
                 self.try_add_relation(
@@ -229,15 +227,15 @@ def _load_leader_info(self):
                 self.context.add_record(
                     step=SyncStep.USERS_RELATIONSHIP, success=False, username=info.username, error=_("用户信息不存在")
                 )
-                logger.warning("profile<%s> will not be synced, skip", info.code)
+                logger.warning("profile<%s:%s> not exists, will not be synced, skip", info.code, info.username)
                 continue
 
             for leader_id in info.leaders:
                 if leader_id == profile.code:
                     self.context.add_record(
                         step=SyncStep.USERS_RELATIONSHIP, success=False, username=info.username, error=_("无法设置自己为上级")
                     )
-                    logger.warning("profile can not regard self as leader, skip")
+                    logger.warning("profile<%s:%s> can not regard self as leader, skip", info.code, info.username)
                     continue
 
                 leader = self.db_sync_manager.magic_get(self._get_code(leader_id), CustomProfileMeta)
@@ -248,7 +246,7 @@ def _load_leader_info(self):
                         username=info.username,
                         error=_("上级【{username}】不存在").format(username=leader_id),
                     )
-                    logger.warning("the leader<%s> of profile<%s> is missing", leader_id, info.code)
+                    logger.warning("the leader<%s> of profile<%s:%s> is missing", leader_id, info.code, info.username)
                     continue
 
                 self.try_add_relation(

src/api/bkuser_core/categories/plugins/ldap/client.py

@@ -71,8 +71,10 @@ def initialize(
             return Connection(**connection_params)
 
         except KeyError:
+            logger.exception("failed to initialize ldap server. KeyError. [url=%s]", connection_url)
             raise local_exceptions.LDAPSettingNotReady
         except ldap3.core.exceptions.LDAPSocketReceiveError:
+            logger.exception("failed to initialize ldap server. LDAPSocketReceiveError. [url=%s]", connection_url)
             raise local_exceptions.LdapCannotBeInitialized
         except Exception:
             logger.exception("failed to initialize ldap server. [url=%s]", connection_url)

src/api/bkuser_core/categories/plugins/ldap/handlers.py

@@ -73,7 +73,7 @@ def delete_sync_tasks(sender, instance: "ProfileCategory", **kwargs):
     if instance.type not in [CategoryType.LDAP.value, CategoryType.MAD.value]:
         return
 
-    logger.info("going to delete periodic task for Category<%s>", instance.id)
+    logger.info("going to delete periodic task for Category<%s>, the category type is %s", instance.id, instance.type)
     delete_periodic_sync_task(instance.id)
 
 

src/api/bkuser_core/config/common/.env-tmpl

@@ -9,10 +9,10 @@ BK_IAM_SYSTEM_ID="bk-xxxx"
 
 # 如果需要使用到本地 broker
 CELERY_BROKER_URL="redis://:password@localhost:32768/11"
-CELERY_RESULT_BACKEND="redis://:password@localhost:32768/11"
+CELERY_RESULT_BACKEND="django-db"
 
 DB_NAME="bk-user-api"
 DB_USER="root"
 DB_PASSWORD="123456"
 DB_HOST="127.0.0.1"
-DB_PORT="3306"
+DB_PORT="3306"

src/api/bkuser_core/config/common/django_basic.py

@@ -65,6 +65,7 @@
     "rest_framework",
     "mptt",
     "django_celery_beat",
+    "django_celery_results",
     "django_prometheus",
     # core API
     "bkuser_core.apis",

src/api/bkuser_core/config/common/storage.py

@@ -34,7 +34,7 @@
 # Celery
 # ==============================================================================
 CELERY_BROKER_URL = env("CELERY_BROKER_URL")
-CELERY_RESULT_BACKEND = env("CELERY_RESULT_BACKEND")
+CELERY_RESULT_BACKEND = "django-db"
 CELERY_TASK_DEFAULT_QUEUE = env("CELERY_TASK_DEFAULT_QUEUE", default="bk_user")
 
 

src/api/bkuser_core/config/overlays/unittest.py

@@ -1,4 +1,6 @@
 # -*- coding: utf-8 -*-
+import os
+
 import ldap3
 
 from bkuser_core.config.common.django_basic import *  # noqa
@@ -85,6 +87,9 @@ def get_loggers(package_name: str, log_level: str) -> dict:
 # Ldap connection mock
 LDAP_CONNECTION_EXTRAS_PARAMS = {"client_strategy": ldap3.MOCK_SYNC}
 
+# celery results backend use from env(in unittest)
+CELERY_RESULT_BACKEND = os.getenv("CELERY_RESULT_BACKEND")
+
 
 # ==============================================================================
 # Email

src/api/pyproject.toml

@@ -41,6 +41,7 @@ pyyaml = "^6.0"
 sentry-sdk = "1.5.6"
 pyjwt = "^2.3.0"
 apigw-manager = "^1.0.3"
+django-celery-results = "2.0.1"
 
 [tool.poetry.dev-dependencies]
 ipython = "^7.15.0"