issue-1032/oracle查看表结构增加注释列

sql/engines/goinception.py

@@ -3,6 +3,7 @@
 import re
 import traceback
 import MySQLdb
+import simplejson as json
 
 from common.config import SysConfig
 from sql.models import AliyunRdsConfig
@@ -149,6 +150,27 @@ def query_print(self, instance, db_name=None, sql=''):
             raise RuntimeError(print_info.get('errmsg'))
         return print_info
 
+    def query_datamasking(self, instance, db_name=None, sql=''):
+        """
+        将sql交给goInception打印语法树。
+        使用 masking 参数，可参考 https://github.com/hanchuanchuan/goInception/pull/355
+        """
+        # 判断如果配置了隧道则连接隧道
+        host, port, user, password = self.remote_instance_conn(instance)
+        sql = f"""/*--user={user};--password={password};--host={host};--port={port};--masking=1;*/
+                          inception_magic_start;
+                          use `{db_name}`;
+                          {sql}
+                          inception_magic_commit;"""
+        print_info = self.query(db_name=db_name, sql=sql).to_dict()[0]
+        # 兼容语法错误时errlevel=0的场景
+        if print_info['errlevel'] == 0 and print_info['errmsg'] is None :
+            return json.loads(_repair_json_str(print_info['query_tree']))
+        elif print_info['errlevel'] == 0 and print_info['errmsg'] == 'Global environment':
+            raise SyntaxError(f"Inception Error: {print_info['query_tree']}")
+        else:
+            raise RuntimeError(f"Inception Error: {print_info['errmsg']}")
+
     def get_variables(self, variables=None):
         """获取实例参数"""
         if variables:
@@ -247,3 +269,15 @@ def get_session_variables(instance):
     for k, v in variables.items():
         set_session_sql += f"inception set session {k} = '{v}';\n"
     return variables, set_session_sql
+
+def _repair_json_str(json_str):
+    """
+    处理JSONDecodeError: Expecting property name enclosed in double quotes
+    inception语法树出现{"a":1,}、["a":1,]、{'a':1}、[, { }]
+    """
+    json_str = re.sub(r"{\s*'(.+)':", r'{"\1":', json_str)
+    json_str = re.sub(r",\s*?]", "]", json_str)
+    json_str = re.sub(r",\s*?}", "}", json_str)
+    json_str = re.sub(r"\[,\s*?{", "[{", json_str)
+    json_str = json_str.replace("'", "\"")
+    return json_str

sql/engines/inception.py

@@ -206,17 +206,17 @@ def get_rollback(self, workflow):
                     sql = row.get('sql')
                 # 获取备份表名
                 opid_time = sequence.replace("'", "")
-                sql_table = f"""select tablename 
-                                from {backup_db_name}.$_$Inception_backup_information$_$ 
+                sql_table = f"""select tablename
+                                from {backup_db_name}.$_$Inception_backup_information$_$
                                 where opid_time='{opid_time}';"""
 
                 cur.execute(sql_table)
                 list_tables = cur.fetchall()
                 if list_tables:
                     # 获取备份语句
                     table_name = list_tables[0][0]
-                    sql_back = f"""select rollback_statement 
-                                   from {backup_db_name}.{table_name} 
+                    sql_back = f"""select rollback_statement
+                                   from {backup_db_name}.{table_name}
                                    where opid_time='{opid_time}'"""
                     cur.execute(sql_back)
                     list_backup = cur.fetchall()
@@ -271,4 +271,4 @@ def _repair_json_str(json_str):
     json_str = re.sub(r",\s*?}", "}", json_str)
     json_str = re.sub(r"\[,\s*?{", "[{", json_str)
     json_str = json_str.replace("'", "\"")
-    return json_str
+    return json_str

sql/engines/mysql.py

@@ -15,6 +15,7 @@
 from .models import ResultSet, ReviewResult, ReviewSet
 from .inception import InceptionEngine
 from sql.utils.data_masking import data_masking
+from sql.utils.go_data_masking import go_data_masking
 from common.config import SysConfig
 
 logger = logging.getLogger('default')
@@ -112,7 +113,7 @@ def get_all_tables(self, db_name, **kwargs):
 
     def get_all_columns_by_tb(self, db_name, tb_name, **kwargs):
         """获取所有字段, 返回一个ResultSet"""
-        sql = f"""SELECT 
+        sql = f"""SELECT
             COLUMN_NAME,
             COLUMN_TYPE,
             CHARACTER_SET_NAME,
@@ -234,7 +235,13 @@ def query_masking(self, db_name=None, sql='', resultset=None):
         返回一个脱敏后的结果集"""
         # 仅对select语句脱敏
         if re.match(r"^select", sql, re.I):
-            mask_result = data_masking(self.instance, db_name, sql, resultset)
+            ##判断是否设置了inception脱敏，如果未配置inception地址，则使用goinception脱敏
+            if (self.config.get('inception_host') is None):
+                mask_result = go_data_masking(self.instance, db_name, sql, resultset)
+                #print("use goinception")
+            else:
+                mask_result = data_masking(self.instance, db_name, sql, resultset)
+                #print("use inception")
         else:
             mask_result = resultset
         return mask_result

sql/engines/oracle.py

@@ -14,6 +14,7 @@
 from . import EngineBase
 import cx_Oracle
 from .models import ResultSet, ReviewSet, ReviewResult
+#from sql.utils.data_masking import brute_mask
 from sql.utils.data_masking import simple_column_mask
 
 logger = logging.getLogger('default')
@@ -138,13 +139,17 @@ def describe_table(self, db_name, tb_name, **kwargs):
         """return ResultSet"""
         # https://www.thepolyglotdeveloper.com/2015/01/find-tables-oracle-database-column-name/
         sql = f"""SELECT
-        column_name,
+        a.column_name,
         data_type,
         data_length,
         nullable,
-        data_default
-        FROM all_tab_cols
-        WHERE table_name = '{tb_name}' and owner = '{db_name}' order by column_id
+        data_default,
+        b.comments
+        FROM all_tab_cols a, all_col_comments b
+        WHERE a.table_name = b.table_name
+        and a.owner = b.OWNER
+        and a.COLUMN_NAME = b.COLUMN_NAME
+        and a.table_name = '{tb_name}' and a.owner = '{db_name}' order by column_id
         """
         result = self.query(db_name=db_name, sql=sql)
         return result
@@ -365,6 +370,7 @@ def query(self, db_name=None, sql='', limit_num=0, close_conn=True, **kwargs):
     def query_masking(self, db_name=None, sql='', resultset=None):
         """简单字段脱敏规则, 仅对select有效"""
         if re.match(r"^select", sql, re.I):
+            #filtered_result = brute_mask(self.instance, resultset)
             filtered_result = simple_column_mask(self.instance, resultset)
             filtered_result.is_masked = True
         else:

sql/utils/data_masking.py

@@ -257,7 +257,7 @@ def regex(masking_rules, rule_type, value):
 
 
 def brute_mask(instance, sql_result):
-    """输入的是一个resultset 
+    """输入的是一个resultset
     sql_result.full_sql
     sql_result.rows 查询结果列表 List , list内的item为tuple
 
@@ -348,4 +348,3 @@ def simple_column_mask(instance, sql_result):
             sql_result.error = str(e)
 
     return sql_result
-