chore: sync upstream master

bklog/Dockerfile

@@ -0,0 +1,23 @@
+FROM python:3.6.12 as builder
+
+RUN apt-get update && apt-get install -y curl vim
+RUN curl -sL https://deb.nodesource.com/setup_12.x | bash - && apt-get install -y nodejs
+
+COPY . /app/code
+
+WORKDIR /app
+
+RUN python -m venv venv \
+&& venv/bin/pip install --upgrade --no-cache-dir pip==20.2.3 setuptools==57.5.0 \
+&& venv/bin/pip install --no-cache-dir -r code/requirements.txt -i https://mirrors.tencent.com/pypi/simple/ --extra-index-url https://mirrors.tencent.com/repository/pypi/tencent_pypi/simple 
+
+RUN venv/bin/pip uninstall -y pycrypto && venv/bin/pip uninstall -y pycryptodome && venv/bin/pip install pycryptodome
+
+WORKDIR /app/code
+
+# set python env
+ENV VIRTUAL_ENV=/app/venv
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+CMD python manage.py migrate
+

bklog/apps/log_search/exceptions.py

@@ -488,6 +488,11 @@ class MultiFieldsErrorException(BaseSearchException):
     MESSAGE = _("跨集群获取字段结果处理异常")
 
 
+class LogSearchException(BaseSearchException):
+    ERROR_CODE = "447"
+    MESSAGE = _("日志检索异常, 原因: {e}")
+
+
 # =================================================
 # 导出
 # =================================================

bklog/apps/log_search/handlers/search/search_handlers_esquery.py

@@ -85,6 +85,7 @@
     BaseSearchSortListException,
     IntegerErrorException,
     IntegerMaxErrorException,
+    LogSearchException,
     MultiSearchErrorException,
     SearchExceedMaxSizeException,
     SearchIndexNoTimeFieldException,
@@ -750,9 +751,14 @@ def _multi_search(self, once_size: int):
         if not storage_cluster_record_objs:
             try:
                 data = search_func(params)
+                # 把shards中的failures信息解析后raise异常出来
+                if data.get("_shards", {}).get("failed"):
+                    errors = data["_shards"]["failures"][0]["reason"]["reason"]
+                    raise LogSearchException(errors)
+
                 return data
-            except ApiResultError as e:
-                raise ApiResultError(_("搜索出错，请检查查询语句是否正确") + f" => {e}", code=e.code, errors=e.errors)
+            except Exception as e:
+                raise LogSearchException(LogSearchException.MESSAGE.format(e=e))
 
         storage_cluster_ids = {self.storage_cluster_id}
 

bklog/apps/log_trace/serializers.py

@@ -126,6 +126,9 @@ def validate(self, attrs):
     fields = serializers.ListField(child=DateHistogramFieldSerializer(), required=False, default=[])
     interval = serializers.CharField(required=False, default="auto", max_length=16)
 
+    # 自定义索引列表 Eg. -> "2_bklog.0001,2_bklog.0002"
+    custom_indices = serializers.CharField(required=False, allow_null=True, allow_blank=True, default="")
+
 
 class UnionSearchDateHistogramSerializer(DateHistogramSerializer):
     index_set_ids = serializers.ListField(

bklog/support-files/apigw/resources.yaml

@@ -714,28 +714,6 @@ paths:
           appVerifiedRequired: true
           resourcePermissionRequired: true
         descriptionEn:
-  /search_index_set/{index_set_id}/tailf/:
-    post:
-      operationId: search_tailf
-      description: 日志平台-检索-实时日志
-      tags:
-      - 查询
-      x-bk-apigateway-resource:
-        isPublic: true
-        allowApplyPermission: true
-        matchSubpath: false
-        backend:
-          name: default
-          method: post
-          path: /api/v1/search/index_set/{index_set_id}/tailf/
-          matchSubpath: false
-          timeout: 0
-        pluginConfigs: []
-        authConfig:
-          userVerifiedRequired: false
-          appVerifiedRequired: true
-          resourcePermissionRequired: true
-        descriptionEn:
   /databus_collectors/:
     post:
       operationId: create_databus_collectors

bklog/web/src/common/util.js

@@ -889,6 +889,16 @@ export const parseTableRowData = (
           break;
         }
 
+        // 这里用于处理nested field
+        if (Array.isArray(data)) {
+          data = data
+            .map(item =>
+              parseTableRowData(item, keyArr.slice(index).join('.'), fieldType, isFormatDate, emptyCharacter),
+            )
+            .filter(item => item !== emptyCharacter);
+          break;
+        }
+
         if (data[item]) {
           data = data[item];
         } else {
@@ -914,7 +924,7 @@ export const parseTableRowData = (
   }
 
   if (Array.isArray(data)) {
-    return data.toString();
+    return data.toString() || emptyCharacter;
   }
 
   if (typeof data === 'object' && data !== null) {
@@ -1220,3 +1230,29 @@ export const getHaveValueIndexItem = indexList => {
     indexList.find(item => !item.tags.map(item => item.tag_id).includes(4))?.index_set_id || indexList[0].index_set_id
   );
 };
+
+export const isNestedField = (fieldKeys, obj) => {
+  if (!obj) {
+    return false;
+  }
+
+  if (fieldKeys.length > 1) {
+    if (obj[fieldKeys[0]] !== undefined && obj[fieldKeys[0]] !== null) {
+      if (typeof obj[fieldKeys[0]] === 'object') {
+        if (Array.isArray(obj[fieldKeys[0]])) {
+          return true;
+        }
+
+        return isNestedField(fieldKeys.slice(1), obj[fieldKeys[0]]);
+      }
+
+      return false;
+    }
+
+    if (obj[fieldKeys[0]] === undefined) {
+      return isNestedField([`${fieldKeys[0]}.${fieldKeys[1]}`, ...fieldKeys.slice(2)], obj);
+    }
+  }
+
+  return false;
+};

bklog/web/src/hooks/use-text-segmentation.ts

@@ -81,9 +81,9 @@ export default class UseTextSegmentation {
     }
   }
 
-  getChildNodes() {
+  getChildNodes(forceSplit = false) {
     let start = 0;
-    return this.getSplitList(this.options.field, this.options.content).map(item => {
+    return this.getSplitList(this.options.field, this.options.content, forceSplit).map(item => {
       Object.assign(item, {
         startIndex: start,
         endIndex: start + item.text.length,
@@ -111,6 +111,7 @@ export default class UseTextSegmentation {
     const tippyInstance = segmentPopInstance.getInstance();
     const currentValue = this.clickValue;
     const depth = tippyInstance.reference.closest('[data-depth]')?.getAttribute('data-depth');
+    const isNestedField = tippyInstance.reference.closest('[is-nested-value]')?.getAttribute('is-nested-value');
 
     const activeField = this.getField();
     const target = ['date', 'date_nanos'].includes(activeField?.field_type)
@@ -122,6 +123,7 @@ export default class UseTextSegmentation {
       operation: val === 'not' ? 'is not' : val,
       value: (target ?? currentValue).replace(/<mark>/g, '').replace(/<\/mark>/g, ''),
       depth,
+      isNestedField,
     };
 
     this.onSegmentClick?.({ option, isLink });
@@ -217,11 +219,11 @@ export default class UseTextSegmentation {
       : val.replace(RegExp(`(${Object.keys(map).join('|')})`, 'g'), match => map[match]);
   }
 
-  private getSplitList(field: any, content: any) {
+  private getSplitList(field: any, content: any, forceSplit = false) {
     /** 检索高亮分词字符串 */
     const markRegStr = '<mark>(.*?)</mark>';
     const value = this.escapeString(`${content}`);
-    if (this.isAnalyzed(field)) {
+    if (this.isAnalyzed(field) || forceSplit) {
       // 这里进来的都是开了分词的情况
       return this.splitParticipleWithStr(value, this.getCurrentFieldRegStr(field));
     }

bklog/web/src/store/index.js

@@ -1413,6 +1413,7 @@ const store = new Vuex.Store({
       const isLink = newQueryList[0]?.isLink;
       const searchMode = state.indexItem.search_mode;
       const depth = Number(payload.depth ?? '0');
+      const isNestedField = payload?.isNestedField ?? 'false';
       const isNewSearchPage = newQueryList[0].operator === 'new-search-page-is';
       const getFieldType = field => {
         const target = state.indexFieldInfo.fields?.find(item => item.field_name === field);
@@ -1454,7 +1455,7 @@ const store = new Vuex.Store({
           }
         }
 
-        if (depth > 1 && textType === 'keyword') {
+        if ((depth > 1 || isNestedField === 'true') && textType === 'keyword') {
           mappingKey = keywordMappingKey;
         }
         return mappingKey[operator] ?? operator; // is is not 值映射

bklog/web/src/views/retrieve-v2/result-comp/kv-list.vue

@@ -233,10 +233,10 @@
       handleJsonSegmentClick({ isLink, option }, fieldName) {
         // 为了兼容旧的逻辑，先这么写吧
         // 找时间梳理下这块，写的太随意了
-        const { operation, value, depth } = option;
+        const { operation, value, depth, isNestedField } = option;
         const operator = operation === 'not' ? 'is not' : operation;
         const field = this.totalFields.find(f => f.field_name === fieldName);
-        this.$emit('value-click', operator, value, isLink, field, depth);
+        this.$emit('value-click', operator, value, isLink, field, depth, isNestedField);
       },
 
       /**

bklog/web/src/views/retrieve-v2/search-result-panel/log-result/log-rows.tsx

@@ -113,7 +113,7 @@ export default defineComponent({
     const timeField = computed(() => indexFieldInfo.value.time_field);
     const timeFieldType = computed(() => indexFieldInfo.value.time_field_type);
     const isLoading = computed(() => indexSetQueryResult.value.is_loading || indexFieldInfo.value.is_loading);
-    const kvShowFieldsList = computed(() => Object.keys(indexSetQueryResult.value?.fields ?? {}) || []);
+    const kvShowFieldsList = computed(() => indexFieldInfo.value?.fields.map(f => f.field_name));
     const userSettingConfig = computed(() => store.state.retrieve.catchFieldCustomConfig);
     const tableDataSize = computed(() => indexSetQueryResult.value?.list?.length ?? 0);
     const fieldRequestCounter = computed(() => indexFieldInfo.value.request_counter);
@@ -304,7 +304,9 @@ export default defineComponent({
               content={getTableColumnContent(row, field)}
               field={field}
               row={row}
-              onIcon-click={(type, content, isLink, depth) => handleIconClick(type, content, field, row, isLink, depth)}
+              onIcon-click={(type, content, isLink, depth, isNestedField) =>
+                handleIconClick(type, content, field, row, isLink, depth, isNestedField)
+              }
             ></TableColumn>
           );
         },
@@ -441,9 +443,9 @@ export default defineComponent({
       });
     };
 
-    const handleAddCondition = (field, operator, value, isLink = false, depth = undefined) => {
+    const handleAddCondition = (field, operator, value, isLink = false, depth = undefined, isNestedField = 'false') => {
       store
-        .dispatch('setQueryCondition', { field, operator, value, isLink, depth })
+        .dispatch('setQueryCondition', { field, operator, value, isLink, depth, isNestedField })
         .then(([newSearchList, searchMode, isNewSearchPage]) => {
           setRouteParams();
           if (isLink) {
@@ -467,7 +469,7 @@ export default defineComponent({
       }
     };
 
-    const handleIconClick = (type, content, field, row, isLink, depth) => {
+    const handleIconClick = (type, content, field, row, isLink, depth, isNestedField) => {
       let value = ['date', 'date_nanos'].includes(field.field_type) ? row[field.field_name] : content;
       value = String(value)
         .replace(/<mark>/g, '')
@@ -480,7 +482,7 @@ export default defineComponent({
 
       if (type === 'search') {
         // 将表格单元添加到过滤条件
-        handleAddCondition(field.field_name, 'eq', [value], isLink);
+        handleAddCondition(field.field_name, 'eq', [value], isLink, depth, isNestedField);
         return;
       }
 
@@ -491,7 +493,7 @@ export default defineComponent({
       }
 
       if (['is', 'is not', 'new-search-page-is'].includes(type)) {
-        handleAddCondition(field.field_name, type, value === '--' ? [] : [value], isLink, depth);
+        handleAddCondition(field.field_name, type, value === '--' ? [] : [value], isLink, depth, isNestedField);
         return;
       }
     };
@@ -588,8 +590,8 @@ export default defineComponent({
             data={row}
             kv-show-fields-list={kvShowFieldsList.value}
             list-data={row}
-            onValue-click={(type, content, isLink, field, depth) =>
-              handleIconClick(type, content, field, row, isLink, depth)
+            onValue-click={(type, content, isLink, field, depth, isNestedField) =>
+              handleIconClick(type, content, field, row, isLink, depth, isNestedField)
             }
           ></ExpandView>
         );

bklog/web/src/views/retrieve-v2/search-result-panel/log-result/table-column.vue

@@ -114,9 +114,9 @@
       handleJsonSegmentClick({ isLink, option }) {
         // 为了兼容旧的逻辑，先这么写吧
         // 找时间梳理下这块，写的太随意了
-        const { depth, operation, value } = option;
+        const { depth, operation, value, isNestedField } = option;
         const operator = operation === 'not' ? 'is not' : operation;
-        this.$emit('icon-click', operator, value, isLink, depth); // type, content, field, row, isLink
+        this.$emit('icon-click', operator, value, isLink, depth, isNestedField); // type, content, field, row, isLink
       },
     },
   };

bklog/web/src/views/retrieve-v2/search-result-panel/log-result/text-segmentation.tsx

@@ -25,6 +25,7 @@
  */
 import { ref, computed, inject, watch, defineComponent, Ref, onMounted, onBeforeUnmount, onBeforeMount } from 'vue';
 
+import { isNestedField } from '@/common/util';
 import useLocale from '@/hooks/use-locale';
 import useResizeObserve from '@/hooks/use-resize-observe';
 import useStore from '@/hooks/use-store';
@@ -92,6 +93,7 @@ export default defineComponent({
     });
 
     const handleMenuClick = event => {
+      console.log('handleMenuClick', event);
       emit('menu-click', event);
     };
 
@@ -346,9 +348,10 @@ export default defineComponent({
       if (getSegmentRenderType() === 'text') {
         let max = Number.MAX_SAFE_INTEGER;
         if (!showAll.value) {
-          max = 4;
-          pageIndex = 0;
-          refSegmentContent.value.innerHTML = '';
+          // max = 4;
+          // pageIndex = 0;
+          // textSegmentIndex = 0;
+          // refSegmentContent.value.innerHTML = '';
         }
         setTextSegmentChildNodes(max);
       }
@@ -398,13 +401,49 @@ export default defineComponent({
       }
 
       if (getSegmentRenderType() === 'text') {
+        refSegmentContent.value.setAttribute('is-nested-value', `${isNestedValue}`);
         setTextSegmentChildNodes(maxLength);
       }
     };
 
+    // const isNestedField = (fieldKeys: string[], obj: Record<string, any>) => {
+    //   if (!obj) {
+    //     return false;
+    //   }
+
+    //   if (fieldKeys.length > 1) {
+    //     if (obj[fieldKeys[0]] !== undefined && obj[fieldKeys[0]] !== null) {
+    //       if (typeof obj[fieldKeys[0]] === 'object') {
+    //         if (Array.isArray(obj[fieldKeys[0]])) {
+    //           return true;
+    //         }
+
+    //         return isNestedField(fieldKeys.slice(1), obj[fieldKeys[0]]);
+    //       }
+
+    //       return false;
+    //     }
+
+    //     if (obj[fieldKeys[0]] === undefined) {
+    //       return isNestedField([`${fieldKeys[0]}.${fieldKeys[1]}`, ...fieldKeys.slice(2)], obj);
+    //     }
+    //   }
+
+    //   return false;
+    // };
+
+    let isNestedValue = false; // data-depth
+    const setWordList = () => {
+      const fieldName = props.field.field_name;
+      const fieldKeys = fieldName.split('.');
+      isNestedValue = isNestedField(fieldKeys, props.data);
+
+      wordList = textSegmentInstance.getChildNodes(isNestedValue);
+    };
+
     onBeforeMount(() => {
       isDispose = false;
-      wordList = textSegmentInstance.getChildNodes();
+      setWordList();
     });
 
     onMounted(() => {
@@ -446,7 +485,7 @@ export default defineComponent({
           },
         });
 
-        wordList = textSegmentInstance.getChildNodes();
+        setWordList();
         resetMounted();
         setMounted();
       },