regex - ElasticSearch可视化按字段 “message”中的某些关键字分组

Question

我有带有“消息”字段的日志，其中包含如下内容:

...
com.example.exception.ServiceException: Order is already completed with different status.
    --------------------------------
{
  "errorCode" : "com.example.exception.ErrorCode.CONTROVERSIAL_PAYMENT_STATUS",
  "notification_status" : "failed", // <<<< could be other values
  "order_id" : "6b30d1dd-be97-4d52-a498-4c9d3ddea84d",
  "order_status" : "accepted" // <<<<< also could be other values
}
...

我想按"notification_status"和"order_status"之后的词进行汇总，这样我就可以知道这两种组合比其他组合发生的次数更多。
我能做什么？
编辑:我认为最好在字段stack_trace中进行搜索。
完整文档如下:

{
    "_index": "application-prod",
    "_type": "application-prod",
    "_id": "1hDDeHMB_M3-uozCT51K",
    "_version": 1,
    "_score": null,
    "_source": {
        "container_id": "88078d7f80436eae2fd571663c36c935eb99b03990eff25f39b92b49c017b976",
        "container_name": "/app-container",
        "source": "stdout",
        "log": "{\"timestamp\":\"2020-07-22T23:02:00.959+00:00\",\"message\":\"Failed processing notification. NotificationDto(orderId=efbb5571-cf70-46f7-9cc9-28437394c63c, reference=null, state=error, checkoutStatus=FAIL, signature=hmac-sha256:xxxxxxx, transaction=null, subReferences={})\",\"logger_name\":\"com.example.NotificationApiService\",\"thread_name\":\"elastic-1583\",\"severity\":\"ERROR\",\"stack_trace\":\"com.example.ServiceException: Order is already completed with different status.\\n\\t--------------------------------\\n{\\n  \\\"errorCode\\\" : \\\"com.example.CONTROVERSIAL_PAYMENT_STATUS\\\",\\n  \\\"notification_status\\\" : \\\"error\\\",\\n  \\\"order_id\\\" : \\\"efbb5571-cf70-46f7-9cc9-28437394c63c\\\",\\n  \\\"order_status\\\" : \\\"accepted\\\"\\n}\\n\\t--------------------------------\\n\\tat com.example.NotificationFlowSteps.updateOrderStatusAndTransactions(NotificationFlowSteps.java:38)\\n\\tat com.example.NotificationFlowSteps$$FastClassBySpringCGLIB$$e8bc6a04.invoke(<generated>)\\n\\tat org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:218)\\n\\tat org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:771)\\n\\tat org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)\\n\\tat org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:749)\\n\\tat org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:366)\\n\\tat org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:118)\\n\\tat org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186)\\n\\tat org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:749)\\n\\tat org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:691)\\n\\tat com.example.NotificationFlowSteps$$EnhancerBySpringCGLIB$$3f256eb1.updateOrderStatusAndTransactions(<generated>)\\n\\tat com.example.NotificationFlowService.lambda$processNotification$0(NotificationFlowService.java:58)\\n\\t... 7 frames excluded\\n\\tat java.util.concurrent.FutureTask.run(Unknown Source)\\n\\tat java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source)\\n\\tat java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)\\n\\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)\\n\\tat java.lang.Thread.run(Unknown Source)\\n\",\"ASYNC_INCLUDE_CALLER_DATA\":\"false\",\"HOSTNAME\":\"88078d7f8043\",\"ASYNC_DISCARDING_THRESHOLD\":\"-1\",\"ASYNC_MAX_FLUSH_TIME\":\"1000\",\"ASYNC_QUEUE_SIZE\":\"256\",\"env\":\"prod\",\"productName\":\"my-app\",\"ASYNC_NEVER_BLOCK\":\"false\",\"host\":\"88078d7f8043\",\"message\":\"Failed processing notification. NotificationDto(orderId=efbb5571-cf70-46f7-9cc9-28437394c63c, reference=null, state=error, checkoutStatus=FAIL, signature=hmac-sha256:xxxxxxx, transaction=null, subReferences={})\",\"timestamp\":\"2020-07-22T23:02:00.959Z\",\"productName\":\"my-app\",\"env\":\"prod\",\"severity\":\"ERROR\",\"trace\":\"\",\"span\":\"\",\"thread\":\"elastic-1583\",\"class\":\"e.v.c.c.s.NotificationApiService\",\"type\":\"APPLICATION\"}",
        "timestamp": "2020-07-22T23:02:00.959Z",
        "message": "Failed processing notification. NotificationDto(orderId=efbb5571-cf70-46f7-9cc9-28437394c63c, reference=null, state=error, checkoutStatus=FAIL, signature=hmac-sha256:xxxxxxx, transaction=null, subReferences={})",
        "logger_name": "com.example.NotificationApiService",
        "thread_name": "elastic-1583",
        "severity": "ERROR",
        "stack_trace": "com.example.exception.ServiceException: Order is already completed with different status.\n\t--------------------------------\n{\n  \"errorCode\" : \"com.example.exception.ErrorCode.CONTROVERSIAL_PAYMENT_STATUS\",\n  \"notification_status\" : \"error\",\n  \"order_id\" : \"efbb5571-cf70-46f7-9cc9-28437394c63c\",\n  \"order_status\" : \"accepted\"\n}\n\t--------------------------------\n\tat com.example.NotificationFlowSteps.updateOrderStatusAndTransactions(NotificationFlowSteps.java:38)\n\tat com.example.NotificationFlowSteps$$FastClassBySpringCGLIB$$e8bc6a04.invoke(<generated>)\n\tat org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:218)\n\tat org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:771)\n\tat org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)\n\tat org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:749)\n\tat org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:366)\n\tat org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:118)\n\tat org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186)\n\tat org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:749)\n\tat org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:691)\n\tat com.example.NotificationFlowSteps$$EnhancerBySpringCGLIB$$3f256eb1.updateOrderStatusAndTransactions(<generated>)\n\tat com.example.NotificationFlowService.lambda$processNotification$0(NotificationFlowService.java:58)\n\t... 7 frames excluded\n\tat java.util.concurrent.FutureTask.run(Unknown Source)\n\tat java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)\n\tat java.lang.Thread.run(Unknown Source)\n",
        "ASYNC_INCLUDE_CALLER_DATA": "false",
        "HOSTNAME": "88078d7f8043",
        "ASYNC_DISCARDING_THRESHOLD": "-1",
        "ASYNC_MAX_FLUSH_TIME": "1000",
        "ASYNC_QUEUE_SIZE": "256",
        "env": "prod",
        "productName": "my-app",
        "ASYNC_NEVER_BLOCK": "false",
        "host": "88078d7f8043",
        "trace": "",
        "span": "",
        "thread": "elastic-1583",
        "class": "e.v.c.c.s.NotificationApiService",
        "type": "APPLICATION"
    },
    "fields": {
        "timestamp": [
            "2020-07-22T23:02:00.959Z"
        ]
    },
    "highlight": {
        "log": [
            "NotificationDto(orderId=efbb5571-cf70-46f7-9cc9-28437394c63c, reference=null, state=error, checkoutStatus=FAIL, signature=hmac-sha256:xxxxxxx, transaction=null, subReferences={})\",\"logger_name\":\"com.example.NotificationApiService\",\"thread_name\":\"elastic-1583\",\"severity\":\"ERROR\",\"stack_trace\":\"com.example.exception.ServiceException: @kibana-highlighted-field@Order@/kibana-highlighted-field@ @kibana-highlighted-field@is@/kibana-highlighted-field@ @kibana-highlighted-field@already@/kibana-highlighted-field@ @kibana-highlighted-field@completed@/kibana-highlighted-field@ @kibana-highlighted-field@with@/kibana-highlighted-field@ @kibana-highlighted-field@different@/kibana-highlighted-field@ @kibana-highlighted-field@status@/kibana-highlighted-field@.\\n\\t--------------------------------\\n{\\n  \\\"errorCode\\\" : \\\"com.example.exception.ErrorCode.CONTROVERSIAL_PAYMENT_STATUS\\\",\\n  \\\"notification_status\\\" : \\\"error\\\",\\n  \\\"order_id\\\" : \\\"efbb5571-cf70-46f7-9cc9-28437394c63c\\\",\\n  \\\"order_status\\\" : \\\"accepted\\\"\\n}\\n\\t--------------------------------\\n\\tat com.example.NotificationFlowSteps.updateOrderStatusAndTransactions(NotificationFlowSteps.java:38)\\n\\tat com.example.NotificationFlowSteps$$FastClassBySpringCGLIB$$e8bc6a04.invoke(<generated>)\\n\\tat org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:218)\\n\\tat org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:771)\\n\\tat org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)\\n\\tat org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:749)\\n\\tat org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:366)\\n\\tat org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:118)\\n\\tat org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186)\\n\\tat org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:749)\\n\\tat org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:691)\\n\\tat com.example.NotificationFlowSteps$$EnhancerBySpringCGLIB$$3f256eb1.updateOrderStatusAndTransactions(<generated>)\\n\\tat com.example.NotificationFlowService.lambda$processNotification$0(NotificationFlowService.java:58)\\n\\t... 7 frames excluded\\n\\tat java.util.concurrent.FutureTask.run(Unknown Source)\\n\\tat java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source)\\n\\tat java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)\\n\\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)\\n\\tat java.lang.Thread.run(Unknown Source)\\n\",\"ASYNC_INCLUDE_CALLER_DATA\":\"false\",\"HOSTNAME\":\"88078d7f8043\",\"ASYNC_DISCARDING_THRESHOLD\":\"-1\",\"ASYNC_MAX_FLUSH_TIME\":\"1000\",\"ASYNC_QUEUE_SIZE\":\"256\",\"env\":\"prod\",\"productName\":\"my-app\",\"ASYNC_NEVER_BLOCK\":\"false\",\"host\":\"88078d7f8043\",\"message\":\"Failed processing notification. NotificationDto(orderId=efbb5571-cf70-46f7-9cc9-28437394c63c, reference=null, state=error, checkoutStatus=FAIL, signature=hmac-sha256:xxxxxxx, transaction=null, subReferences={})\",\"timestamp\":\"2020-07-22T23:02:00.959Z\",\"productName\":\"my-app\",\"env\":\"prod\",\"severity\":\"ERROR\",\"trace\":\"\",\"span\":\"\",\"thread\":\"elastic-1583\",\"class\":\"e.v.c.c.s.NotificationApiService\",\"type\":\"APPLICATION\"}"
        ],
        "stack_trace": [
            "com.example.exception.ServiceException: @kibana-highlighted-field@Order@/kibana-highlighted-field@ @kibana-highlighted-field@is@/kibana-highlighted-field@ @kibana-highlighted-field@already@/kibana-highlighted-field@ @kibana-highlighted-field@completed@/kibana-highlighted-field@ @kibana-highlighted-field@with@/kibana-highlighted-field@ @kibana-highlighted-field@different@/kibana-highlighted-field@ @kibana-highlighted-field@status@/kibana-highlighted-field@.\n\t--------------------------------\n{\n  \"errorCode\" : \"com.example.exception.ErrorCode.CONTROVERSIAL_PAYMENT_STATUS\",\n  \"notification_status\" : \"error\",\n  \"order_id\" : \"efbb5571-cf70-46f7-9cc9-28437394c63c\",\n  \"order_status\" : \"accepted\"\n}\n\t--------------------------------\n\tat com.example.NotificationFlowSteps.updateOrderStatusAndTransactions(NotificationFlowSteps.java:38)\n\tat com.example.NotificationFlowSteps$$FastClassBySpringCGLIB$$e8bc6a04.invoke(<generated>)\n\tat org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:218)\n\tat org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:771)\n\tat org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)\n\tat org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:749)\n\tat org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:366)\n\tat org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:118)\n\tat org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186)\n\tat org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:749)\n\tat org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:691)\n\tat com.example.NotificationFlowSteps$$EnhancerBySpringCGLIB$$3f256eb1.updateOrderStatusAndTransactions(<generated>)\n\tat com.example.NotificationFlowService.lambda$processNotification$0(NotificationFlowService.java:58)\n\t... 7 frames excluded\n\tat java.util.concurrent.FutureTask.run(Unknown Source)\n\tat java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)\n\tat java.lang.Thread.run(Unknown Source)"
        ]
    },
    "sort": [
        1595458920959
    ]
}

Answer 1

您可以使用以下模板，其中使用了示例映射，文档，使用的聚合查询以及最后的响应:
对应:

PUT my_error_index
{
  "mappings": {
    "properties": {
      "errorCode": {
        "type": "keyword"
      },
      "notification_status": {
        "type": "keyword"
      },
      "order_id": {
        "type": "keyword"
      },
      "order_status": {
        "type": "keyword"
      }
    }
  }
}

请注意，您也可以通过使用 text 来使用 multi-field ，但是对于您所拥有的查询的性质，我需要利用聚合查询，从而使用keyword类型。
样本文件:

POST my_error_index/_doc/1
{
  "errorCode" : "com.example.exception.ErrorCode.CONTROVERSIAL_PAYMENT_STATUS",
  "notification_status" : "failed",
  "order_id" : "6b30d1dd-be97-4d52-a498-4c9d3ddea84d",
  "order_status" : "accepted"
}

POST my_error_index/_doc/2
{
  "errorCode" : "com.example.exception.ErrorCode.CONTROVERSIAL_PAYMENT_STATUS",
  "notification_status" : "success",
  "order_id" : "6b30d1dd-be97-4d52-a498-4c9d3ddea85d",
  "order_status" : "rejected"
}

POST my_error_index/_doc/3
{
  "errorCode" : "com.example.exception.ErrorCode.CONTROVERSIAL_PAYMENT_STATUS",
  "notification_status" : "success",
  "order_id" : "6b30d1dd-be97-4d52-a498-4c9d3ddea86d",
  "order_status" : "accepted"
}

POST my_error_index/_doc/4
{
  "errorCode" : "com.example.exception.ErrorCode.CONTROVERSIAL_PAYMENT_STATUS",
  "notification_status" : "failed",
  "order_id" : "6b30d1dd-be97-4d52-a498-4c9d3ddea87d",
  "order_status" : "rejected"
}

请注意，我用failed和success的notification_status和accepted和rejected值的order_status构建了4个不同的文档
汇总查询:

POST my_error_index/_search
{
  "size": 0,
  "aggs": {
    "my_notification_status": {
      "terms": {
        "field": "notification_status",
        "size": 10
      },
      "aggs": {
        "my_error_status": {
          "terms": {
            "field": "order_status",
            "size": 10
          }
        }
      }
    }
  }
}

请注意，我已经将Terms Aggregation Query用于notification status，并进一步将另一个Terms Aggregation Query链接到它以用于order_status响应:

{
  "took" : 198,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 4,
      "relation" : "eq"
    },
    "max_score" : null,
    "hits" : [ ]
  },
  "aggregations" : {
    "my_notification_status" : {
      "doc_count_error_upper_bound" : 0,
      "sum_other_doc_count" : 0,
      "buckets" : [
        {
          "key" : "failed",                          <---- For Failed notification status
          "doc_count" : 2,
          "my_error_status" : {
            "doc_count_error_upper_bound" : 0,
            "sum_other_doc_count" : 0,
            "buckets" : [
              {
                "key" : "accepted",
                "doc_count" : 1
              },
              {
                "key" : "rejected",
                "doc_count" : 1
              }
            ]
          }
        },
        {
          "key" : "success",                         <--- For success notification status
          "doc_count" : 2,
          "my_error_status" : {
            "doc_count_error_upper_bound" : 0,
            "sum_other_doc_count" : 0,
            "buckets" : [
              {
                "key" : "accepted",
                "doc_count" : 1
              },
              {
                "key" : "rejected",
                "doc_count" : 1
              }
            ]
          }
        }
      ]
    }
  }
}

让我知道这是否有帮助!