From 500af61cc461d3220685a3d61e3238cb87c3bfd2 Mon Sep 17 00:00:00 2001
From: wuxiangkai <wuxiangkai@yonyou.com>
Date: Mon, 30 Dec 2024 19:31:46 +0800
Subject: [PATCH] =?UTF-8?q?=E9=98=85=E8=AF=BBes=20pipeline=20aggregation?=
 =?UTF-8?q?=E6=96=87=E6=A1=A3?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 elastic search/elastic search.md | 964 ++++++++++++++++++++++++++++++-
 1 file changed, 963 insertions(+), 1 deletion(-)

diff --git a/elastic search/elastic search.md b/elastic search/elastic search.md
index 80c7f65..69050ae 100644
--- a/elastic search/elastic search.md	
+++ b/elastic search/elastic search.md	
@@ -1496,4 +1496,966 @@ GET kibana_sample_data_ecommerce/_search
 - `quarter, 1q`
 - `year, 1y`
 
-#### 
\ No newline at end of file
+#### compare category performance
+```
+GET kibana_sample_data_ecommerce/_search
+{
+ "size": 0,
+ "aggs": {
+   "categories": {
+     "terms": {
+       "field": "category.keyword",
+       "size": 5,
+       "order": { "total_revenue": "desc" } 
+     },
+     "aggs": { 
+       "total_revenue": { 
+         "sum": {
+           "field": "taxful_total_price"
+         }
+       },
+       "avg_order_value": { 
+         "avg": {
+           "field": "taxful_total_price"
+         }
+       },
+       "total_items": { 
+         "sum": {
+           "field": "total_quantity"
+         }
+       }
+     }
+   }
+ }
+}
+```
+
+如上所示，在按`category.keyword`分组聚合后，其又在`aggs.categories.aggs`中指定了针对每个category计算的metrics。
+
+上述请求的返回结果如下所示：
+```
+{
+ "aggregations": {
+   "categories": {
+     "buckets": [
+       {
+         "key": "Men's Clothing", 
+         "doc_count": 2179, 
+         "total_revenue": { 
+           "value": 156729.453125
+         },
+         "avg_order_value": { 
+           "value": 71.92726898715927
+         },
+         "total_items": { 
+           "value": 8716
+         }
+       },
+       {
+         "key": "Women's Clothing",
+         "doc_count": 2262,
+         ...
+       }
+     ]
+   }
+ }
+}
+```
+#### analyze daily sale performance
+```
+GET kibana_sample_data_ecommerce/_search
+{
+ "size": 0,
+ "aggs": {
+   "daily_sales": {
+     "date_histogram": {
+       "field": "order_date",
+       "calendar_interval": "day",
+       "format": "yyyy-MM-dd"
+     },
+     "aggs": {
+       "revenue": { 
+         "sum": {
+           "field": "taxful_total_price"
+         }
+       },
+       "unique_customers": { 
+         "cardinality": {
+           "field": "customer_id"
+         }
+       },
+       "avg_basket_size": { 
+         "avg": {
+           "field": "total_quantity"
+         }
+       }
+     }
+   }
+ }
+}
+```
+上述请求首先按照order_date字段按天进行了分组，然后针对每天的订单，继续计算sum, cardinality, avg聚合。
+
+其中，cardinality会计算每天不重复的客户个数（distinct）。
+
+上述请求返回结构如下所示：
+```
+{
+  "took": 119,
+  "timed_out": false,
+  "_shards": {
+    "total": 5,
+    "successful": 5,
+    "skipped": 0,
+    "failed": 0
+  },
+  "hits": {
+    "total": {
+      "value": 4675,
+      "relation": "eq"
+    },
+    "max_score": null,
+    "hits": []
+  },
+  "aggregations": {
+    "daily_sales": {
+      "buckets": [
+        {
+          "key_as_string": "2024-11-14",
+          "key": 1731542400000,
+          "doc_count": 146,
+          "unique_customers": { 
+            "value": 42
+          },
+          "revenue": { 
+            "value": 10578.53125
+          },
+          "avg_basket_size": { 
+            "value": 2.1780821917808217
+          }
+        },
+        {
+          "key_as_string": "2024-11-15",
+          "key": 1731628800000,
+          "doc_count": 153,
+          "unique_customers": {
+            "value": 44
+          },
+          "revenue": {
+            "value": 10448
+          },
+          "avg_basket_size": {
+            "value": 2.183006535947712
+          }
+        },
+        {
+          "key_as_string": "2024-11-16",
+          "key": 1731715200000,
+          "doc_count": 143,
+          "unique_customers": {
+            "value": 45
+          },
+          "revenue": {
+            "value": 10283.484375
+          },
+          "avg_basket_size": {
+            "value": 2.111888111888112
+          }
+        },
+        {
+          "key_as_string": "2024-11-17",
+          "key": 1731801600000,
+          "doc_count": 140,
+          "unique_customers": {
+            "value": 42
+          },
+          "revenue": {
+            "value": 10145.5234375
+          },
+          "avg_basket_size": {
+            "value": 2.142857142857143
+          }
+        },
+        {
+          "key_as_string": "2024-11-18",
+          "key": 1731888000000,
+          "doc_count": 139,
+          "unique_customers": {
+            "value": 42
+          },
+          "revenue": {
+            "value": 12012.609375
+          },
+          "avg_basket_size": {
+            "value": 2.158273381294964
+          }
+        },
+        {
+          "key_as_string": "2024-11-19",
+          "key": 1731974400000,
+          "doc_count": 157,
+          "unique_customers": {
+            "value": 43
+          },
+          "revenue": {
+            "value": 11009.45703125
+          },
+          "avg_basket_size": {
+            "value": 2.0955414012738856
+          }
+        },
+        {
+          "key_as_string": "2024-11-20",
+          "key": 1732060800000,
+          "doc_count": 145,
+          "unique_customers": {
+            "value": 44
+          },
+          "revenue": {
+            "value": 10720.59375
+          },
+          "avg_basket_size": {
+            "value": 2.179310344827586
+          }
+        },
+        {
+          "key_as_string": "2024-11-21",
+          "key": 1732147200000,
+          "doc_count": 152,
+          "unique_customers": {
+            "value": 43
+          },
+          "revenue": {
+            "value": 11185.3671875
+          },
+          "avg_basket_size": {
+            "value": 2.1710526315789473
+          }
+        },
+        {
+          "key_as_string": "2024-11-22",
+          "key": 1732233600000,
+          "doc_count": 163,
+          "unique_customers": {
+            "value": 44
+          },
+          "revenue": {
+            "value": 13560.140625
+          },
+          "avg_basket_size": {
+            "value": 2.2576687116564416
+          }
+        },
+        {
+          "key_as_string": "2024-11-23",
+          "key": 1732320000000,
+          "doc_count": 141,
+          "unique_customers": {
+            "value": 45
+          },
+          "revenue": {
+            "value": 9884.78125
+          },
+          "avg_basket_size": {
+            "value": 2.099290780141844
+          }
+        },
+        {
+          "key_as_string": "2024-11-24",
+          "key": 1732406400000,
+          "doc_count": 151,
+          "unique_customers": {
+            "value": 44
+          },
+          "revenue": {
+            "value": 11075.65625
+          },
+          "avg_basket_size": {
+            "value": 2.0927152317880795
+          }
+        },
+        {
+          "key_as_string": "2024-11-25",
+          "key": 1732492800000,
+          "doc_count": 143,
+          "unique_customers": {
+            "value": 41
+          },
+          "revenue": {
+            "value": 10323.8515625
+          },
+          "avg_basket_size": {
+            "value": 2.167832167832168
+          }
+        },
+        {
+          "key_as_string": "2024-11-26",
+          "key": 1732579200000,
+          "doc_count": 143,
+          "unique_customers": {
+            "value": 44
+          },
+          "revenue": {
+            "value": 10369.546875
+          },
+          "avg_basket_size": {
+            "value": 2.167832167832168
+          }
+        },
+        {
+          "key_as_string": "2024-11-27",
+          "key": 1732665600000,
+          "doc_count": 142,
+          "unique_customers": {
+            "value": 46
+          },
+          "revenue": {
+            "value": 11711.890625
+          },
+          "avg_basket_size": {
+            "value": 2.1971830985915495
+          }
+        },
+        {
+          "key_as_string": "2024-11-28",
+          "key": 1732752000000,
+          "doc_count": 161,
+          "unique_customers": {
+            "value": 43
+          },
+          "revenue": {
+            "value": 12612.6640625
+          },
+          "avg_basket_size": {
+            "value": 2.1180124223602483
+          }
+        },
+        {
+          "key_as_string": "2024-11-29",
+          "key": 1732838400000,
+          "doc_count": 144,
+          "unique_customers": {
+            "value": 42
+          },
+          "revenue": {
+            "value": 10176.87890625
+          },
+          "avg_basket_size": {
+            "value": 2.0347222222222223
+          }
+        },
+        {
+          "key_as_string": "2024-11-30",
+          "key": 1732924800000,
+          "doc_count": 157,
+          "unique_customers": {
+            "value": 43
+          },
+          "revenue": {
+            "value": 11480.33203125
+          },
+          "avg_basket_size": {
+            "value": 2.159235668789809
+          }
+        },
+        {
+          "key_as_string": "2024-12-01",
+          "key": 1733011200000,
+          "doc_count": 158,
+          "unique_customers": {
+            "value": 42
+          },
+          "revenue": {
+            "value": 11533.265625
+          },
+          "avg_basket_size": {
+            "value": 2.0822784810126582
+          }
+        },
+        {
+          "key_as_string": "2024-12-02",
+          "key": 1733097600000,
+          "doc_count": 144,
+          "unique_customers": {
+            "value": 43
+          },
+          "revenue": {
+            "value": 10499.8125
+          },
+          "avg_basket_size": {
+            "value": 2.201388888888889
+          }
+        },
+        {
+          "key_as_string": "2024-12-03",
+          "key": 1733184000000,
+          "doc_count": 151,
+          "unique_customers": {
+            "value": 40
+          },
+          "revenue": {
+            "value": 12111.6875
+          },
+          "avg_basket_size": {
+            "value": 2.172185430463576
+          }
+        },
+        {
+          "key_as_string": "2024-12-04",
+          "key": 1733270400000,
+          "doc_count": 145,
+          "unique_customers": {
+            "value": 40
+          },
+          "revenue": {
+            "value": 10530.765625
+          },
+          "avg_basket_size": {
+            "value": 2.0965517241379312
+          }
+        },
+        {
+          "key_as_string": "2024-12-05",
+          "key": 1733356800000,
+          "doc_count": 157,
+          "unique_customers": {
+            "value": 43
+          },
+          "revenue": {
+            "value": 11872.5625
+          },
+          "avg_basket_size": {
+            "value": 2.1464968152866244
+          }
+        },
+        {
+          "key_as_string": "2024-12-06",
+          "key": 1733443200000,
+          "doc_count": 158,
+          "unique_customers": {
+            "value": 42
+          },
+          "revenue": {
+            "value": 12109.453125
+          },
+          "avg_basket_size": {
+            "value": 2.151898734177215
+          }
+        },
+        {
+          "key_as_string": "2024-12-07",
+          "key": 1733529600000,
+          "doc_count": 153,
+          "unique_customers": {
+            "value": 42
+          },
+          "revenue": {
+            "value": 11057.40625
+          },
+          "avg_basket_size": {
+            "value": 2.111111111111111
+          }
+        },
+        {
+          "key_as_string": "2024-12-08",
+          "key": 1733616000000,
+          "doc_count": 165,
+          "unique_customers": {
+            "value": 42
+          },
+          "revenue": {
+            "value": 13095.609375
+          },
+          "avg_basket_size": {
+            "value": 2.1818181818181817
+          }
+        },
+        {
+          "key_as_string": "2024-12-09",
+          "key": 1733702400000,
+          "doc_count": 153,
+          "unique_customers": {
+            "value": 41
+          },
+          "revenue": {
+            "value": 12574.015625
+          },
+          "avg_basket_size": {
+            "value": 2.2287581699346406
+          }
+        },
+        {
+          "key_as_string": "2024-12-10",
+          "key": 1733788800000,
+          "doc_count": 158,
+          "unique_customers": {
+            "value": 42
+          },
+          "revenue": {
+            "value": 11188.1875
+          },
+          "avg_basket_size": {
+            "value": 2.151898734177215
+          }
+        },
+        {
+          "key_as_string": "2024-12-11",
+          "key": 1733875200000,
+          "doc_count": 160,
+          "unique_customers": {
+            "value": 42
+          },
+          "revenue": {
+            "value": 12117.65625
+          },
+          "avg_basket_size": {
+            "value": 2.20625
+          }
+        },
+        {
+          "key_as_string": "2024-12-12",
+          "key": 1733961600000,
+          "doc_count": 159,
+          "unique_customers": {
+            "value": 45
+          },
+          "revenue": {
+            "value": 11558.25
+          },
+          "avg_basket_size": {
+            "value": 2.1823899371069184
+          }
+        },
+        {
+          "key_as_string": "2024-12-13",
+          "key": 1734048000000,
+          "doc_count": 152,
+          "unique_customers": {
+            "value": 45
+          },
+          "revenue": {
+            "value": 11921.1171875
+          },
+          "avg_basket_size": {
+            "value": 2.289473684210526
+          }
+        },
+        {
+          "key_as_string": "2024-12-14",
+          "key": 1734134400000,
+          "doc_count": 142,
+          "unique_customers": {
+            "value": 45
+          },
+          "revenue": {
+            "value": 11135.03125
+          },
+          "avg_basket_size": {
+            "value": 2.183098591549296
+          }
+        }
+      ]
+    }
+  }
+}
+```
+#### pipeline aggregation demo 
+可以对其他聚合的返回结果应用pipeline aggregation，示例如下：
+```
+GET kibana_sample_data_ecommerce/_search
+{
+  "size": 0,
+  "aggs": {
+    "daily_sales": {
+      "date_histogram": {
+        "field": "order_date",
+        "calendar_interval": "day"
+      },
+      "aggs": {
+        "daily_revenue": {  
+          "sum": {
+            "field": "taxful_total_price"
+          }
+        },
+        "smoothed_revenue": { 
+          "moving_fn": { 
+            "buckets_path": "daily_revenue", 
+            "window": 3, 
+            "script": "MovingFunctions.unweightedAvg(values)" 
+          }
+        }
+      }
+    }
+  }
+}
+```
+上述示例中，`aggs.daily_sales.aggs.smoothed_revenue`针对`aggs.daily_sales.aggs.daily_revenue`的聚合结果进行了聚合操作，
+
+上述返回结果如下所示：
+```
+{
+  "took": 13,
+  "timed_out": false,
+  "_shards": {
+    "total": 5,
+    "successful": 5,
+    "skipped": 0,
+    "failed": 0
+  },
+  "hits": {
+    "total": {
+      "value": 4675,
+      "relation": "eq"
+    },
+    "max_score": null,
+    "hits": []
+  },
+  "aggregations": {
+    "daily_sales": {
+      "buckets": [
+        {
+          "key_as_string": "2024-11-14T00:00:00.000Z",  
+          "key": 1731542400000,
+          "doc_count": 146, 
+          "daily_revenue": { 
+            "value": 10578.53125
+          },
+          "smoothed_revenue": { 
+            "value": null
+          }
+        },
+        {
+          "key_as_string": "2024-11-15T00:00:00.000Z",
+          "key": 1731628800000,
+          "doc_count": 153,
+          "daily_revenue": {
+            "value": 10448
+          },
+          "smoothed_revenue": { 
+            "value": 10578.53125
+          }
+        },
+        {
+          "key_as_string": "2024-11-16T00:00:00.000Z",
+          "key": 1731715200000,
+          "doc_count": 143,
+          "daily_revenue": {
+            "value": 10283.484375
+          },
+          "smoothed_revenue": {
+            "value": 10513.265625
+          }
+        },
+        {
+          "key_as_string": "2024-11-17T00:00:00.000Z",
+          "key": 1731801600000,
+          "doc_count": 140,
+          "daily_revenue": {
+            "value": 10145.5234375
+          },
+          "smoothed_revenue": {
+            "value": 10436.671875
+          }
+        },
+        {
+          "key_as_string": "2024-11-18T00:00:00.000Z",
+          "key": 1731888000000,
+          "doc_count": 139,
+          "daily_revenue": {
+            "value": 12012.609375
+          },
+          "smoothed_revenue": {
+            "value": 10292.3359375
+          }
+        },
+        {
+          "key_as_string": "2024-11-19T00:00:00.000Z",
+          "key": 1731974400000,
+          "doc_count": 157,
+          "daily_revenue": {
+            "value": 11009.45703125
+          },
+          "smoothed_revenue": {
+            "value": 10813.872395833334
+          }
+        },
+        {
+          "key_as_string": "2024-11-20T00:00:00.000Z",
+          "key": 1732060800000,
+          "doc_count": 145,
+          "daily_revenue": {
+            "value": 10720.59375
+          },
+          "smoothed_revenue": {
+            "value": 11055.86328125
+          }
+        },
+        {
+          "key_as_string": "2024-11-21T00:00:00.000Z",
+          "key": 1732147200000,
+          "doc_count": 152,
+          "daily_revenue": {
+            "value": 11185.3671875
+          },
+          "smoothed_revenue": {
+            "value": 11247.553385416666
+          }
+        },
+        {
+          "key_as_string": "2024-11-22T00:00:00.000Z",
+          "key": 1732233600000,
+          "doc_count": 163,
+          "daily_revenue": {
+            "value": 13560.140625
+          },
+          "smoothed_revenue": {
+            "value": 10971.805989583334
+          }
+        },
+        {
+          "key_as_string": "2024-11-23T00:00:00.000Z",
+          "key": 1732320000000,
+          "doc_count": 141,
+          "daily_revenue": {
+            "value": 9884.78125
+          },
+          "smoothed_revenue": {
+            "value": 11822.033854166666
+          }
+        },
+        {
+          "key_as_string": "2024-11-24T00:00:00.000Z",
+          "key": 1732406400000,
+          "doc_count": 151,
+          "daily_revenue": {
+            "value": 11075.65625
+          },
+          "smoothed_revenue": {
+            "value": 11543.4296875
+          }
+        },
+        {
+          "key_as_string": "2024-11-25T00:00:00.000Z",
+          "key": 1732492800000,
+          "doc_count": 143,
+          "daily_revenue": {
+            "value": 10323.8515625
+          },
+          "smoothed_revenue": {
+            "value": 11506.859375
+          }
+        },
+        {
+          "key_as_string": "2024-11-26T00:00:00.000Z",
+          "key": 1732579200000,
+          "doc_count": 143,
+          "daily_revenue": {
+            "value": 10369.546875
+          },
+          "smoothed_revenue": {
+            "value": 10428.096354166666
+          }
+        },
+        {
+          "key_as_string": "2024-11-27T00:00:00.000Z",
+          "key": 1732665600000,
+          "doc_count": 142,
+          "daily_revenue": {
+            "value": 11711.890625
+          },
+          "smoothed_revenue": {
+            "value": 10589.684895833334
+          }
+        },
+        {
+          "key_as_string": "2024-11-28T00:00:00.000Z",
+          "key": 1732752000000,
+          "doc_count": 161,
+          "daily_revenue": {
+            "value": 12612.6640625
+          },
+          "smoothed_revenue": {
+            "value": 10801.763020833334
+          }
+        },
+        {
+          "key_as_string": "2024-11-29T00:00:00.000Z",
+          "key": 1732838400000,
+          "doc_count": 144,
+          "daily_revenue": {
+            "value": 10176.87890625
+          },
+          "smoothed_revenue": {
+            "value": 11564.700520833334
+          }
+        },
+        {
+          "key_as_string": "2024-11-30T00:00:00.000Z",
+          "key": 1732924800000,
+          "doc_count": 157,
+          "daily_revenue": {
+            "value": 11480.33203125
+          },
+          "smoothed_revenue": {
+            "value": 11500.477864583334
+          }
+        },
+        {
+          "key_as_string": "2024-12-01T00:00:00.000Z",
+          "key": 1733011200000,
+          "doc_count": 158,
+          "daily_revenue": {
+            "value": 11533.265625
+          },
+          "smoothed_revenue": {
+            "value": 11423.291666666666
+          }
+        },
+        {
+          "key_as_string": "2024-12-02T00:00:00.000Z",
+          "key": 1733097600000,
+          "doc_count": 144,
+          "daily_revenue": {
+            "value": 10499.8125
+          },
+          "smoothed_revenue": {
+            "value": 11063.4921875
+          }
+        },
+        {
+          "key_as_string": "2024-12-03T00:00:00.000Z",
+          "key": 1733184000000,
+          "doc_count": 151,
+          "daily_revenue": {
+            "value": 12111.6875
+          },
+          "smoothed_revenue": {
+            "value": 11171.13671875
+          }
+        },
+        {
+          "key_as_string": "2024-12-04T00:00:00.000Z",
+          "key": 1733270400000,
+          "doc_count": 145,
+          "daily_revenue": {
+            "value": 10530.765625
+          },
+          "smoothed_revenue": {
+            "value": 11381.588541666666
+          }
+        },
+        {
+          "key_as_string": "2024-12-05T00:00:00.000Z",
+          "key": 1733356800000,
+          "doc_count": 157,
+          "daily_revenue": {
+            "value": 11872.5625
+          },
+          "smoothed_revenue": {
+            "value": 11047.421875
+          }
+        },
+        {
+          "key_as_string": "2024-12-06T00:00:00.000Z",
+          "key": 1733443200000,
+          "doc_count": 158,
+          "daily_revenue": {
+            "value": 12109.453125
+          },
+          "smoothed_revenue": {
+            "value": 11505.005208333334
+          }
+        },
+        {
+          "key_as_string": "2024-12-07T00:00:00.000Z",
+          "key": 1733529600000,
+          "doc_count": 153,
+          "daily_revenue": {
+            "value": 11057.40625
+          },
+          "smoothed_revenue": {
+            "value": 11504.260416666666
+          }
+        },
+        {
+          "key_as_string": "2024-12-08T00:00:00.000Z",
+          "key": 1733616000000,
+          "doc_count": 165,
+          "daily_revenue": {
+            "value": 13095.609375
+          },
+          "smoothed_revenue": {
+            "value": 11679.807291666666
+          }
+        },
+        {
+          "key_as_string": "2024-12-09T00:00:00.000Z",
+          "key": 1733702400000,
+          "doc_count": 153,
+          "daily_revenue": {
+            "value": 12574.015625
+          },
+          "smoothed_revenue": {
+            "value": 12087.489583333334
+          }
+        },
+        {
+          "key_as_string": "2024-12-10T00:00:00.000Z",
+          "key": 1733788800000,
+          "doc_count": 158,
+          "daily_revenue": {
+            "value": 11188.1875
+          },
+          "smoothed_revenue": {
+            "value": 12242.34375
+          }
+        },
+        {
+          "key_as_string": "2024-12-11T00:00:00.000Z",
+          "key": 1733875200000,
+          "doc_count": 160,
+          "daily_revenue": {
+            "value": 12117.65625
+          },
+          "smoothed_revenue": {
+            "value": 12285.9375
+          }
+        },
+        {
+          "key_as_string": "2024-12-12T00:00:00.000Z",
+          "key": 1733961600000,
+          "doc_count": 159,
+          "daily_revenue": {
+            "value": 11558.25
+          },
+          "smoothed_revenue": {
+            "value": 11959.953125
+          }
+        },
+        {
+          "key_as_string": "2024-12-13T00:00:00.000Z",
+          "key": 1734048000000,
+          "doc_count": 152,
+          "daily_revenue": {
+            "value": 11921.1171875
+          },
+          "smoothed_revenue": {
+            "value": 11621.364583333334
+          }
+        },
+        {
+          "key_as_string": "2024-12-14T00:00:00.000Z",
+          "key": 1734134400000,
+          "doc_count": 142,
+          "daily_revenue": {
+            "value": 11135.03125
+          },
+          "smoothed_revenue": {
+            "value": 11865.674479166666
+          }
+        }
+      ]
+    }
+  }
+}
+```