在 Elasticsearch 中检索最新的文档需要很长时间

Question

我有一个 Elasticsearch 7.2 索引，目前包含大约 35 亿个文档。大多数查询运行没有任何问题，但由于某种原因，按时间戳排序和检索最新文档需要很长时间。

所以这是查询：

{
  "query": { "match_all": {} },
  "size": 1,
  "sort": [{
      "@timestamp": {
        "order": "desc"
      }
    }
  ]
}

这是我的时间戳字段映射：

{
  "mapping": {
    "properties": {
      "@timestamp": {
        "type": "date"
      }
      // ....
    }
  }
}

这是我的查询配置文件：

{
  "took": 227405,
  "timed_out": false,
  "_shards": {
    "total": 4,
    "successful": 4,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": {
      "value": 10000,
      "relation": "gte"
    },
    "max_score": null,
    "hits": [
      {
        "_index": "my-index",
        "_type": "_doc",
        "_id": "7726392481",
        "_score": null,
        "_source": {
          "@timestamp": "2019-07-08T13:11:21.500",
          // ...
        },
        "sort": [
          1562591481500
        ]
      }
    ]
  },
  "profile": {
    "shards": [
      {
        "id": "[sdKq0LgLRAqxbCjpAPKn_g][my-index][0]",
        "searches": [
          {
            "query": [
              {
                "type": "MatchAllDocsQuery",
                "description": "*:*",
                "time_in_nanos": 40925661356,
                "breakdown": {
                  "set_min_competitive_score_count": 0,
                  "match_count": 0,
                  "shallow_advance_count": 0,
                  "set_min_competitive_score": 0,
                  "next_doc": 40011007556,
                  "match": 0,
                  "next_doc_count": 913921033,
                  "score_count": 0,
                  "compute_max_score_count": 0,
                  "compute_max_score": 0,
                  "advance": 0,
                  "advance_count": 0,
                  "score": 0,
                  "build_scorer_count": 164,
                  "create_weight": 10615,
                  "shallow_advance": 0,
                  "create_weight_count": 1,
                  "build_scorer": 721987
                }
              }
            ],
            "rewrite_time": 6503,
            "collector": [
              {
                "name": "CancellableCollector",
                "reason": "search_cancelled",
                "time_in_nanos": 155966573371,
                "children": [
                  {
                    "name": "SimpleFieldCollector",
                    "reason": "search_top_hits",
                    "time_in_nanos": 78972338307
                  }
                ]
              }
            ]
          }
        ],
        "aggregations": []
      },
      {
        "id": "[sdKq0LgLRAqxbCjpAPKn_g][my-index][1]",
        "searches": [
          {
            "query": [
              {
                "type": "MatchAllDocsQuery",
                "description": "*:*",
                "time_in_nanos": 41782046319,
                "breakdown": {
                  "set_min_competitive_score_count": 0,
                  "match_count": 0,
                  "shallow_advance_count": 0,
                  "set_min_competitive_score": 0,
                  "next_doc": 40847593118,
                  "match": 0,
                  "next_doc_count": 933744955,
                  "score_count": 0,
                  "compute_max_score_count": 0,
                  "compute_max_score": 0,
                  "advance": 0,
                  "advance_count": 0,
                  "score": 0,
                  "build_scorer_count": 166,
                  "create_weight": 2479,
                  "shallow_advance": 0,
                  "create_weight_count": 1,
                  "build_scorer": 705600
                }
              }
            ],
            "rewrite_time": 4551,
            "collector": [
              {
                "name": "CancellableCollector",
                "reason": "search_cancelled",
                "time_in_nanos": 155800230172,
                "children": [
                  {
                    "name": "SimpleFieldCollector",
                    "reason": "search_top_hits",
                    "time_in_nanos": 78854977582
                  }
                ]
              }
            ]
          }
        ],
        "aggregations": []
      },
      {
        "id": "[yUQy-JyKQxigULToVE2VSA][my-index][2]",
        "searches": [
          {
            "query": [
              {
                "type": "MatchAllDocsQuery",
                "description": "*:*",
                "time_in_nanos": 41297021897,
                "breakdown": {
                  "set_min_competitive_score_count": 0,
                  "match_count": 0,
                  "shallow_advance_count": 0,
                  "set_min_competitive_score": 0,
                  "next_doc": 40377677681,
                  "match": 0,
                  "next_doc_count": 918611705,
                  "score_count": 0,
                  "compute_max_score_count": 0,
                  "compute_max_score": 0,
                  "advance": 0,
                  "advance_count": 0,
                  "score": 0,
                  "build_scorer_count": 166,
                  "create_weight": 2962,
                  "shallow_advance": 0,
                  "create_weight_count": 1,
                  "build_scorer": 729382
                }
              }
            ],
            "rewrite_time": 3312,
            "collector": [
              {
                "name": "CancellableCollector",
                "reason": "search_cancelled",
                "time_in_nanos": 163874355059,
                "children": [
                  {
                    "name": "SimpleFieldCollector",
                    "reason": "search_top_hits",
                    "time_in_nanos": 87826427892
                  }
                ]
              }
            ]
          }
        ],
        "aggregations": []
      },
      {
        "id": "[yUQy-JyKQxigULToVE2VSA][my-index][3]",
        "searches": [
          {
            "query": [
              {
                "type": "MatchAllDocsQuery",
                "description": "*:*",
                "time_in_nanos": 41461752119,
                "breakdown": {
                  "set_min_competitive_score_count": 0,
                  "match_count": 0,
                  "shallow_advance_count": 0,
                  "set_min_competitive_score": 0,
                  "next_doc": 40536907380,
                  "match": 0,
                  "next_doc_count": 924035258,
                  "score_count": 0,
                  "compute_max_score_count": 0,
                  "compute_max_score": 0,
                  "advance": 0,
                  "advance_count": 0,
                  "score": 0,
                  "build_scorer_count": 178,
                  "create_weight": 2959,
                  "shallow_advance": 0,
                  "create_weight_count": 1,
                  "build_scorer": 806343
                }
              }
            ],
            "rewrite_time": 3316,
            "collector": [
              {
                "name": "CancellableCollector",
                "reason": "search_cancelled",
                "time_in_nanos": 163785688664,
                "children": [
                  {
                    "name": "SimpleFieldCollector",
                    "reason": "search_top_hits",
                    "time_in_nanos": 87831077038
                  }
                ]
              }
            ]
          }
        ],
        "aggregations": []
      }
    ]
  }
}

它在next_doc 上花费了大量时间，next_doc_count 为 933,744,955。但是既然@timestamp字段被索引了，为什么还要遍历所有文档来排序并返回最新的呢？

编辑：

这些是我的索引统计数据：

{
  "_shards": {
    "total": 4,
    "successful": 4,
    "failed": 0
  },
  "stats": {
    "uuid": "ub-FElCTTsOAihR0aoNlPw",
    "primaries": {
      "docs": {
        "count": 3467397640,
        "deleted": 234593502
      },
      "store": {
        "size_in_bytes": 1087736351822
      },
      "indexing": {
        "index_total": 204398539,
        "index_time_in_millis": 80325395,
        "index_current": 0,
        "index_failed": 0,
        "delete_total": 0,
        "delete_time_in_millis": 0,
        "delete_current": 0,
        "noop_update_total": 0,
        "is_throttled": false,
        "throttle_time_in_millis": 0
      },
      "get": {
        "total": 0,
        "time_in_millis": 0,
        "exists_total": 0,
        "exists_time_in_millis": 0,
        "missing_total": 0,
        "missing_time_in_millis": 0,
        "current": 0
      },
      "search": {
        "open_contexts": 0,
        "query_total": 1524,
        "query_time_in_millis": 14514546,
        "query_current": 0,
        "fetch_total": 1018,
        "fetch_time_in_millis": 171319,
        "fetch_current": 0,
        "scroll_total": 0,
        "scroll_time_in_millis": 0,
        "scroll_current": 0,
        "suggest_total": 0,
        "suggest_time_in_millis": 0,
        "suggest_current": 0
      },
      "merges": {
        "current": 0,
        "current_docs": 0,
        "current_size_in_bytes": 0,
        "total": 192984,
        "total_time_in_millis": 87010422,
        "total_docs": 1248881022,
        "total_size_in_bytes": 394269971046,
        "total_stopped_time_in_millis": 0,
        "total_throttled_time_in_millis": 35698367,
        "total_auto_throttle_in_bytes": 20971520
      },
      "refresh": {
        "total": 377992,
        "total_time_in_millis": 38968248,
        "external_total": 377333,
        "external_total_time_in_millis": 40675901,
        "listeners": 0
      },
      "flush": {
        "total": 383,
        "periodic": 155,
        "total_time_in_millis": 695510
      },
      "warmer": {
        "current": 0,
        "total": 377329,
        "total_time_in_millis": 18259
      },
      "query_cache": {
        "memory_size_in_bytes": 0,
        "total_count": 0,
        "hit_count": 0,
        "miss_count": 0,
        "cache_size": 0,
        "cache_count": 0,
        "evictions": 0
      },
      "fielddata": {
        "memory_size_in_bytes": 17973802844,
        "evictions": 0
      },
      "completion": {
        "size_in_bytes": 0
      },
      "segments": {
        "count": 319,
        "memory_in_bytes": 1174162151,
        "terms_memory_in_bytes": 942165919,
        "stored_fields_memory_in_bytes": 126159816,
        "term_vectors_memory_in_bytes": 0,
        "norms_memory_in_bytes": 192640,
        "points_memory_in_bytes": 102814280,
        "doc_values_memory_in_bytes": 2829496,
        "index_writer_memory_in_bytes": 12899088,
        "version_map_memory_in_bytes": 29403,
        "fixed_bit_set_memory_in_bytes": 0,
        "max_unsafe_auto_id_timestamp": -1,
        "file_sizes": {}
      },
      "translog": {
        "operations": 3766740,
        "size_in_bytes": 2307404347,
        "uncommitted_operations": 2893204,
        "uncommitted_size_in_bytes": 1782804035,
        "earliest_last_modified_age": 0
      },
      "request_cache": {
        "memory_size_in_bytes": 0,
        "evictions": 0,
        "hit_count": 0,
        "miss_count": 268
      },
      "recovery": {
        "current_as_source": 0,
        "current_as_target": 0,
        "throttle_time_in_millis": 17024187
      }
    },
    "total": {
      "docs": {
        "count": 3467397640,
        "deleted": 234593502
      },
      "store": {
        "size_in_bytes": 1087736351822
      },
      "indexing": {
        "index_total": 204398539,
        "index_time_in_millis": 80325395,
        "index_current": 0,
        "index_failed": 0,
        "delete_total": 0,
        "delete_time_in_millis": 0,
        "delete_current": 0,
        "noop_update_total": 0,
        "is_throttled": false,
        "throttle_time_in_millis": 0
      },
      "get": {
        "total": 0,
        "time_in_millis": 0,
        "exists_total": 0,
        "exists_time_in_millis": 0,
        "missing_total": 0,
        "missing_time_in_millis": 0,
        "current": 0
      },
      "search": {
        "open_contexts": 0,
        "query_total": 1524,
        "query_time_in_millis": 14514546,
        "query_current": 0,
        "fetch_total": 1018,
        "fetch_time_in_millis": 171319,
        "fetch_current": 0,
        "scroll_total": 0,
        "scroll_time_in_millis": 0,
        "scroll_current": 0,
        "suggest_total": 0,
        "suggest_time_in_millis": 0,
        "suggest_current": 0
      },
      "merges": {
        "current": 0,
        "current_docs": 0,
        "current_size_in_bytes": 0,
        "total": 192984,
        "total_time_in_millis": 87010422,
        "total_docs": 1248881022,
        "total_size_in_bytes": 394269971046,
        "total_stopped_time_in_millis": 0,
        "total_throttled_time_in_millis": 35698367,
        "total_auto_throttle_in_bytes": 20971520
      },
      "refresh": {
        "total": 377992,
        "total_time_in_millis": 38968248,
        "external_total": 377333,
        "external_total_time_in_millis": 40675901,
        "listeners": 0
      },
      "flush": {
        "total": 383,
        "periodic": 155,
        "total_time_in_millis": 695510
      },
      "warmer": {
        "current": 0,
        "total": 377329,
        "total_time_in_millis": 18259
      },
      "query_cache": {
        "memory_size_in_bytes": 0,
        "total_count": 0,
        "hit_count": 0,
        "miss_count": 0,
        "cache_size": 0,
        "cache_count": 0,
        "evictions": 0
      },
      "fielddata": {
        "memory_size_in_bytes": 17973802844,
        "evictions": 0
      },
      "completion": {
        "size_in_bytes": 0
      },
      "segments": {
        "count": 319,
        "memory_in_bytes": 1174162151,
        "terms_memory_in_bytes": 942165919,
        "stored_fields_memory_in_bytes": 126159816,
        "term_vectors_memory_in_bytes": 0,
        "norms_memory_in_bytes": 192640,
        "points_memory_in_bytes": 102814280,
        "doc_values_memory_in_bytes": 2829496,
        "index_writer_memory_in_bytes": 12899088,
        "version_map_memory_in_bytes": 29403,
        "fixed_bit_set_memory_in_bytes": 0,
        "max_unsafe_auto_id_timestamp": -1,
        "file_sizes": {}
      },
      "translog": {
        "operations": 3766740,
        "size_in_bytes": 2307404347,
        "uncommitted_operations": 2893204,
        "uncommitted_size_in_bytes": 1782804035,
        "earliest_last_modified_age": 0
      },
      "request_cache": {
        "memory_size_in_bytes": 0,
        "evictions": 0,
        "hit_count": 0,
        "miss_count": 268
      },
      "recovery": {
        "current_as_source": 0,
        "current_as_target": 0,
        "throttle_time_in_millis": 17024187
      }
    }
  }
}

Answer 1

推荐的分片大小是10-40GB，你大约有250GB。此外，看起来您有许多已删除的文档，这是因为您正在使用相同的 _id 覆盖文档。它们被标记为已删除，但它们仍在消耗 RAM 并参与操作 - More info

排序很慢，因为它不反转Lucene 中的倒排索引。选项是热切地预加载字段数据 - More info，您应该在 @timestamp 字段上使用 doc_value - More info。

另一种选择是，如果您总是按某个字段排序（例如按timestamp desc），您可以对索引进行排序并告诉ES 如何对磁盘上的文档进行排序。 Check this.

希望它会有所帮助。