MongoDB - 对范围查询进行排序和限制时未使用索引答案

【问题标题】：MongoDB - Index not being used when sorting and limiting on ranged queryMongoDB - 对范围查询进行排序和限制时未使用索引
【发布时间】：2018-02-12 04:36:28
【问题描述】：

我正在尝试对包含公告板数据的集合使用范围查询来获取排序的项目列表。 “线程”文档的数据结构是：

{
    "_id" : ObjectId("5a779b47f4fa72412126526a"),
    "title" : "necessitatibus tincidunt libris assueverit",
    "content" : "Corrumpitvenenatis cubilia adipiscing sollicitudin",
    "flagged" : false,
    "locked" : false,
    "sticky" : false,
    "lastPostAt" : ISODate("2018-02-05T06:35:24.656Z"),
    "postCount" : 42,
    "user" : ObjectId("5a779b46f4fa72412126525a"),
    "category" : ObjectId("5a779b31f4fa724121265164"),
    "createdAt" : ISODate("2018-02-04T23:46:15.852Z"),
    "updatedAt" : ISODate("2018-02-05T06:35:24.656Z")
}

查询是：

db.threads.find({
    category: ObjectId('5a779b31f4fa724121265142'), 
    _id : { $gt: ObjectId('5a779b5cf4fa724121269be8') }
}).sort({ sticky: -1, lastPostAt: -1, _id: 1 }).limit(25)

我设置了以下索引来支持它：

{ category: 1, _id: 1 }
{ category: 1, _id: 1, sticky: 1, lastPostAt: 1 }
{ sticky: 1, lastPostAt: 1, _id: 1 }

尽管如此，它仍然根据执行统计扫描数百个文档/键：

{
    "executionStats" : {
    "executionSuccess" : true,
    "nReturned" : 772,
    "executionTimeMillis" : 17,
    "totalKeysExamined" : 772,
    "totalDocsExamined" : 772,
    "executionStages" : {
        "stage" : "SORT",
        "nReturned" : 772,
        "executionTimeMillisEstimate" : 0,
        "works" : 1547,
        "advanced" : 772,
        "needTime" : 774,
        "needYield" : 0,
        "saveState" : 33,
        "restoreState" : 33,
        "isEOF" : 1,
        "invalidates" : 0,
        "sortPattern" : {
            "sticky" : -1,
            "lastPostAt" : -1,
            "_id" : 1
        },
        "memUsage" : 1482601,
        "memLimit" : 33554432,
        "inputStage" : {
            "stage" : "SORT_KEY_GENERATOR",
            "nReturned" : 772,
            "executionTimeMillisEstimate" : 0,
            "works" : 774,
            "advanced" : 772,
            "needTime" : 1,
            "needYield" : 0,
            "saveState" : 33,
            "restoreState" : 33,
            "isEOF" : 1,
            "invalidates" : 0,
            "inputStage" : {
                "stage" : "FETCH",
                "nReturned" : 772,
                "executionTimeMillisEstimate" : 0,
                "works" : 773,
                "advanced" : 772,
                "needTime" : 0,
                "needYield" : 0,
                "saveState" : 33,
                "restoreState" : 33,
                "isEOF" : 1,
                "invalidates" : 0,
                "docsExamined" : 772,
                "alreadyHasObj" : 0,
                "inputStage" : {
                    "stage" : "IXSCAN",
                    "nReturned" : 772,
                    "executionTimeMillisEstimate" : 0,
                    "works" : 773,
                    "advanced" : 772,
                    "needTime" : 0,
                    "needYield" : 0,
                    "saveState" : 33,
                    "restoreState" : 33,
                    "isEOF" : 1,
                    "invalidates" : 0,
                    "keyPattern" : {
                        "category" : 1,
                        "_id" : 1,
                        "sticky" : 1,
                        "lastPostAt" : 1
                    },
                    "indexName" : "category_1__id_1_sticky_1_lastPostAt_1",
                    "isMultiKey" : false,
                    "multiKeyPaths" : {
                        "category" : [ ],
                        "_id" : [ ],
                        "sticky" : [ ],
                        "lastPostAt" : [ ]
                    },
                    "isUnique" : false,
                    "isSparse" : false,
                    "isPartial" : false,
                    "indexVersion" : 2,
                    "direction" : "forward",
                    "indexBounds" : {
                        "category" : [
                            "[ObjectId('5a779b31f4fa724121265142'), ObjectId('5a779b31f4fa724121265142')]"
                        ],
                        "_id" : [
                            "(ObjectId('5a779b5cf4fa724121269be8'), ObjectId('ffffffffffffffffffffffff')]"
                        ],
                        "sticky" : [
                            "[MinKey, MaxKey]"
                        ],
                        "lastPostAt" : [
                            "[MinKey, MaxKey]"
                        ]
                    },
                    "keysExamined" : 772,
                    "seeks" : 1,
                    "dupsTested" : 0,
                    "dupsDropped" : 0,
                    "seenInvalidated" : 0
                }
            }
        }
    }
}

当我取出分拣台时，它只正确扫描了 25 个文档。无论我在排序函数中放置哪些字段，检查的键 (772) 都保持不变。

这是已排序查询的完整 explain()：

{
    "queryPlanner" : {
        "plannerVersion" : 1,
        "namespace" : "database.threads",
        "indexFilterSet" : false,
        "parsedQuery" : {
            "$and" : [
                {
                    "category" : {
                        "$eq" : ObjectId("5a779b31f4fa724121265142")
                    }
                },
                {
                    "_id" : {
                        "$gt" : ObjectId("5a779b5cf4fa724121269be8")
                    }
                }
            ]
        },
        "winningPlan" : {
            "stage" : "SORT",
            "sortPattern" : {
                "sticky" : -1,
                "lastPostAt" : -1,
                "_id" : 1
            },
            "limitAmount" : 25,
            "inputStage" : {
                "stage" : "SORT_KEY_GENERATOR",
                "inputStage" : {
                    "stage" : "FETCH",
                    "inputStage" : {
                        "stage" : "IXSCAN",
                        "keyPattern" : {
                            "category" : 1,
                            "_id" : 1,
                            "sticky" : 1,
                            "lastPostAt" : 1
                        },
                        "indexName" : "category_1__id_1_sticky_1_lastPostAt_1",
                        "isMultiKey" : false,
                        "multiKeyPaths" : {
                            "category" : [ ],
                            "_id" : [ ],
                            "sticky" : [ ],
                            "lastPostAt" : [ ]
                        },
                        "isUnique" : false,
                        "isSparse" : false,
                        "isPartial" : false,
                        "indexVersion" : 2,
                        "direction" : "forward",
                        "indexBounds" : {
                            "category" : [
                                "[ObjectId('5a779b31f4fa724121265142'), ObjectId('5a779b31f4fa724121265142')]"
                            ],
                            "_id" : [
                                "(ObjectId('5a779b5cf4fa724121269be8'), ObjectId('ffffffffffffffffffffffff')]"
                            ],
                            "sticky" : [
                                "[MinKey, MaxKey]"
                            ],
                            "lastPostAt" : [
                                "[MinKey, MaxKey]"
                            ]
                        }
                    }
                }
            }
        },
        "rejectedPlans" : [
            {
                "stage" : "SORT",
                "sortPattern" : {
                    "sticky" : -1,
                    "lastPostAt" : -1,
                    "_id" : 1
                },
                "limitAmount" : 25,
                "inputStage" : {
                    "stage" : "SORT_KEY_GENERATOR",
                    "inputStage" : {
                        "stage" : "FETCH",
                        "filter" : {
                            "_id" : {
                                "$gt" : ObjectId("5a779b5cf4fa724121269be8")
                            }
                        },
                        "inputStage" : {
                            "stage" : "IXSCAN",
                            "keyPattern" : {
                                "category" : 1
                            },
                            "indexName" : "category_1",
                            "isMultiKey" : false,
                            "multiKeyPaths" : {
                                "category" : [ ]
                            },
                            "isUnique" : false,
                            "isSparse" : false,
                            "isPartial" : false,
                            "indexVersion" : 2,
                            "direction" : "forward",
                            "indexBounds" : {
                                "category" : [
                                    "[ObjectId('5a779b31f4fa724121265142'), ObjectId('5a779b31f4fa724121265142')]"
                                ]
                            }
                        }
                    }
                }
            },
            {
                "stage" : "SORT",
                "sortPattern" : {
                    "sticky" : -1,
                    "lastPostAt" : -1,
                    "_id" : 1
                },
                "limitAmount" : 25,
                "inputStage" : {
                    "stage" : "SORT_KEY_GENERATOR",
                    "inputStage" : {
                        "stage" : "FETCH",
                        "inputStage" : {
                            "stage" : "IXSCAN",
                            "keyPattern" : {
                                "category" : 1,
                                "_id" : 1
                            },
                            "indexName" : "category_1__id_1",
                            "isMultiKey" : false,
                            "multiKeyPaths" : {
                                "category" : [ ],
                                "_id" : [ ]
                            },
                            "isUnique" : false,
                            "isSparse" : false,
                            "isPartial" : false,
                            "indexVersion" : 2,
                            "direction" : "forward",
                            "indexBounds" : {
                                "category" : [
                                    "[ObjectId('5a779b31f4fa724121265142'), ObjectId('5a779b31f4fa724121265142')]"
                                ],
                                "_id" : [
                                    "(ObjectId('5a779b5cf4fa724121269be8'), ObjectId('ffffffffffffffffffffffff')]"
                                ]
                            }
                        }
                    }
                }
            },
            {
                "stage" : "SORT",
                "sortPattern" : {
                    "sticky" : -1,
                    "lastPostAt" : -1,
                    "_id" : 1
                },
                "limitAmount" : 25,
                "inputStage" : {
                    "stage" : "SORT_KEY_GENERATOR",
                    "inputStage" : {
                        "stage" : "FETCH",
                        "filter" : {
                            "category" : {
                                "$eq" : ObjectId("5a779b31f4fa724121265142")
                            }
                        },
                        "inputStage" : {
                            "stage" : "IXSCAN",
                            "keyPattern" : {
                                "_id" : 1
                            },
                            "indexName" : "_id_",
                            "isMultiKey" : false,
                            "multiKeyPaths" : {
                                "_id" : [ ]
                            },
                            "isUnique" : true,
                            "isSparse" : false,
                            "isPartial" : false,
                            "indexVersion" : 2,
                            "direction" : "forward",
                            "indexBounds" : {
                                "_id" : [
                                    "(ObjectId('5a779b5cf4fa724121269be8'), ObjectId('ffffffffffffffffffffffff')]"
                                ]
                            }
                        }
                    }
                }
            }
        ]
    },
    "serverInfo" : {
        "host" : "CRF-MBP.local",
        "port" : 27017,
        "version" : "3.6.2",
        "gitVersion" : "489d177dbd0f0420a8ca04d39fd78d0a2c539420"
    },
    "ok" : 1
}

这里是未排序查询的完整 explain()：

{
    "queryPlanner" : {
        "plannerVersion" : 1,
        "namespace" : "database.threads",
        "indexFilterSet" : false,
        "parsedQuery" : {
            "$and" : [
                {
                    "category" : {
                        "$eq" : ObjectId("5a779b31f4fa724121265142")
                    }
                },
                {
                    "_id" : {
                        "$gt" : ObjectId("5a779b5cf4fa724121269be8")
                    }
                }
            ]
        },
        "winningPlan" : {
            "stage" : "LIMIT",
            "limitAmount" : 25,
            "inputStage" : {
                "stage" : "FETCH",
                "inputStage" : {
                    "stage" : "IXSCAN",
                    "keyPattern" : {
                        "category" : 1,
                        "_id" : 1,
                        "sticky" : 1,
                        "lastPostAt" : 1
                    },
                    "indexName" : "category_1__id_1_sticky_1_lastPostAt_1",
                    "isMultiKey" : false,
                    "multiKeyPaths" : {
                        "category" : [ ],
                        "_id" : [ ],
                        "sticky" : [ ],
                        "lastPostAt" : [ ]
                    },
                    "isUnique" : false,
                    "isSparse" : false,
                    "isPartial" : false,
                    "indexVersion" : 2,
                    "direction" : "forward",
                    "indexBounds" : {
                        "category" : [
                            "[ObjectId('5a779b31f4fa724121265142'), ObjectId('5a779b31f4fa724121265142')]"
                        ],
                        "_id" : [
                            "(ObjectId('5a779b5cf4fa724121269be8'), ObjectId('ffffffffffffffffffffffff')]"
                        ],
                        "sticky" : [
                            "[MinKey, MaxKey]"
                        ],
                        "lastPostAt" : [
                            "[MinKey, MaxKey]"
                        ]
                    }
                }
            }
        },
        "rejectedPlans" : [
            {
                "stage" : "LIMIT",
                "limitAmount" : 25,
                "inputStage" : {
                    "stage" : "FETCH",
                    "filter" : {
                        "_id" : {
                            "$gt" : ObjectId("5a779b5cf4fa724121269be8")
                        }
                    },
                    "inputStage" : {
                        "stage" : "IXSCAN",
                        "keyPattern" : {
                            "category" : 1
                        },
                        "indexName" : "category_1",
                        "isMultiKey" : false,
                        "multiKeyPaths" : {
                            "category" : [ ]
                        },
                        "isUnique" : false,
                        "isSparse" : false,
                        "isPartial" : false,
                        "indexVersion" : 2,
                        "direction" : "forward",
                        "indexBounds" : {
                            "category" : [
                                "[ObjectId('5a779b31f4fa724121265142'), ObjectId('5a779b31f4fa724121265142')]"
                            ]
                        }
                    }
                }
            },
            {
                "stage" : "LIMIT",
                "limitAmount" : 25,
                "inputStage" : {
                    "stage" : "FETCH",
                    "inputStage" : {
                        "stage" : "IXSCAN",
                        "keyPattern" : {
                            "category" : 1,
                            "_id" : 1
                        },
                        "indexName" : "category_1__id_1",
                        "isMultiKey" : false,
                        "multiKeyPaths" : {
                            "category" : [ ],
                            "_id" : [ ]
                        },
                        "isUnique" : false,
                        "isSparse" : false,
                        "isPartial" : false,
                        "indexVersion" : 2,
                        "direction" : "forward",
                        "indexBounds" : {
                            "category" : [
                                "[ObjectId('5a779b31f4fa724121265142'), ObjectId('5a779b31f4fa724121265142')]"
                            ],
                            "_id" : [
                                "(ObjectId('5a779b5cf4fa724121269be8'), ObjectId('ffffffffffffffffffffffff')]"
                            ]
                        }
                    }
                }
            },
            {
                "stage" : "LIMIT",
                "limitAmount" : 25,
                "inputStage" : {
                    "stage" : "FETCH",
                    "filter" : {
                        "category" : {
                            "$eq" : ObjectId("5a779b31f4fa724121265142")
                        }
                    },
                    "inputStage" : {
                        "stage" : "IXSCAN",
                        "keyPattern" : {
                            "_id" : 1
                        },
                        "indexName" : "_id_",
                        "isMultiKey" : false,
                        "multiKeyPaths" : {
                            "_id" : [ ]
                        },
                        "isUnique" : true,
                        "isSparse" : false,
                        "isPartial" : false,
                        "indexVersion" : 2,
                        "direction" : "forward",
                        "indexBounds" : {
                            "_id" : [
                                "(ObjectId('5a779b5cf4fa724121269be8'), ObjectId('ffffffffffffffffffffffff')]"
                            ]
                        }
                    }
                }
            }
        ]
    },
    "serverInfo" : {
        "host" : "CRF-MBP.local",
        "port" : 27017,
        "version" : "3.6.2",
        "gitVersion" : "489d177dbd0f0420a8ca04d39fd78d0a2c539420"
    },
    "ok" : 1
}

有谁知道为什么这可能没有完全使用索引？

【问题讨论】：

您能否发布已排序和未排序查询的explain() 输出？
此外，查询似乎返回了 772 个文档（来自 nReturned 编号）。这是您期望的正确文件数量吗？不排序的查询是否只返回25个文档？
不，我只是想检索前 25 个文档。没有排序的 queyr 确实只返回 25 个文档。我已经用更多信息更新了我的帖子。感谢您的帮助！
嗯，它正在正常工作。您在限制之前排序，因此它对整个集合进行排序，然后从排序结果中选择 25。
无论我在限制之前还是之后排序（我都尝试过），它总是扫描772个文档。我只是好奇是否有办法至少减少这个数字。

标签： mongodb mongodb-query morphia

【解决方案1】：

问题是您的所有索引实际上都没有帮助排序查询。这就是扫描对象数量多且存在SORT_KEY_GENERATOR 阶段（内存中排序，限制为32MB）的原因。

另一方面，非排序查询可以使用{ category: 1, _id: 1 } 或{ category: 1, _id: 1, sticky: 1, lastPostAt: 1 } 索引。请注意，使用其中任何一个都是完全有效的，因为其中一个包含另一个的 prefix。详情请见Prefixes。

MongoDB find() 查询通常只使用一个索引，因此单个 compound index 应该满足查询的所有参数。这将包括find() 和sort() 的参数。

Optimizing MongoDB Compound Indexes 中提供了有关如何创建索引的好文章。先来看看文章的重点，这里的复合索引排序应该是equality --> sort --> range：

您的查询“形状”是：

db.collection.find({category:..., _id: {$gt:...}})
             .sort({sticky:-1, lastPostAt:-1, _id:1})
             .limit(25)

我们看到：

category:... 是平等
sticky:-1, lastPostAt:-1, _id:1 是排序
_id: {$gt:...} 是范围

所以你需要的复合索引是：

{category:1, sticky:-1, lastPostAt:-1, _id:1}

上面索引的查询的explain() 输出的获胜计划在哪里显示：

"winningPlan": {
      "stage": "LIMIT",
      "limitAmount": 25,
      "inputStage": {
        "stage": "FETCH",
        "inputStage": {
          "stage": "IXSCAN",
          "keyPattern": {
            "category": 1,
            "sticky": -1,
            "lastPostAt": -1,
            "_id": 1
          },
          "indexName": "category_1_sticky_-1_lastPostAt_-1__id_1",
          "isMultiKey": false,
          "multiKeyPaths": {
            "category": [ ],
            "sticky": [ ],
            "lastPostAt": [ ],
            "_id": [ ]
          },
          "isUnique": false,
          "isSparse": false,
          "isPartial": false,
          "indexVersion": 2,
          "direction": "forward",
          "indexBounds": {
            "category": [
              "[ObjectId('5a779b31f4fa724121265142'), ObjectId('5a779b31f4fa724121265142')]"
            ],
            "sticky": [
              "[MaxKey, MinKey]"
            ],
            "lastPostAt": [
              "[MaxKey, MinKey]"
            ],
            "_id": [
              "(ObjectId('5a779b5cf4fa724121269be8'), ObjectId('ffffffffffffffffffffffff')]"
            ]
          }
        }
      }
    }

请注意，获胜计划不包含SORT_KEY_GENERATOR 阶段。这意味着可以充分利用索引来响应排序查询。

【讨论】：

【解决方案2】：

我相信有两个问题都与您的排序有关。这些问题直接来自documentations，但如果您愿意发表评论，我会帮助解释（并且可能自己学到一些东西）

第一个也是最大的问题是你必须按照索引给出的顺序进行排序。来自文档：

您可以对索引的所有键或子集指定排序；但是，排序键必须按照它们出现的顺序列出在索引中。例如，索引键模式 { a: 1, b: 1 } 可以支持对 { a: 1, b: 1 } 排序，但不支持对 { b: 1, a: 1 } 排序。

这意味着您必须按照获胜计划给出的顺序进行排序：category, _id, sticky, lastPostAt（或该顺序的任何前缀，例如 category, _id, sticky 或 category _id）。如果不是，mongodb 将识别使用您的获胜计划索引的 772 个文档，但随后必须梳理每个键以评估值并提供所需的排序顺序。如果要按当前查询的顺序排序，则必须按该顺序提供索引：

第二个问题是必须按照索引提供的方向（或反方向）进行排序。

对于使用复合索引进行排序的查询，指定的排序 cursor.sort() 文档中所有键的方向必须与索引键模式或匹配索引键模式的反转。为了例如，索引键模式 { a: 1, b: -1 } 可以支持对 { a: 1, b: -1 } 和 { a: -1, b: 1 } 但不在 { a: -1, b: -1 } 或 {a: 1, b: 1}。

因为您的索引都是按升序排列的，所以您必须对所有索引按升序排序，或者对所有索引按降序排序。如果不是，我们会遇到同样的问题，mongo 会找到所有相关文档，但必须梳理它们以提供所需的顺序。

我相信您可以通过提供以下索引来获得改进的功能：

{ sticky: -1, lastPostAt: -1, _id: 1 }

或它的倒数：

{ sticky: 1, lastPostAt: 1, _id: -1 }

这会造成mongo使用你的第一个索引的情况

{ category: 1, _id: 1 }

为了识别潜在的未排序文档，然后使用新索引之一（上面提供），因为它们已经排序。然后，该限制将负责为您提供 25 个文档。

我很确定这会创建一个覆盖查询（一个没有检查文档的查询）。让我知道进展如何，干杯！

【讨论】：