【发布时间】：2019-12-20 14:42:18
【问题描述】：
MongoDB 未使用索引 - 可能存在排序问题？
我们有一个完全索引的 MongoDB 查询，它应该最多扫描 4/5 行。然而，查询似乎只使用索引的一个元素（整数）并忽略字符串部分。
我们使用的是不区分大小写的排序规则（强度=2），但我们指定与否都没有区别。文档：https://docs.mongodb.com/manual/core/index-case-insensitive/
排序规则使用索引吗？有没有更有效的方法来手动进行反规范化？注意：我们实际上并没有使用任何非标准字符，排序规则纯粹是为了不区分大小写而指定的。
版本（支持排序规则）：
MongoDB server version: 3.6.13
db.version() => 3.6.13
db.adminCommand( { getParameter: 1, featureCompatibilityVersion: 1 } )
gives:
{
"featureCompatibilityVersion" : {
"version" : "3.6"
},
"ok" : 1,
"operationTime" : Timestamp(1565754388, 51),
"$clusterTime" : {
"clusterTime" : Timestamp(1565754388, 51),
"signature" : {
"hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
"keyId" : NumberLong(0)
}
}
}
数据库结构（总结）是
{
"PrimaryID": (int)XXX,
"aTables": {
"userExt": [
{
"userExtPlatform": (int)XXX,
"userExtID": (string)XXX,
"userExtActive": (int 1 | 0)XXX,
},
{
"userExtPlatform": (int)XXX,
"userExtID": (string)XXX,
"userExtActive": (int 1 | 0)XXX,
},
...
],
"userOtherData": [
{
"otherDataField1": XXX,
"otherDataField2": XXX,
},
...
],
...
}
}
索引设置如下（注意 - 排序规则指定为 {locale:en, strength: 2}:
{
"v" : 2,
"key" : {
"aTables.userExt.userExtPlatform" : 1,
"aTables.userExt.userExtID" : 1
},
"name" : "extPlatform",
"background" : false,
"ns" : "archive.users",
"collation" : {
"locale" : "en",
"caseLevel" : false,
"caseFirst" : "off",
"strength" : 2,
"numericOrdering" : false,
"alternate" : "non-ignorable",
"maxVariable" : "punct",
"normalization" : false,
"backwards" : false,
"version" : "57.1"
}
}
查询（尝试有/没有指定排序规则）
use archive;
db.users.find(
{
"aTables.userExt.userExtPlatform": 4,
"aTables.userExt.userExtID": "AStringValue",
"aTables.userExt.userExtActive": 1,
"deleted": { "$exists": false }
}
)
db.users.find(
{
"aTables.userExt.userExtPlatform": 4,
"aTables.userExt.userExtID": "AStringValue",
"aTables.userExt.userExtActive": 1,
"deleted": { "$exists": false }
}
).collation( { locale: "en", strength: 2 } )
注意：删除“已删除”子句对速度/结果/解释没有影响。
这是解释，它显示了查询的大量键和文档。
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "archive.users",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"aTables.userExt.userExtActive" : {
"$eq" : 1
}
},
{
"aTables.userExt.userExtID" : {
"$eq" : "PrivateStringRemoved"
}
},
{
"aTables.userExt.userExtPlatform" : {
"$eq" : 4
}
},
{
"$nor" : [
{
"deleted" : {
"$exists" : true
}
}
]
}
]
},
"collation" : {
"locale" : "en",
"caseLevel" : false,
"caseFirst" : "off",
"strength" : 2,
"numericOrdering" : false,
"alternate" : "non-ignorable",
"maxVariable" : "punct",
"normalization" : false,
"backwards" : false,
"version" : "57.1"
},
"winningPlan" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"aTables.userExt.userExtActive" : {
"$eq" : 1
}
},
{
"aTables.userExt.userExtID" : {
"$eq" : "PrivateStringRemoved"
}
},
{
"$nor" : [
{
"deleted" : {
"$exists" : true
}
}
]
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"aTables.userExt.userExtPlatform" : 1,
"aTables.userExt.userExtID" : 1
},
"indexName" : "extPlatform",
"collation" : {
"locale" : "en",
"caseLevel" : false,
"caseFirst" : "off",
"strength" : 2,
"numericOrdering" : false,
"alternate" : "non-ignorable",
"maxVariable" : "punct",
"normalization" : false,
"backwards" : false,
"version" : "57.1"
},
"isMultiKey" : true,
"multiKeyPaths" : {
"aTables.userExt.userExtPlatform" : [
"aTables.userExt"
],
"aTables.userExt.userExtID" : [
"aTables.userExt"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"aTables.userExt.userExtPlatform" : [
"[4.0, 4.0]"
],
"aTables.userExt.userExtID" : [
"[MinKey, MaxKey]"
]
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 0,
"executionTimeMillis" : 1304,
"totalKeysExamined" : 290114,
"totalDocsExamined" : 290114,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"aTables.userExt.userExtActive" : {
"$eq" : 1
}
},
{
"aTables.userExt.userExtID" : {
"$eq" : "PrivateStringRemoved"
}
},
{
"$nor" : [
{
"deleted" : {
"$exists" : true
}
}
]
}
]
},
"nReturned" : 0,
"executionTimeMillisEstimate" : 1245,
"works" : 290115,
"advanced" : 0,
"needTime" : 290114,
"needYield" : 0,
"saveState" : 2267,
"restoreState" : 2267,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 290114,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 290114,
"executionTimeMillisEstimate" : 270,
"works" : 290115,
"advanced" : 290114,
"needTime" : 0,
"needYield" : 0,
"saveState" : 2267,
"restoreState" : 2267,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"aTables.userExt.userExtPlatform" : 1,
"aTables.userExt.userExtID" : 1
},
"indexName" : "extPlatform",
"collation" : {
"locale" : "en",
"caseLevel" : false,
"caseFirst" : "off",
"strength" : 2,
"numericOrdering" : false,
"alternate" : "non-ignorable",
"maxVariable" : "punct",
"normalization" : false,
"backwards" : false,
"version" : "57.1"
},
"isMultiKey" : true,
"multiKeyPaths" : {
"aTables.userExt.userExtPlatform" : [
"aTables.userExt"
],
"aTables.userExt.userExtID" : [
"aTables.userExt"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"aTables.userExt.userExtPlatform" : [
"[4.0, 4.0]"
],
"aTables.userExt.userExtID" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 290114,
"seeks" : 1,
"dupsTested" : 290114,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
"serverInfo" : {
"host" : "api-mdb-archive-03",
"port" : 27017,
"version" : "3.6.13",
"gitVersion" : "db3c76679b7a3d9b443a0e1b3e45ed02b88c539f"
},
"ok" : 1,
"operationTime" : Timestamp(1565753056, 9),
"$clusterTime" : {
"clusterTime" : Timestamp(1565753056, 9),
"signature" : {
"hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
"keyId" : NumberLong(0)
}
}
}
日志输出还确认它正在检查大量文档，并且排序通过。
2019-08-14T03:23:53.912+0000 I COMMAND [conn20679] command archive.users appName: "MongoDB Shell" command: find { find: "users", filter: { aTables.userExt.userExtPlatform: 4.0, aTables.userExt.userExtID: "PrivateStringRemoved", aTables.userExt.userExtActive: 1.0, deleted: { $exists: false } }, collation: { locale: "en", strength: 2.0 }, lsid: { id: UUID("3178aa31-5ee9-4a79-9848-f01c1842f542") }, $clusterTime: { clusterTime: Timestamp(1565753015, 41), signature: { hash: BinData(0, 0000000000000000000000000000000000000000), keyId: 0 } }, $db: "archive" } planSummary: IXSCAN { aTables.userExt.userExtPlatform: 1, aTables.userExt.userExtID: 1 } keysExamined:289966 docsExamined:289966 cursorExhausted:1 numYields:2267 nreturned:0 reslen:228 locks:{ Global: { acquireCount: { r: 4536 } }, Database: { acquireCount: { r: 2268 } }, Collection: { acquireCount: { r: 2268 } } } protocol:op_msg 1546ms
2019-08-14T03:24:16.864+0000 I COMMAND [conn20679] command archive.users appName: "MongoDB Shell" command: explain { explain: { find: "users", filter: { aTables.userExt.userExtPlatform: 4.0, aTables.userExt.userExtID: "PrivateStringRemoved", aTables.userExt.userExtActive: 1.0, deleted: { $exists: false } }, collation: { locale: "en", strength: 2.0 } }, verbosity: "executionStats", lsid: { id: UUID("3178aa31-5ee9-4a79-9848-f01c1842f542") }, $clusterTime: { clusterTime: Timestamp(1565753033, 128), signature: { hash: BinData(0, 0000000000000000000000000000000000000000), keyId: 0 } }, $db: "archive" } numYields:2267 reslen:3578 locks:{ Global: { acquireCount: { r: 4536 } }, Database: { acquireCount: { r: 2268 } }, Collection: { acquireCount: { r: 2268 } } } protocol:op_msg 1341ms
为了完整起见，这是由 PHP 驱动的，但如上所述在 Mongo CMD 中进行了验证。这是PHP：
$aParams = [
'aTables.userExt.userExtID' => 4,
'aTables.userExt.userExtPlatform' => 'PrivateStringRemoved',
'aTables.userExt.userExtActive' => 1,
'deleted': [
'$exists' => false
]
];
$aOptions = [
'readPreference' => new \MongoDB\Driver\ReadPreference(\MongoDB\Driver\ReadPreference::RP_NEAREST),
'skip' => $start,
'limit' => $limit,
'typeMap' => [
'root' => 'array',
'document' => 'array',
'array' => 'array'
],
'collation' => [
'locale' => 'en',
'strength' => 2
],
];
try {
$aResults = $collectionArchive->find($aParams, $aOptions);
} catch (\Exception $exception) {
throw new ArchiverException('Mongo Error', ArchiverRequest::ERROR_MONGO, $exception->getMessage());
}
【问题讨论】：
-
你能用排序规则 1 试试这个吗？可能不同的字符类型没有得到匹配。另外，表的总大小是多少？
-
实际的字符串类似于“g12345678901234567890”（字母g，后跟数字）。所以我们不应该需要 1 级比赛。事实上，对于这个指标，我们甚至不需要2级，但我们需要其他的。目前表的大小约为 2/3 百万行 - 我中止填充它，直到问题得到解决，但它会达到 20+百万 - 这对于扫描来说非常慢。
-
explain 仅显示 2268 行，这意味着由于索引而发生了一些过滤
-
是的，没错。它似乎在“userExtPlatform”上进行过滤，但随后不使用索引的 userExtID 部分 - 因此在
[MinKey,MaxKey]上使用 IDXScan。一些谷歌搜索表明我们需要“$elemMatch”，但我不知道为什么上述方法不起作用。