了解 explain() 以优化复杂查询

【问题标题】：Understanding explain() to optimize complex queries了解 explain() 以优化复杂查询
【发布时间】：2018-08-14 13:33:14
【问题描述】：

我有一个聚合管道需要 5 秒以上才能返回 200 行。 我正在尝试使用 explain("executionStats") 优化我当前的管道。

这是我的管道：

db.getCollection("content_topics").explain("executionStats").aggregate([{
    "$lookup": {
      from: "users",
      localField: "creator",
      foreignField: "_id",
      as: "user"
    }
  },
  {
    $unwind: "$user"
  },
  {
    "$match": {
      created_at: {
        "$gte": 1528914600,
        "$lte": 1534271400
      },
      dash_status: 3,
      language: "hi",
      parent_topic_id: {
        "$eq": null
      },
      status: 1,
      "user.device_os": {
        "$ne": "BOT"
      }
    }
  },
  {
    "$sort": {
      created_at: -1
    }
  },
  {
    "$addFields": {
      user_handle: "$user.handle",
      user_phone: "$user.phone",
      user_status: "$user.status"
    }
  },
  {
    "$project": {
      topic_id: 1,
      n_vokes: 1,
      message: 1,
      title: 1,
      language: 1,
      description: 1,
      voice_desc: 1,
      image: 1,
      image_share: 1,
      hashtag: 1,
      location: 1,
      default_text: 1,
      creator: 1,
      created_at: 1,
      status: 1,
      ref_id: 1,
      weightage: 1,
      username: 1,
      slug_generated: 1,
      user_handle: 1,
      user_phone: 1,
      user_status: 1
    }
  },
  {
    "$skip": 0
  },
  {
    "$limit": 200
  }
]);

这就是 mongo explain()ed:

{ 
    "stages" : [
        {
            "$cursor" : {
                "query" : {
                    "$and" : [
                        {
                            "created_at" : {
                                "$gte" : 1528914600.0
                            }
                        }, 
                        {
                            "created_at" : {
                                "$lte" : 1534271400.0
                            }
                        }, 
                        {
                            "dash_status" : {
                                "$eq" : 3.0
                            }
                        }, 
                        {
                            "language" : {
                                "$eq" : "hi"
                            }
                        }, 
                        {
                            "parent_topic_id" : {
                                "$eq" : null
                            }
                        }, 
                        {
                            "status" : {
                                "$eq" : 1.0
                            }
                        }
                    ]
                }, 
                "fields" : {
                    "created_at" : 1.0, 
                    "creator" : 1.0, 
                    "default_text" : 1.0, 
                    "description" : 1.0, 
                    "hashtag" : 1.0, 
                    "image" : 1.0, 
                    "image_share" : 1.0, 
                    "language" : 1.0, 
                    "location" : 1.0, 
                    "message" : 1.0, 
                    "n_vokes" : 1.0, 
                    "ref_id" : 1.0, 
                    "slug_generated" : 1.0, 
                    "status" : 1.0, 
                    "title" : 1.0, 
                    "topic_id" : 1.0, 
                    "user.handle" : 1.0, 
                    "user.phone" : 1.0, 
                    "user.status" : 1.0, 
                    "user_handle" : 1.0, 
                    "user_phone" : 1.0, 
                    "user_status" : 1.0, 
                    "username" : 1.0, 
                    "voice_desc" : 1.0, 
                    "weightage" : 1.0, 
                    "_id" : 1.0
                }, 
                "queryPlanner" : {
                    "plannerVersion" : 1.0, 
                    "namespace" : "vokalapp.content_topics", 
                    "indexFilterSet" : false, 
                    "parsedQuery" : {
                        "$and" : [
                            {
                                "dash_status" : {
                                    "$eq" : 3.0
                                }
                            }, 
                            {
                                "language" : {
                                    "$eq" : "hi"
                                }
                            }, 
                            {
                                "parent_topic_id" : {
                                    "$eq" : null
                                }
                            }, 
                            {
                                "status" : {
                                    "$eq" : 1.0
                                }
                            }, 
                            {
                                "created_at" : {
                                    "$lte" : 1534271400.0
                                }
                            }, 
                            {
                                "created_at" : {
                                    "$gte" : 1528914600.0
                                }
                            }
                        ]
                    }, 
                    "winningPlan" : {
                        "stage" : "FETCH", 
                        "filter" : {
                            "parent_topic_id" : {
                                "$eq" : null
                            }
                        }, 
                        "inputStage" : {
                            "stage" : "IXSCAN", 
                            "keyPattern" : {
                                "dash_status" : 1.0, 
                                "status" : 1.0, 
                                "language" : 1.0, 
                                "parent_topic_id" : 1.0, 
                                "created_at" : 1.0
                            }, 
                            "indexName" : "index_for_dashboard", 
                            "isMultiKey" : false, 
                            "multiKeyPaths" : {
                                "dash_status" : [

                                ], 
                                "status" : [

                                ], 
                                "language" : [

                                ], 
                                "parent_topic_id" : [

                                ], 
                                "created_at" : [

                                ]
                            }, 
                            "isUnique" : false, 
                            "isSparse" : false, 
                            "isPartial" : false, 
                            "indexVersion" : 2.0, 
                            "direction" : "forward", 
                            "indexBounds" : {
                                "dash_status" : [
                                    "[3.0, 3.0]"
                                ], 
                                "status" : [
                                    "[1.0, 1.0]"
                                ], 
                                "language" : [
                                    "[\"hi\", \"hi\"]"
                                ], 
                                "parent_topic_id" : [
                                    "[null, null]"
                                ], 
                                "created_at" : [
                                    "[1528914600.0, 1534271400.0]"
                                ]
                            }
                        }
                    }, 
                    "rejectedPlans" : [
                        {
                            "stage" : "FETCH", 
                            "filter" : {
                                "$and" : [
                                    {
                                        "dash_status" : {
                                            "$eq" : 3.0
                                        }
                                    }, 
                                    {
                                        "language" : {
                                            "$eq" : "hi"
                                        }
                                    }, 
                                    {
                                        "parent_topic_id" : {
                                            "$eq" : null
                                        }
                                    }, 
                                    {
                                        "status" : {
                                            "$eq" : 1.0
                                        }
                                    }
                                ]
                            }, 
                            "inputStage" : {
                                "stage" : "IXSCAN", 
                                "keyPattern" : {
                                    "created_at" : 1.0
                                }, 
                                "indexName" : "created_at_index", 
                                "isMultiKey" : false, 
                                "multiKeyPaths" : {
                                    "created_at" : [

                                    ]
                                }, 
                                "isUnique" : false, 
                                "isSparse" : false, 
                                "isPartial" : false, 
                                "indexVersion" : 2.0, 
                                "direction" : "forward", 
                                "indexBounds" : {
                                    "created_at" : [
                                        "[1528914600.0, 1534271400.0]"
                                    ]
                                }
                            }
                        }, 
                        {
                            "stage" : "FETCH", 
                            "filter" : {
                                "$and" : [
                                    {
                                        "dash_status" : {
                                            "$eq" : 3.0
                                        }
                                    }, 
                                    {
                                        "parent_topic_id" : {
                                            "$eq" : null
                                        }
                                    }
                                ]
                            }, 
                            "inputStage" : {
                                "stage" : "IXSCAN", 
                                "keyPattern" : {
                                    "language" : 1.0, 
                                    "status" : 1.0, 
                                    "created_at" : 1.0, 
                                    "answers.type" : 1.0
                                }, 
                                "indexName" : "language_status_created_at_answer_type_index", 
                                "isMultiKey" : true, 
                                "multiKeyPaths" : {
                                    "language" : [

                                    ], 
                                    "status" : [

                                    ], 
                                    "created_at" : [

                                    ], 
                                    "answers.type" : [
                                        "answers"
                                    ]
                                }, 
                                "isUnique" : false, 
                                "isSparse" : false, 
                                "isPartial" : false, 
                                "indexVersion" : 2.0, 
                                "direction" : "forward", 
                                "indexBounds" : {
                                    "language" : [
                                        "[\"hi\", \"hi\"]"
                                    ], 
                                    "status" : [
                                        "[1.0, 1.0]"
                                    ], 
                                    "created_at" : [
                                        "[1528914600.0, 1534271400.0]"
                                    ], 
                                    "answers.type" : [
                                        "[MinKey, MaxKey]"
                                    ]
                                }
                            }
                        }, 
                        {
                            "stage" : "FETCH", 
                            "filter" : {
                                "$and" : [
                                    {
                                        "dash_status" : {
                                            "$eq" : 3.0
                                        }
                                    }, 
                                    {
                                        "parent_topic_id" : {
                                            "$eq" : null
                                        }
                                    }
                                ]
                            }, 
                            "inputStage" : {
                                "stage" : "IXSCAN", 
                                "keyPattern" : {
                                    "language" : -1.0, 
                                    "status" : -1.0, 
                                    "weightage" : -1.0, 
                                    "created_at" : -1.0
                                }, 
                                "indexName" : "language_status_weightage_created_at_index", 
                                "isMultiKey" : false, 
                                "multiKeyPaths" : {
                                    "language" : [

                                    ], 
                                    "status" : [

                                    ], 
                                    "weightage" : [

                                    ], 
                                    "created_at" : [

                                    ]
                                }, 
                                "isUnique" : false, 
                                "isSparse" : false, 
                                "isPartial" : false, 
                                "indexVersion" : 2.0, 
                                "direction" : "forward", 
                                "indexBounds" : {
                                    "language" : [
                                        "[\"hi\", \"hi\"]"
                                    ], 
                                    "status" : [
                                        "[1.0, 1.0]"
                                    ], 
                                    "weightage" : [
                                        "[MaxKey, MinKey]"
                                    ], 
                                    "created_at" : [
                                        "[1534271400.0, 1528914600.0]"
                                    ]
                                }
                            }
                        }
                    ]
                }, 
                "executionStats" : {
                    "executionSuccess" : true, 
                    "nReturned" : 7072.0, 
                    "executionTimeMillis" : 984.0, 
                    "totalKeysExamined" : 7072.0, 
                    "totalDocsExamined" : 7072.0, 
                    "executionStages" : {
                        "stage" : "FETCH", 
                        "filter" : {
                            "parent_topic_id" : {
                                "$eq" : null
                            }
                        }, 
                        "nReturned" : 7072.0, 
                        "executionTimeMillisEstimate" : 40.0, 
                        "works" : 7073.0, 
                        "advanced" : 7072.0, 
                        "needTime" : 0.0, 
                        "needYield" : 0.0, 
                        "saveState" : 63.0, 
                        "restoreState" : 63.0, 
                        "isEOF" : 1.0, 
                        "invalidates" : 0.0, 
                        "docsExamined" : 7072.0, 
                        "alreadyHasObj" : 0.0, 
                        "inputStage" : {
                            "stage" : "IXSCAN", 
                            "nReturned" : 7072.0, 
                            "executionTimeMillisEstimate" : 0.0, 
                            "works" : 7073.0, 
                            "advanced" : 7072.0, 
                            "needTime" : 0.0, 
                            "needYield" : 0.0, 
                            "saveState" : 63.0, 
                            "restoreState" : 63.0, 
                            "isEOF" : 1.0, 
                            "invalidates" : 0.0, 
                            "keyPattern" : {
                                "dash_status" : 1.0, 
                                "status" : 1.0, 
                                "language" : 1.0, 
                                "parent_topic_id" : 1.0, 
                                "created_at" : 1.0
                            }, 
                            "indexName" : "index_for_dashboard", 
                            "isMultiKey" : false, 
                            "multiKeyPaths" : {
                                "dash_status" : [

                                ], 
                                "status" : [

                                ], 
                                "language" : [

                                ], 
                                "parent_topic_id" : [

                                ], 
                                "created_at" : [

                                ]
                            }, 
                            "isUnique" : false, 
                            "isSparse" : false, 
                            "isPartial" : false, 
                            "indexVersion" : 2.0, 
                            "direction" : "forward", 
                            "indexBounds" : {
                                "dash_status" : [
                                    "[3.0, 3.0]"
                                ], 
                                "status" : [
                                    "[1.0, 1.0]"
                                ], 
                                "language" : [
                                    "[\"hi\", \"hi\"]"
                                ], 
                                "parent_topic_id" : [
                                    "[null, null]"
                                ], 
                                "created_at" : [
                                    "[1528914600.0, 1534271400.0]"
                                ]
                            }, 
                            "keysExamined" : 7072.0, 
                            "seeks" : 1.0, 
                            "dupsTested" : 0.0, 
                            "dupsDropped" : 0.0, 
                            "seenInvalidated" : 0.0
                        }
                    }
                }
            }
        }, 
        {
            "$lookup" : {
                "from" : "users", 
                "as" : "user", 
                "localField" : "creator", 
                "foreignField" : "_id", 
                "unwinding" : {
                    "preserveNullAndEmptyArrays" : false
                }, 
                "matching" : {
                    "$nor" : [
                        {
                            "device_os" : {
                                "$eq" : "BOT"
                            }
                        }
                    ]
                }
            }
        }, 
        {
            "$sort" : {
                "sortKey" : {
                    "created_at" : -1.0
                }, 
                "limit" : NumberLong(200)
            }
        }, 
        {
            "$addFields" : {
                "user_handle" : "$user.handle", 
                "user_phone" : "$user.phone", 
                "user_status" : "$user.status"
            }
        }, 
        {
            "$project" : {
                "_id" : true, 
                "hashtag" : true, 
                "voice_desc" : true, 
                "description" : true, 
                "location" : true, 
                "language" : true, 
                "topic_id" : true, 
                "image" : true, 
                "slug_generated" : true, 
                "n_vokes" : true, 
                "ref_id" : true, 
                "image_share" : true, 
                "message" : true, 
                "created_at" : true, 
                "default_text" : true, 
                "weightage" : true, 
                "user_handle" : true, 
                "title" : true, 
                "status" : true, 
                "creator" : true, 
                "user_status" : true, 
                "username" : true, 
                "user_phone" : true
            }
        }
    ], 
    "ok" : 1.0
}

这是在我使用“$match”中使用的键创建索引“index_for_dashboard”之后。我还从用户集合中为“device_os”创建了一个索引。但毫无疑问，响应时间没有任何改进。

可能的罪魁祸首：

  • $lookup 是否昂贵。如果可以，我可以只获取我需要的字段。
  • 索引是否可以做得更好。我应该使用一组不同的 字段？
  • $addField 是否昂贵。如果是，我可以将其卸载到 应用级别。

我应该如何对解释结果进行故障排除（和理解）以帮助自己优化查询？ 我不能在这里打个有根据的电话，需要一些指示。

【问题讨论】：

    标签： mongodb indexing lookup explain


    【解决方案1】：

    首先，您要确保至少运行 MongoDB v3.6.3，因为在 v3.6 中有一个新功能允许$lookup 指定子管道。这些管道实际上可以使用索引，但是，有一个 bug 仅在上述版本中得到修复。

    以下内容应该尽可能快地到达这里：

    db.getCollection("content_topics").createIndex({ created_at: -1, dash_status: 1, language: 1, parent_topic_id: 1, status: 1 }); // this index will get used by the main $match and the $sort stage

db.getCollection("users").createIndex({ device_os: 1 }); // this index will get used by the sub-pipeline in $lookup

db.getCollection("content_topics").aggregate([{
    "$match": { // filter at the start in order to be able to use indexes
      created_at: {
        "$gte": 1528914600,
        "$lte": 1534271400
      },
      dash_status: 3,
      language: "hi",
      parent_topic_id: {
        "$eq": null
      },
      status: 1
      // see the below $lookup stage in case you're wondering where the user filter went
    }
  },
  {
    "$sort": {
      created_at: -1 // sort straight away so the index can be used
    }
  },
  {
    "$lookup": {
      from: "users",
      let: { "creator": "$creator" },
      pipeline: [{ // use new v3.6 pipeline syntax to be able to leverage indexes
          $match: {
              $expr: {
                  $and: [
                      { $eq: [ "$_id", "$$creator" ] },
                      { $ne: [ "$device_os", "BOT" ] } // here is the "device_os" filter inside the pipeline so index can be used
                  ]
              }
          }
      }, {
          $project: {
              _id: 0, // "_id" field is not needed
              user_handle: 1, // only those fields are of interest
              user_phone: 1,
              user_status: 1
          }
      }],
      as: "user"
    }
  },
  {
    $unwind: "$user"
  },
  {
    "$project": {
      topic_id: 1,
      n_vokes: 1,
      message: 1,
      title: 1,
      language: 1,
      description: 1,
      voice_desc: 1,
      image: 1,
      image_share: 1,
      hashtag: 1,
      location: 1,
      default_text: 1,
      creator: 1,
      created_at: 1,
      status: 1,
      ref_id: 1,
      weightage: 1,
      username: 1,
      slug_generated: 1,
      user_handle: "$user.handle", // no extra $addFields stage needed
      user_phone: "$user.phone",   // same here
      user_status: "$user.status"  // and here
    }
  },
  {
    "$skip": 0
  },
  {
    "$limit": 200
  }
]);

    【讨论】：

    • 感谢您抽出宝贵时间详细解释这一点。我完全不知道我们可以在 $lookup 中使用管道，这非常漂亮，让我可以将匹配放在查找之前。它还有助于缩短响应时间，这可以通过将nReturned/works 从 7k 降低到 2k 来确认。 :)
    猜你喜欢
    • 2021-11-16
    • 2021-08-09
    • 2012-01-07
    • 1970-01-01
    • 2016-06-11
    • 1970-01-01
    • 1970-01-01
    相关资源
    最近更新 更多
    热门标签
    Java Python linux javascript Mysql C# Docker 算法 前端 SpringBoot Redis Vue spring 设计模式 .net core .net kubernetes c++ 数据库 数据结构 大数据 js 机器学习 微服务 Android Go 程序员 面试 JVM ASP.net core 云原生 人工智能 后端 PHP git CSS golang k8s Nginx Django mybatis 深度学习 多线程 React 架构 devops 爬虫 云计算 Spring Boot LeetCode