Mongo / Mongoose 聚合 - $redact 和 $cond 问题

【问题标题】：Mongo / Mongoose Aggregation - $redact and $cond issuesMongo / Mongoose 聚合 - $redact 和 $cond 问题
【发布时间】：2017-10-12 13:00:07
【问题描述】：

我很幸运地从@chridam 获得了另一个 SO 问题Mongo / Mongoose - Aggregating by Date 的精彩答案，其中给出了一组文档，例如：

{ "_id" : ObjectId("5907a5850b459d4fdcdf49ac"), "amount" : -33.3, "name" : "RINGGO", "method" : "VIS", "date" : ISODate("2017-04-26T23:00:00Z"), "importDate" : ISODate("2017-05-01T21:15:49.581Z"), "category" : "Not Set", "__v" : 0 }
{ "_id" : ObjectId("5907a5850b459d4fdcdf49ba"), "amount" : -61.3, "name" : "Amazon", "method" : "VIS", "date" : ISODate("2017-03-23T00:00:00Z"), "importDate" : ISODate("2017-05-01T21:15:49.592Z"), "category" : "Not Set", "__v" : 0 }
{ "_id" : ObjectId("5907a5850b459d4fdcdf49ce"), "amount" : -3.3, "name" : "Tesco", "method" : "VIS", "date" : ISODate("2017-03-15T00:00:00Z"), "importDate" : ISODate("2017-05-01T21:15:49.601Z"), "category" : "Not Set", "__v" : 0 }
{ "_id" : ObjectId("5907a5850b459d4fdcdf49cc"), "amount" : -26.3, "name" : "RINGGO", "method" : "VIS", "date" : ISODate("2017-03-16T00:00:00Z"), "importDate" : ISODate("2017-05-01T21:15:49.600Z"), "category" : "Not Set", "__v" : 0 }
{ "_id" : ObjectId("5907a5850b459d4fdcdf49f7"), "amount" : -63.3, "name" : "Sky", "method" : "VIS", "date" : ISODate("2017-03-02T00:00:00Z"), "importDate" : ISODate("2017-05-01T21:15:49.617Z"), "category" : "Not Set", "__v" : 0 }
{ "_id" : ObjectId("5907a5850b459d4fdcdf49be"), "amount" : -3.3, "name" : "RINGGO", "method" : "VIS", "date" : ISODate("2017-03-22T00:00:00Z"), "importDate" : ISODate("2017-05-01T21:15:49.593Z"), "category" : "Not Set", "__v" : 0 }

需要一个按供应商、年、月和周汇总支出的查询。查询如下，它几乎工作得非常好，但是当我在我的应用程序中使用它时，我注意到了一个重大问题

db.statements.aggregate([
  { "$match": { "name": "RINGGO" } },
  {
  "$redact": {
      "$cond": [
          {
              "$and": [
                 { "$eq": [{ "$year": "$date" },  2017  ]}, // within my route this uses parseInt(req.params.year)
                 { "$eq": [{ "$month": "$date" }, 3 ]}, // within my route this uses parseInt(req.params.month)
                 { "$eq": [{ "$week": "$date" },  12  ]} // within my route this uses parseInt(req.params.week)
            ]
        },
        "$$KEEP",
        "$$PRUNE"
    ]
}
},{
    "$group": {
        "_id": {
            "name": "$name",
            "year": { "$year": "$date" },
            "month": { "$month": "$date" },
            "week": { "$week": "$date" }
        },
        "total": { "$sum": "$amount" }
    }
},
{
    "$group": {
        "_id": {
            "name": "$_id.name",
            "year": "$_id.year"
        },
        "YearlySpends": { "$push": "$total" },
        "totalYearlyAmount": { "$sum": "$total" },
        "data": { "$push": "$$ROOT" }
    }
},
{ "$unwind": "$data" },
{
    "$group": {
        "_id": {
            "name": "$_id.name",
            "month": "$data._id.month"
        },
        "YearlySpends": { "$first": "$YearlySpends" },
        "totalYearlyAmount": { "$first": "$totalYearlyAmount" },
        "MonthlySpends": { "$push": "$data.total" },
        "totalMonthlyAmount": { "$sum": "$data.total" },
        "data": { "$push": "$data" }
    }
},
{ "$unwind": "$data" },
{
    "$group": {
        "_id": {
            "name": "$_id.name",
            "week": "$data._id.week"
        },
        "YearlySpends": { "$first": "$YearlySpends" },
        "totalYearlyAmount": { "$first": "$totalYearlyAmount" },
        "MonthlySpends": { "$first": "$MonthlySpends" },
        "totalMonthlyAmount": { "$first": "$totalMonthlyAmount" },
        "WeeklySpends": { "$push": "$data.total" },
        "totalWeeklyAmount": { "$sum": "$data.total" },
        "data": { "$push": "$data" }
    }
},
{ "$unwind": "$data" },
{
    "$group": {
        "_id": "$data._id",
        "YearlySpends": { "$first": "$YearlySpends" },
        "totalYearlyAmount": { "$first": "$totalYearlyAmount" },
        "MonthlySpends": { "$first": "$MonthlySpends" },
        "totalMonthlyAmount": { "$first": "$totalMonthlyAmount" },
        "WeeklySpends": { "$first": "$WeeklySpends" },
        "totalWeeklyAmount": { "$first": "$totalWeeklyAmount" }
    }
}
])

运行此查询返回

{ "_id" :
 { "name" : "RINGGO", 
   "year" : 2017, 
   "month" : 3, 
   "week" : 12 }, 
   "YearlySpends" : [ -9.6 ], 
   "totalYearlyAmount" : -9.6, 
   "MonthlySpends" : [ -9.6 ], 
   "totalMonthlyAmount" : -9.6, 
   "WeeklySpends" : [ -9.6 ], 
   "totalWeeklyAmount" : -9.6 
}

当我改为想查看当月的支出时

"$cond": [
          {
            "$and": [
                 { "$eq": [{ "$year": "$date" },  2017  ]},
                 { "$eq": [{ "$month": "$date" }, 3 ]}
            ]
          },
        "$$KEEP",
        "$$PRUNE"
      ]

我明白了：

{ "_id" : { "name" : "RINGGO", "year" : 2017, "month" : 3, "week" : 12 }, "YearlySpends" : [ -3.3, -9.6, -9.6, -3.3 ], "totalYearlyAmount" : -25.799999999999997, "MonthlySpends" : [ -3.3, -9.6, -9.6, -3.3 ], "totalMonthlyAmount" : -25.799999999999997, "WeeklySpends" : [ -9.6 ], "totalWeeklyAmount" : -9.6 }
{ "_id" : { "name" : "RINGGO", "year" : 2017, "month" : 3, "week" : 9 }, "YearlySpends" : [ -3.3, -9.6, -9.6, -3.3 ], "totalYearlyAmount" : -25.799999999999997, "MonthlySpends" : [ -3.3, -9.6, -9.6, -3.3 ], "totalMonthlyAmount" : -25.799999999999997, "WeeklySpends" : [ -3.3 ], "totalWeeklyAmount" : -3.3 }
{ "_id" : { "name" : "RINGGO", "year" : 2017, "month" : 3, "week" : 11 }, "YearlySpends" : [ -3.3, -9.6, -9.6, -3.3 ], "totalYearlyAmount" : -25.799999999999997, "MonthlySpends" : [ -3.3, -9.6, -9.6, -3.3 ], "totalMonthlyAmount" : -25.799999999999997, "WeeklySpends" : [ -9.6 ], "totalWeeklyAmount" : -9.6 }
{ "_id" : { "name" : "RINGGO", "year" : 2017, "month" : 3, "week" : 13 }, "YearlySpends" : [ -3.3, -9.6, -9.6, -3.3 ], "totalYearlyAmount" : -25.799999999999997, "MonthlySpends" : [ -3.3, -9.6, -9.6, -3.3 ], "totalMonthlyAmount" : -25.799999999999997, "WeeklySpends" : [ -3.3 ], "totalWeeklyAmount" : -3.3 }

但是，当我运行一个简单的 db.statements.find({"name":"RINGGO"}) 时，我得到：

{ "_id" : ObjectId("5907a5850b459d4fdcdf49ac"), "amount" : -3.3, "name" : "RINGGO", "method" : "VIS", "date" : ISODate("2017-03-26T23:00:00Z"), "importDate" : ISODate("2017-05-01T21:15:49.581Z"), "category" : "Not Set", "__v" : 0 }
{ "_id" : ObjectId("5907a5850b459d4fdcdf49ba"), "amount" : -6.3, "name" : "RINGGO", "method" : "VIS", "date" : ISODate("2017-03-23T00:00:00Z"), "importDate" : ISODate("2017-05-01T21:15:49.592Z"), "category" : "Not Set", "__v" : 0 }
{ "_id" : ObjectId("5907a5850b459d4fdcdf49ce"), "amount" : -3.3, "name" : "RINGGO", "method" : "VIS", "date" : ISODate("2017-03-15T00:00:00Z"), "importDate" : ISODate("2017-05-01T21:15:49.601Z"), "category" : "Not Set", "__v" : 0 }
{ "_id" : ObjectId("5907a5850b459d4fdcdf49cc"), "amount" : -6.3, "name" : "RINGGO", "method" : "VIS", "date" : ISODate("2017-03-16T00:00:00Z"), "importDate" : ISODate("2017-05-01T21:15:49.600Z"), "category" : "Not Set", "__v" : 0 }
{ "_id" : ObjectId("5907a5850b459d4fdcdf49f7"), "amount" : -3.3, "name" : "RINGGO", "method" : "VIS", "date" : ISODate("2017-03-02T00:00:00Z"), "importDate" : ISODate("2017-05-01T21:15:49.617Z"), "category" : "Not Set", "__v" : 0 }
{ "_id" : ObjectId("5907a5850b459d4fdcdf49be"), "amount" : -3.3, "name" : "RINGGO", "method" : "VIS", "date" : ISODate("2017-03-22T00:00:00Z"), "importDate" : ISODate("2017-05-01T21:15:49.593Z"), "category" : "Not Set", "__v" : 0 }

因此您可以看到，与按名称查找的输出中显示的相比，先前输出中 MonthlySpends 中的项目数量不同。您还可以看到，一些值在 MonthlySpends 中不应该被加在一起。

理想情况下，我希望得到以下输出： 当我有$redact 包含：

"$cond": [
        {
            "$and": [
                 { "$eq": [{ "$year": "$date" },  2017  ]}, 
                 { "$eq": [{ "$month": "$date" }, 3 ]}, 
                 { "$eq": [{ "$week": "$date" },  12  ]} 
            ]
        },
        "$$KEEP",
        "$$PRUNE"
    ]

返回

{ "_id" : { "name" : "RINGGO", "year" : 2017, "month" : 3, "week" : 12 }, "YearlySpends" : [ -3.3, -9.6, -9.6, -3.3 ], "totalYearlyAmount" : -25.799999999999997, "MonthlySpends" : [ -3.3, -9.6, -9.6, -3.3 ], "totalMonthlyAmount" : -25.799999999999997, "WeeklySpends" : [ -9.6 ], "totalWeeklyAmount" : -9.6 }

当我有 $redact 包含：

"$cond": [
        {
            "$and": [
                 { "$eq": [{ "$year": "$date" },  2017  ]}, 
                 { "$eq": [{ "$month": "$date" }, 3 ]},
            ]
        },
        "$$KEEP",
        "$$PRUNE"
        ]

返回

{ "_id" : { "name" : "RINGGO", "year" : 2017, "month" : 3 }, "YearlySpends" : [ -3.3, -9.6, -9.6, -3.3 ], "totalYearlyAmount" : -25.799999999999997, "MonthlySpends" : [ -3.3, -9.6, -9.6, -3.3 ], "totalMonthlyAmount" : -25.799999999999997 }

当我有 $redact 包含：

"$cond": [
        {
            "$and": [
                 { "$eq": [{ "$year": "$date" },  2017  ]}
            ]
        },
        "$$KEEP",
        "$$PRUNE"
        ]

返回

{ "_id" : { "name" : "RINGGO", "year" : 2017 }, "YearlySpends" : [ -3.3, -9.6, -9.6, -3.3 ], "totalYearlyAmount" : -25.799999999999997}

在这方面需要任何帮助。我已经尝试修改查询，但恐怕我对它的理解还不够，无法正确修改它。

我的 Mongoose 版本是 ^4.9.5，我的 mongo 版本是 3.4.2。

【问题讨论】：

  • 使用$redact 过滤日期范围非常低效。您真的应该在初始 $match 管道阶段使用 $gte 和 $lte 运算符简单地查询“范围”。
  • 您似乎也忽略了这一点，即随后的$group 阶段逐渐降低了这里的粒度。即首先汇总一周，然后是一个月中的所有周，一年中的几个月。如果您打算将最初的选择范围更改为仅一周或仅一个月，那么您跟进较少的$group 阶段。
  • @NeilLunn 感谢您抽出宝贵时间发表评论！我很确定我在这里错过了重点。我是 mongo/mongoose 新手，所以努力学习！我试过删除一些$group 阶段，但它不会改变返回的记录数。此外，这将如何影响 "MonthlySpends" : [ -3.3, -9.6, -9.6, -3.3 ] 中不正确的项目数量以及 totalMonthlyAmount 中的项目总和？您愿意发布一个我可以测试并希望标记为正确的答案吗？
  • @NeilLunn 未使用 $gte 和 $lte 的原因可能是查询实际上是由 URL 驱动的，例如RINGGO/2017/3/12 或 RINGGO/2017/3。您可以在上面的问题中看到代码 cmets 显示 parseInt(req.params.year)、parseInt(req.params.month) 等
  • 为了完整起见，您还应该添加您的 mongodb 和 mongoose 版本。

标签： node.js mongodb mongoose mongodb-query aggregation-framework


【解决方案1】：

您可以在3.4 版本中尝试$facet 和$addFields 进行并行聚合。

这将降低整体复杂性，您可以同时使用自己的匹配输入运行分组。

以下代码根据请求对象动态构建聚合管道。

// Sample request
var request = {
  "name":"RINGGO",
  "year": 2017,
  "month":3,
  "week":12
};

// Build initial match document on name

var match1 = {
  name: request["name"]
};

// Build project & facet document for date based aggregation

var addFields = {};
var facet = {};

// Add year followed by year facet

if (request["year"]) {
    addFields["year"] = { "$year": "$date" },
    facet["Yearly"] = 
      [
        {
          "$match":{ "year": request["year"] }
        },
        {
          "$group": {
            "_id": {
              "name": "$name",
              "year": "$year"
            },
            "spend": { "$push":"$amount" },
            "total": { "$sum": "$amount" }
        }
      }
    ];
}

// Add month followed by month facet

if (request["month"]) {
    addFields["month"] = { "$month": "$date" };
    facet["Monthly"] = 
      [
        {
          "$match":{ "month": request["month"] }
        },
        {
          "$group": {
            "_id": {
              "name": "$name",
              "month": "$month"
            },
            "spend": { "$push":"$amount" },
            "total": { "$sum": "$amount" }
         }
      }
    ];
}

// Add week followed by week facet

if (request["week"]) {
    addFields["week"] = { "$week": "$date" };
    facet["Weekly"] = 
      [
        {
          "$match":{ "week": request["week"] }
        },
        {
          "$group": {
            "_id": {
              "name": "$name",
              "week": "$week"
            },
            "spend": { "$push":"$amount" },
            "total": { "$sum": "$amount" }
         }
      }
    ];
}

// Use aggregate builder

statements.aggregate()
        .match(match1)
        .append({"$addFields": addFields}) // No addFields stage in mongoose builder
        .facet(facet)
        .exec(function(err, data) {});

Mongo Shell 查询 name/year/month/week 条件。

db.statements.aggregate({
    '$match': {
        name: 'RINGGO'
    }
}, {
    '$addFields': {
        year: {
            '$year': '$date'
        },
        month: {
            '$month': '$date'
        },
        week: {
            '$week': '$date'
        }
    }
}, {
    '$facet': {
        Yearly: [{
                '$match': {
                    year: 2017
                }
            },
            {
                '$group': {
                    _id: {
                        name: '$name',
                        year: '$year'
                    },
                    spend: {
                        '$push': '$amount'
                    },
                    total: {
                        '$sum': '$amount'
                    }
                }
            }
        ],
        Monthly: [{
                '$match': {
                    month: 3
                }
            },
            {
                '$group': {
                    _id: {
                        name: '$name',
                        month: '$month'
                    },
                    spend: {
                        '$push': '$amount'
                    },
                    total: {
                        '$sum': '$amount'
                    }
                }
            }
        ],
        Weekly: [{
                '$match': {
                    week: 12
                }
            },
            {
                '$group': {
                    _id: {
                        name: '$name',
                        week: '$week'
                    },
                    spend: {
                        '$push': '$amount'
                    },
                    total: {
                        '$sum': '$amount'
                    }
                }
            }
        ]
    }
})

示例响应

    {
    "Yearly": [{
        "_id": {
            "name": "RINGGO",
            "year": 2017
        },
        "spend": [-3.3, -6.3, -3.3, -6.3, -3.3, -3.3],
        "total": -25.799999999999997
    }],
    "Monthly": [{
        "_id": {
            "name": "RINGGO",
            "month": 3
        },
        "spend": [-3.3, -6.3, -3.3, -6.3, -3.3, -3.3],
        "total": -25.799999999999997
    }],
    "Weekly": [{
        "_id": {
            "name": "RINGGO",
            "week": 12
        },
        "spend": [-6.3, -3.3],
        "total": -9.6
    }]
}

您可以对Year/Month 和Year 输入值运行类似的聚合。

所以你可以看到里面有不同数量的项目 上一个输出中的 MonthlySpends 与输出中显示的比较 从按名称查找。您还可以看到一些值是 在不应将它们汇总到 MonthlySpends 中。

这发生在 $group 1 中，其中 $week 聚合将两个日期 [15, 16] 中的每一个汇总到第 11 周，其他两个日期 [22, 23] 汇总到第 12 周后显示为总计在MonthySpends。

【讨论】：

  • 感谢@veeram，这看起来很有希望。请给我一点时间，今晚在我的应用程序的一系列场景中进行测试。我是否认为只获得Year/Month 而不是week 的结果我只是从$addFields 中删除week: { '$week': '$date' }，然后删除Weekly facet？
  • 不客气。请慢慢来，等待其他解决方案。是的，你的理解是正确的。您可以删除整周的代码块（if (request["week"]){..}），或者如果您在请求对象（var request = { "name":"RINGGO", "year": 2017, "month":3 };）中不包含/缺少星期参数，这会将 if (request["week"]) 评估为 false，因此不会包含该代码堵塞。 ——
  • 很抱歉，我知道这不是最初表达的，但有没有办法在每年、每月、每周spend 内获取支出日期？也许spend 可能看起来像 '"spend": [{date:2017-03-26, amount:-3.3}, {date:2017-03-25, amount:-6.3},]' 等，或者你可以有个更好的主意
  • 这不是问题。您可以将聚合管道的spend 部分更新为spend: { '$push': {'date':{ $dateToString: { 'format': "%Y-%m-%d", 'date': '$date' } }, 'amount':'$amount'} }。使用$dateToString 运算符来格式化日期。
猜你喜欢
  • 2020-06-21
  • 2016-02-10
  • 2017-05-23
  • 2018-07-15
  • 2015-11-18
  • 1970-01-01
  • 2017-10-07
  • 2019-02-03
  • 2018-08-18
相关资源
最近更新 更多
热门标签
Java Python linux javascript Mysql C# Docker 算法 前端 SpringBoot Redis Vue spring 设计模式 .net core .net kubernetes c++ 数据库 数据结构 大数据 js 机器学习 微服务 Android Go 程序员 面试 JVM ASP.net core 云原生 人工智能 后端 PHP git CSS golang k8s Nginx Django mybatis 深度学习 多线程 React 架构 devops 爬虫 云计算 Spring Boot LeetCode