使用 JQ 从 JSON 数据创建 .csv 文件

Question

我以为我对 jq 的了解足够多，能够以我想要的方式格式化 .csv 文件，但总有一个技巧我错过了！。

我的 API 下载看起来像这样......

{
  "status": "ok",
  "meta": {
    "count": 4
  },
  "data": {
    "1019761328": {
      "achievements": {
        "medalCarius": 1,
        "medalHalonen": 3,
        "aimer": 6,
        "invader": 13,
        "armorPiercer": 18,
        "medalMonolith": 2,
        "medalEkins": 1,
        "medalKay": 2,
        "duelist": 409,
        "newMeritPM2": 1,
        "readyForBattleLT": 4,
        "defender": 15,
        "readyForBattleATSPG": 4,
        "medalLeClerc": 2,
        "demolition": 112,
        "supporter": 13,
        "steelwall": 107,
        "medalLehvaslaiho": 28,
        "medalAbrams": 2,
        "readyForBattleSPG": 4,
        "medalPoppel": 1,
        "medalPascucci": 68,
        "reliableComrade": 303,
        "NY19A1": 1,
        "NY19A2": 1,
        "tankwomen": 1,
        "luckyDevil": 10,
        "NY18A3": 1,
        "NY18A2": 1,
        "mainGun": 28,
        "NY18A1": 1,
        "sinai": 5,
        "firstMerit": 1,
        "medalOrlik": 8,
        "bonecrusher": 824,
        "titleSniper": 41,
        "warrior": 5,
        "ironMan": 130,
        "huntsman": 2,
        "even": 35,
        "medalKolobanov": 1,
        "scout": 4,
        "beasthunter": 5,
        "kamikaze": 30,
        "02YearsOfService": 1,
        "tankExpert2": 1,
        "tankExpert1": 1,
        "readyForBattleMT": 4,
        "tankExpert7": 1,
        "tankExpert6": 1,
        "sniper2": 10,
        "arsonist": 106,
        "charmed": 194,
        "medalBillotte": 1,
        "fighter": 147,
        "medalLavrinenko": 2,
        "impenetrable": 155,
        "sturdy": 65,
        "NY19A3": 1,
        "medalKursk": 1,
        "soldierOfFortune": 4,
        "handOfDeath": 4,
        "DdaymarathonMedal": 1,
        "shootToKill": 3029,
        "medalDumitru": 3,
        "evileye": 8,
        "medalKnispel": 1
      },
      "frags": {
        "crucialShotMedal": 0,
        "prematureDetonationMedal": 0,
        "sentinelMedal": 0,
        "infiltratorMedal": 0,
        "fightingReconnaissanceMedal": 0,
        "fireAndSteelMedal": 0,
        "rangerMedal": 0,
        "reliableComrade": 29,
        "pyromaniacMedal": 0,
        "wolfAmongSheepMedal": 0,
        "heavyFireMedal": 0,
        "bruteForceMedal": 0,
        "guerrillaMedal": 0,
        "promisingFighterMedal": 0,
        "beasthunter": 595,
        "geniusForWarMedal": 0,
        "sinai": 523,
        "pattonValley": 62
      },
      "max_series": {
        "armorPiercer": 18,
        "aimer": 6,
        "titleSniper": 41,
        "deathTrack": 0,
        "invincible": 3,
        "victoryMarch": 0,
        "EFC2016": 0,
        "diehard": 6,
        "WFC2014": 0,
        "tacticalBreakthrough": 0,
        "handOfDeath": 4
      }
    },
    "1034967155": {
      "achievements": {},
      "frags": {
        "crucialShotMedal": 0,
        "prematureDetonationMedal": 0,
        "sentinelMedal": 0,
        "infiltratorMedal": 0,
        "fightingReconnaissanceMedal": 0,
        "fireAndSteelMedal": 0,
        "rangerMedal": 0,
        "reliableComrade": 0,
        "pyromaniacMedal": 0,
        "wolfAmongSheepMedal": 0,
        "heavyFireMedal": 0,
        "bruteForceMedal": 0,
        "guerrillaMedal": 0,
        "promisingFighterMedal": 0,
        "beasthunter": 0,
        "geniusForWarMedal": 0,
        "sinai": 0,
        "pattonValley": 0
      },
      "max_series": {
        "armorPiercer": 0,
        "aimer": 0,
        "titleSniper": 0,
        "deathTrack": 0,
        "invincible": 0,
        "victoryMarch": 0,
        "EFC2016": 0,
        "diehard": 0,
        "WFC2014": 0,
        "tacticalBreakthrough": 0,
        "handOfDeath": 0
      }
    }
  }
}  

我的 .csv 输出必须包含一个 ID 字段、一个奖牌字段和一个奖牌数字段，如下所示......

1019761328,"medalCarius",1
1019761328,"medalHalonen",3
1019761328,"aimer",6

...... etc. repeated for every ID

到目前为止，这些命令删除了我需要的数据......

jq -r '.data | to_entries[] | {id: .key, val: .value[]} '

导致....

{
  "id": "1019761328",
  "val": {
    "medalCarius": 1,
    "medalHalonen": 3,
    "aimer": 6,
    "invader": 13,
    "armorPiercer": 18,
    "medalMonolith": 2,
    "medalEkins": 1,
    "medalKay": 2,
    "duelist": 409,
    "newMeritPM2": 1,
    "readyForBattleLT": 4,
    "defender": 15,
    "readyForBattleATSPG": 4,
    "medalLeClerc": 2,
    "demolition": 112,
    "supporter": 13,
    "steelwall": 107,
    "medalLehvaslaiho": 28,
    "medalAbrams": 2,
    "readyForBattleSPG": 4,
    "medalPoppel": 1,
    "medalPascucci": 68,
    "reliableComrade": 303,
    "NY19A1": 1,
    "NY19A2": 1,
    "tankwomen": 1,
    "luckyDevil": 10,
    "NY18A3": 1,
    "NY18A2": 1,
    "mainGun": 28,
    "NY18A1": 1,
    "sinai": 5,
    "firstMerit": 1,
    "medalOrlik": 8,
    "bonecrusher": 824,
    "titleSniper": 41,
    "warrior": 5,
    "ironMan": 130,
    "huntsman": 2,
    "even": 35,
    "medalKolobanov": 1,
    "scout": 4,
    "beasthunter": 5,
    "kamikaze": 30,
    "02YearsOfService": 1,
    "tankExpert2": 1,
    "tankExpert1": 1,
    "readyForBattleMT": 4,
    "tankExpert7": 1,
    "tankExpert6": 1,
    "sniper2": 10,
    "arsonist": 106,
    "charmed": 194,
    "medalBillotte": 1,
    "fighter": 147,
    "medalLavrinenko": 2,
    "impenetrable": 155,
    "sturdy": 65,
    "NY19A3": 1,
    "medalKursk": 1,
    "soldierOfFortune": 4,
    "handOfDeath": 4,
    "DdaymarathonMedal": 1,
    "shootToKill": 3029,
    "medalDumitru": 3,
    "evileye": 8,
    "medalKnispel": 1
  }
}
{
  "id": "1019761328",
  "val": {
    "crucialShotMedal": 0,
    "prematureDetonationMedal": 0,
    "sentinelMedal": 0,
    "infiltratorMedal": 0,
    "fightingReconnaissanceMedal": 0,
    "fireAndSteelMedal": 0,
    "rangerMedal": 0,
    "reliableComrade": 29,
    "pyromaniacMedal": 0,
    "wolfAmongSheepMedal": 0,
    "heavyFireMedal": 0,
    "bruteForceMedal": 0,
    "guerrillaMedal": 0,
    "promisingFighterMedal": 0,
    "beasthunter": 595,
    "geniusForWarMedal": 0,
    "sinai": 523,
    "pattonValley": 62
  }
}
{
  "id": "1019761328",
  "val": {
    "armorPiercer": 18,
    "aimer": 6,
    "titleSniper": 41,
    "deathTrack": 0,
    "invincible": 3,
    "victoryMarch": 0,
    "EFC2016": 0,
    "diehard": 6,
    "WFC2014": 0,
    "tacticalBreakthrough": 0,
    "handOfDeath": 4
  }
}
{
  "id": "1034967155",
  "val": {}
}
{
  "id": "1034967155",
  "val": {
    "crucialShotMedal": 0,
    "prematureDetonationMedal": 0,
    "sentinelMedal": 0,
    "infiltratorMedal": 0,
    "fightingReconnaissanceMedal": 0,
    "fireAndSteelMedal": 0,
    "rangerMedal": 0,
    "reliableComrade": 0,
    "pyromaniacMedal": 0,
    "wolfAmongSheepMedal": 0,
    "heavyFireMedal": 0,
    "bruteForceMedal": 0,
    "guerrillaMedal": 0,
    "promisingFighterMedal": 0,
    "beasthunter": 0,
    "geniusForWarMedal": 0,
    "sinai": 0,
    "pattonValley": 0
  }
}
{
  "id": "1034967155",
  "val": {
    "armorPiercer": 0,
    "aimer": 0,
    "titleSniper": 0,
    "deathTrack": 0,
    "invincible": 0,
    "victoryMarch": 0,
    "EFC2016": 0,
    "diehard": 0,
    "WFC2014": 0,
    "tacticalBreakthrough": 0,
    "handOfDeath": 0
  }
}

我现在如何将其转换为我需要的 .csv 格式？

任何帮助将不胜感激 - 谢谢！

Answer 1

您可以使用to_entries[] 的另一个应用程序再次扩展.value，然后使用string interpolation 生成csv 输出。 例如

.data | to_entries[] | {id: .key, val: (.value[] | to_entries[])} | "\(.id),\(.val.key),\(.val.value)"

示例执行：使用test.jq 中的上述命令和test.json 中的数据并使用head 仅显示前十行：

$ jq -Mr -f test.jq test.json | head
1019761328,medalCarius,1
1019761328,medalHalonen,3
1019761328,aimer,6
1019761328,invader,13
1019761328,armorPiercer,18
1019761328,medalMonolith,2
1019761328,medalEkins,1
1019761328,medalKay,2
1019761328,duelist,409
1019761328,newMeritPM2,1

对于您的具体情况，更可靠的方法是使用 @csv 而不是字符串插值。例如

.data | to_entries[] | {id: .key, val: (.value[] | to_entries[])} | [.id, .val.key, .val.value] | @csv

示例执行

$ jq -Mr -f test.jq test.json | head
"1019761328","medalCarius",1
"1019761328","medalHalonen",3
"1019761328","aimer",6
"1019761328","invader",13
"1019761328","armorPiercer",18
"1019761328","medalMonolith",2
"1019761328","medalEkins",1
"1019761328","medalKay",2
"1019761328","duelist",409
"1019761328","newMeritPM2",1

如果您知道 id 是一个数字并希望输出反映您也可以使用 tonumber：

.data | to_entries[] | {id: .key, val: (.value[] | to_entries[])} | [(.id|tonumber), .val.key, .val.value] | @csv

示例执行

$ jq -Mr -f test.jq test.json | head
1019761328,"medalCarius",1
1019761328,"medalHalonen",3
1019761328,"aimer",6
1019761328,"invader",13
1019761328,"armorPiercer",18
1019761328,"medalMonolith",2
1019761328,"medalEkins",1
1019761328,"medalKay",2
1019761328,"duelist",409
1019761328,"newMeritPM2",1

Answer 2

在某种程度上，您需要应用从宽到长的转换。

您可以在不使用 Miller (https://github.com/johnkerl/miller/issues) 的情况下完成您的 jq 工作，并将此命令应用于您的 json 输出

mlr --j2c unsparsify  then reshape -r "val:"  -o item,value then filter -S '$value!=""' then put -S '$item=gsub($item,"val:","")' input.json >output.csv

你会有

id,item,value
1019761328,medalCarius,1
1019761328,medalHalonen,3
1019761328,aimer,6
1019761328,invader,13
1019761328,armorPiercer,18
1019761328,medalMonolith,2
1019761328,medalEkins,1
1019761328,medalKay,2
1019761328,duelist,409
1019761328,newMeritPM2,1
1019761328,readyForBattleLT,4
1019761328,defender,15
1019761328,readyForBattleATSPG,4
1019761328,medalLeClerc,2
1019761328,demolition,112
1019761328,supporter,13
1019761328,steelwall,107
1019761328,medalLehvaslaiho,28
1019761328,medalAbrams,2
1019761328,readyForBattleSPG,4
1019761328,medalPoppel,1
1019761328,medalPascucci,68
1019761328,reliableComrade,303
1019761328,NY19A1,1
1019761328,NY19A2,1
1019761328,tankwomen,1
1019761328,luckyDevil,10
1019761328,NY18A3,1
1019761328,NY18A2,1
1019761328,mainGun,28
1019761328,NY18A1,1
1019761328,sinai,5
1019761328,firstMerit,1
1019761328,medalOrlik,8
1019761328,bonecrusher,824
1019761328,titleSniper,41
1019761328,warrior,5
1019761328,ironMan,130
1019761328,huntsman,2
1019761328,even,35
1019761328,medalKolobanov,1
1019761328,scout,4
1019761328,beasthunter,5
1019761328,kamikaze,30
1019761328,02YearsOfService,1
1019761328,tankExpert2,1
1019761328,tankExpert1,1
1019761328,readyForBattleMT,4
1019761328,tankExpert7,1
1019761328,tankExpert6,1
1019761328,sniper2,10
1019761328,arsonist,106
1019761328,charmed,194
1019761328,medalBillotte,1
1019761328,fighter,147
1019761328,medalLavrinenko,2
1019761328,impenetrable,155
1019761328,sturdy,65
1019761328,NY19A3,1
1019761328,medalKursk,1
1019761328,soldierOfFortune,4
1019761328,handOfDeath,4
1019761328,DdaymarathonMedal,1
1019761328,shootToKill,3029
1019761328,medalDumitru,3
1019761328,evileye,8
1019761328,medalKnispel,1
1019761328,reliableComrade,29
1019761328,sinai,523
1019761328,beasthunter,595
1019761328,crucialShotMedal,0
1019761328,prematureDetonationMedal,0
1019761328,sentinelMedal,0
1019761328,infiltratorMedal,0
1019761328,fightingReconnaissanceMedal,0
1019761328,fireAndSteelMedal,0
1019761328,rangerMedal,0
1019761328,pyromaniacMedal,0
1019761328,wolfAmongSheepMedal,0
1019761328,heavyFireMedal,0
1019761328,bruteForceMedal,0
1019761328,guerrillaMedal,0
1019761328,promisingFighterMedal,0
1019761328,geniusForWarMedal,0
1019761328,pattonValley,62
1019761328,aimer,6
1019761328,armorPiercer,18
1019761328,titleSniper,41
1019761328,handOfDeath,4
1019761328,deathTrack,0
1019761328,invincible,3
1019761328,victoryMarch,0
1019761328,EFC2016,0
1019761328,diehard,6
1019761328,WFC2014,0
1019761328,tacticalBreakthrough,0
1034967155,reliableComrade,0
1034967155,sinai,0
1034967155,beasthunter,0
1034967155,crucialShotMedal,0
1034967155,prematureDetonationMedal,0
1034967155,sentinelMedal,0
1034967155,infiltratorMedal,0
1034967155,fightingReconnaissanceMedal,0
1034967155,fireAndSteelMedal,0
1034967155,rangerMedal,0
1034967155,pyromaniacMedal,0
1034967155,wolfAmongSheepMedal,0
1034967155,heavyFireMedal,0
1034967155,bruteForceMedal,0
1034967155,guerrillaMedal,0
1034967155,promisingFighterMedal,0
1034967155,geniusForWarMedal,0
1034967155,pattonValley,0
1034967155,aimer,0
1034967155,armorPiercer,0
1034967155,titleSniper,0
1034967155,handOfDeath,0
1034967155,deathTrack,0
1034967155,invincible,0
1034967155,victoryMarch,0
1034967155,EFC2016,0
1034967155,diehard,0
1034967155,WFC2014,0
1034967155,tacticalBreakthrough,0