【问题标题】:Best way to extract/format data in JSON format using Python?使用 Python 以 JSON 格式提取/格式化数据的最佳方法?
【发布时间】:2020-07-11 02:58:13
【问题描述】:

我正在尝试对批量专利数据进行一些数据分析(数据通常在此处找到,但目前已关闭 - https://ped.uspto.gov/peds/)。

这是 JSON 文件中的第一个条目:

{
  "PatentBulkData":[
    {
      "patentCaseMetadata":{
        "applicationNumberText":{
          "value":"15733015",
          "electronicText":"15733015"
        },
        "filingDate":"2020-01-01",
        "applicationTypeCategory":"Utility",
        "partyBag":{
          "applicantBagOrInventorBagOrOwnerBag":[
            {
              "applicant":[
                {
                  "contactOrPublicationContact":[
                    {
                      "name":{
                        "personNameOrOrganizationNameOrEntityName":[
                          {
                            "personStructuredName":{
                              "firstName":"Birol",
                              "middleName":"",
                              "lastName":"Cimen"
                            }
                          }
                        ]
                      },
                      "cityName":"Hengelo",
                      "geographicRegionName":{
                        "value":"",
                        "geographicRegionCategory":"STATE"
                      },
                      "countryCode":"NL"
                    }
                  ]
                }
              ]
            },
            {
              "partyIdentifierOrContact":[
                {
                  "name":{
                    "personNameOrOrganizationNameOrEntityName":[
                      {
                        "personStructuredName":{
                          "lastName":"Oppedahl Patent Law Firm LLC (Mink)"
                        }
                      }
                    ]
                  },
                  "postalAddressBag":{
                    "postalAddress":[
                      {
                        "postalStructuredAddress":{
                          "addressLineText":[
                            {
                              "value":"P O Box 351240"
                            }
                          ],
                          "cityName":"Westminster",
                          "geographicRegionName":[
                            {
                              "value":"CO"
                            }
                          ],
                          "countryCode":"US",
                          "postalCode":"80035"
                        }
                      }
                    ]
                  }
                },
                {
                  "value":"133517"
                }
              ]
            }
          ]
        },
        "groupArtUnitNumber":{
          "value":"3771",
          "electronicText":"3771"
        },
        "applicationConfirmationNumber":"7897",
        "applicantFileReference":"FP01.P035 SST02US",
        "priorityClaimBag":{
          "priorityClaim":[
            {
              "ipOfficeName":"NETHERLANDS",
              "applicationNumber":{
                "applicationNumberText":"2019179"
              },
              "filingDate":"2017-07-05",
              "sequenceNumber":"1"
            }
          ]
        },
        "patentClassificationBag":{
          "cpcClassificationBagOrIPCClassificationOrECLAClassificationBag":[
            {
              "ipOfficeCode":"US",
              "mainNationalClassification":{
                "nationalClass":"606",
                "nationalSubclass":"133000"
              }
            }
          ]
        },
        "businessEntityStatusCategory":"SMALL",
        "firstInventorToFileIndicator":"true",
        "inventionTitle":{
          "content":[
            "Hair removal device for removing body hair on a body surface"
          ]
        },
        "applicationStatusCategory":"Application Dispatched from Preexam, Not Yet Docketed",
        "applicationStatusDate":"2020-05-08",
        "officialFileLocationCategory":"ELECTRONIC",
        "patentPublicationIdentification":{
          "publicationNumber":"US20200170371A1",
          "publicationDate":"2020-06-04"
        },
        "relatedDocumentData":{
          "parentDocumentDataOrChildDocumentData":[
            {
              "descriptionText":"This application is National Stage Entry of",
              "applicationNumberText":"PCT/NL2018/050434",
              "filingDate":"2018-07-04",
              "parentDocumentStatusCode":"Published",
              "patentNumber":""
            }
          ]
        }
      },
      "prosecutionHistoryDataBag":{
        "prosecutionHistoryData":[
          {
            "eventDate":"2020-06-05",
            "eventCode":"PG-ISSUE",
            "eventDescriptionText":"PG-Pub Issue Notification"
          },
          {
            "eventDate":"2020-05-11",
            "eventCode":"M903",
            "eventDescriptionText":"Notice of DO/EO Acceptance Mailed"
          },
          {
            "eventDate":"2020-05-11",
            "eventCode":"FLRCPT.U",
            "eventDescriptionText":"Filing Receipt - Updated"
          },
          {
            "eventDate":"2020-05-11",
            "eventCode":"MPEN",
            "eventDescriptionText":"Mail Pre-Exam Notice"
          },
          {
            "eventDate":"2020-02-26",
            "eventCode":"EML_NTR",
            "eventDescriptionText":"Email Notification"
          },
          {
            "eventDate":"2020-02-26",
            "eventCode":"EML_NTR",
            "eventDescriptionText":"Email Notification"
          },
          {
            "eventDate":"2020-02-26",
            "eventCode":"CCRDY",
            "eventDescriptionText":"Application ready for PDX access by participating foreign offices"
          },
          {
            "eventDate":"2020-01-05",
            "eventCode":"371COMP",
            "eventDescriptionText":"371 Completion Date"
          },
          {
            "eventDate":"2020-02-25",
            "eventCode":"PGPC",
            "eventDescriptionText":"Sent to Classification Contractor"
          },
          {
            "eventDate":"2020-02-25",
            "eventCode":"FTFS",
            "eventDescriptionText":"FITF set to YES - revise initial setting"
          },
          {
            "eventDate":"2020-01-02",
            "eventCode":"PTA.RFE",
            "eventDescriptionText":"Patent Term Adjustment - Ready for Examination"
          },
          {
            "eventDate":"2020-02-26",
            "eventCode":"FLRCPT.O",
            "eventDescriptionText":"Filing Receipt"
          },
          {
            "eventDate":"2020-02-26",
            "eventCode":"M903",
            "eventDescriptionText":"Notice of DO/EO Acceptance Mailed"
          },
          {
            "eventDate":"2019-12-31",
            "eventCode":"SREXR141",
            "eventDescriptionText":"PTO/SB/69-Authorize EPO Access to Search Results"
          },
          {
            "eventDate":"2019-12-31",
            "eventCode":"APPERMS",
            "eventDescriptionText":"Applicants have given acceptable permission for participating foreign "
          },
          {
            "eventDate":"2020-02-25",
            "eventCode":"SMAL",
            "eventDescriptionText":"Applicant Has Filed a Verified Statement of Small Entity Status in Compliance with 37 CFR 1.27"
          },
          {
            "eventDate":"2019-12-31",
            "eventCode":"L194",
            "eventDescriptionText":"Cleared by OIPE CSR"
          },
          {
            "eventDate":"2019-12-31",
            "eventCode":"WIDS",
            "eventDescriptionText":"Information Disclosure Statement (IDS) Filed"
          },
          {
            "eventDate":"2019-12-31",
            "eventCode":"WIDS",
            "eventDescriptionText":"Information Disclosure Statement (IDS) Filed"
          },
          {
            "eventDate":"2019-12-31",
            "eventCode":"BIG.",
            "eventDescriptionText":"ENTITY STATUS SET TO UNDISCOUNTED (INITIAL DEFAULT SETTING OR STATUS CHANGE)"
          },
          {
            "eventDate":"2019-12-31",
            "eventCode":"IEXX",
            "eventDescriptionText":"Initial Exam Team nn"
          }
        ]
      },
      "st96Version":"V3_1",
      "ipoVersion":"US_V8_0"
    },

我将 json 数据作为字典导入。但是,获取我想要检索的信息的最佳方式是什么。我应该使用 json.normalize 将其展平并转换为 Dataframe 吗?

我想专门检索“prosecutionHistoryData”中的信息。例如,对于其他专利申请,这将提供有关已发出多少审查意见的具体信息。

最终,我希望通过专利审查员交叉引用该办公室行动数据(分配给审查员时,可以在“applicantBagOrInventorBagOrOwnerBag”中找到)。

是否有任何好的资源可以解释如何清理 json 数据,以便我可以将这些信息分成单独的列?

感谢您提供的信息!这是一个审查员的例子:

   {
         "patentCaseMetadata":{
            "applicationNumberText":{
               "value":"16732312",
               "electronicText":"16732312"
            },
            "filingDate":"2020-01-01",
            "applicationTypeCategory":"Utility",
            "partyBag":{
               "applicantBagOrInventorBagOrOwnerBag":[
                  {
                     "primaryExaminerOrAssistantExaminerOrAuthorizedOfficer":[
                        {
                           "name":{
                              "personNameOrOrganizationNameOrEntityName":[
                                 {
                                    "personFullName":"ORGAD, EDAN"
                                 }
                              ]
                           }
                        }
                     ]
                  },
                  {
                     "applicant":[
                        {
                           "contactOrPublicationContact":[
                              {
                                 "name":{
                                    "personNameOrOrganizationNameOrEntityName":[
                                       {
                                          "organizationStandardName":{
                                             "content":[
                                                "Communication Systems LLC"
                                             ]
                                          }
                                       }
                                    ]
                                 },
                                 "cityName":"Santa Fe",
                                 "geographicRegionName":{
                                    "value":"NM",
                                    "geographicRegionCategory":"STATE"
                                 },
                                 "countryCode":""
                              }
                           ]
                        }
                     ]
                  }
               ]
            },
            "groupArtUnitNumber":{
               "value":"2414",
               "electronicText":"2414"
            },
            "applicationConfirmationNumber":"8996",
            "applicantFileReference":"CS1003US03",
            "patentClassificationBag":{
               "cpcClassificationBagOrIPCClassificationOrECLAClassificationBag":[
                  {
                     "ipOfficeCode":"US",
                     "mainNationalClassification":{
                        "nationalClass":"370",
                        "nationalSubclass":"329000"
                     }
                  }
               ]
            },
            "businessEntityStatusCategory":"SMALL",
            "firstInventorToFileIndicator":"true",
            "inventionTitle":{
               "content":[
                  "APPARATUSES, METHODS, AND COMPUTER-READABLE MEDIUM FOR COMMUNICATION IN A WIRELESS LOCAL AREA NETWORK"
               ]
            },
            "applicationStatusCategory":"Docketed New Case - Ready for Examination",
            "applicationStatusDate":"2020-02-07",
            "officialFileLocationCategory":"ELECTRONIC",
            "patentPublicationIdentification":{
               "publicationNumber":"US20200154403A1",
               "publicationDate":"2020-05-14"
            }
         },
         "prosecutionHistoryDataBag":{
            "prosecutionHistoryData":[
               {
                  "eventDate":"2020-05-19",
                  "eventCode":"PG-ISSUE",
                  "eventDescriptionText":"PG-Pub Issue Notification"
               }
            ]
         },
         "assignmentDataBag":{
            "assignmentData":[
               {
                  "reelNumber":"52436",
                  "frameNumber":"295",
                  "documentReceivedDate":"2020-04-20",
                  "recordedDate":"2020-04-20",
                  "mailDate":"2020-04-21",
                  "pageTotalQuantity":3,
                  "conveyanceText":"ASSIGNMENT OF ASSIGNORS INTEREST (SEE DOCUMENT FOR DETAILS).",
                  "assignorBag":{
                     "assignor":[
                        {
                           "executionDate":"2016-07-14",
                           "contactOrPublicationContact":[
                              {
                                 "name":{
                                    "personNameOrOrganizationNameOrEntityName":[
                                       {
                                          "value":"ATEFI, ALI"
                                       }
                                    ]
                                 }
                              }
                           ]
                        }
                     ]
                  },
                  "assigneeBag":{
                     "assignee":[
                        {
                           "contactOrPublicationContact":[
                              {
                                 "name":{
                                    "personNameOrOrganizationNameOrEntityName":[
                                       {
                                          "value":"COMMUNICATION SYSTEMS LLC"
                                       }
                                    ]
                                 },
                                 "postalAddressBag":{
                                    "postalAddress":[
                                       {
                                          "postalAddressText":[
                                             {
                                                "sequenceNumber":"1",
                                                "value":"530-B HARKLE ROAD"
                                             },
                                             {
                                                "sequenceNumber":"2",
                                                "value":"STE. 100"
                                             },
                                             {
                                                "sequenceNumber":"3",
                                                "value":"SANTA FE NEW MEXICO 87505"
                                             }
                                          ]
                                       }
                                    ]
                                 }
                              }
                           ]
                        }
                     ]
                  },
                  "correspondenceAddress":{
                     "partyIdentifierOrContact":[
                        {
                           "name":{
                              "personNameOrOrganizationNameOrEntityName":[
                                 {
                                    "value":"ALI ATEFI"
                                 }
                              ]
                           },
                           "postalAddressBag":{
                              "postalAddress":[
                                 {
                                    "postalAddressText":[
                                       {
                                          "sequenceNumber":"1",
                                          "value":"530-B HARKLE ROAD"
                                       },
                                       {
                                          "sequenceNumber":"2",
                                          "value":"STE. 100"
                                       },
                                       {
                                          "sequenceNumber":"3",
                                          "value":"SANTA FE, NM 87505"
                                       }
                                    ]
                                 }
                              ]
                           }
                        }
                     ]
                  },
                  "sequenceNumber":"1"
               }
            ],
            "assignmentTotalQuantity":1
         },
         "st96Version":"V3_1",
         "ipoVersion":"US_V8_0"
      },

我的解析不会超过申请人BagOrInventorBagOrOwnerBag。这是我尝试获取检查者名称的示例解析,它返回一个空数据框:

jsonpath_expression = parse('PatentBulkData[*].patentCaseMetadata.partyBag.applicantBagOrInventorBagOrOwnerBag.primaryExaminerOrAssistantExaminerOrAuthorizedOfficer.name.personNameOrOrganizationNameOrEntityName.personFullName[*]')

如果我以申请人BagOrInventorBagOrOwnerBag 结尾,我会返回一个包含正确信息的数据框 - 仅带有括号和所有其他 JSON 表示法。我是否缺少关键结构?

再次感谢!

【问题讨论】:

    标签: python json pandas data-analysis


    【解决方案1】:

    要解析或多或少复杂的 JSON 文档,您可能需要查看JSONPath“查询语言”。

    jsonpath-rw 中有一个不错的 Python 实现。由于你需要的数据是这样嵌套的

    {
      "PatentBulkData": [
        {
          "prosecutionHistoryDataBag": {
            "prosecutionHistoryData": [
              {
                "eventDate": "2020-06-05",
                "eventCode": "PG-ISSUE",
                "eventDescriptionText": "PG-Pub Issue Notification"
              },
    

    JSONPath 是

    PatentBulkData键下,获取数组的每个元素,然后是prosecutionHistoryDataBag键,然后是prosecutionHistoryData键,最后是其下的所有数组元素。

    或者

    PatentBulkData[*].prosecutionHistoryDataBag.prosecutionHistoryData[*]
    

    这就是你在 Python 中要做的事情

    import json
    
    from jsonpath_rw import jsonpath, parse
    import pandas as pd
    
    # Parse the string containing the whole JSON document
    data = json.loads(<YOUR_JSON_STRING>)
    
    jsonpath_expr = parse('PatentBulkData[*].prosecutionHistoryDataBag.prosecutionHistoryData[*]')
    
    # Extract the raw value from each matching element,
    # i.e. every element of the JSON array
    matches = [match.value for match in jsonpath_expr.find(data)]
    
    # Create dataframe from the list of dictionaries
    df = pd.DataFrame.from_records(matches)
    

    结果:

    | eventDate   | eventCode   | eventDescriptionText              |
    |-------------|:------------|:----------------------------------|
    | 2020-06-05  | PG-ISSUE    | PG-Pub Issue Notification         |
    | 2020-05-11  | M903        | Notice of DO/EO Acceptance Mailed |
    | 2020-05-11  | FLRCPT.U    | Filing Receipt - Updated          |
    | 2020-05-11  | MPEN        | Mail Pre-Exam Notice              |
    | 2020-02-26  | EML_NTR     | Email Notification                |
    

    编辑

    对于审查员查询,您需要注意嵌套数组。每次获取树中的一个数组时,都需要获取一个([0][1] 等)或数组中的所有元素([*]):

    examiner_expr = parse(
        "PatentBulkData[*].patentCaseMetadata.partyBag"
        ".applicantBagOrInventorBagOrOwnerBag[*]"
        ".primaryExaminerOrAssistantExaminerOrAuthorizedOfficer[*]"
        ".name.personNameOrOrganizationNameOrEntityName[*]"
        ".personFullName"
    )
    [match.value for match in examiner_expr.find(data)]                                                                                                  
    # ['ORGAD, EDAN']
    

    【讨论】:

      猜你喜欢
      • 1970-01-01
      • 1970-01-01
      • 1970-01-01
      • 2015-12-18
      • 2021-08-07
      • 1970-01-01
      • 2010-09-26
      • 2018-11-02
      • 2013-07-29
      相关资源
      最近更新 更多