如何从多个 JSON 对象中选择一个并在 python 中导航其层次结构

Question

我有一个包含多个 javascript 元素的网页。我只想访问一个，名为SOURCE.pdp.propertyJSON，并以 PYTHONIC 方式访问属性。

HTML 源代码的编辑（为了便于阅读）版本如下；以下是我的python代码。

任何指针将不胜感激！

<script type="text/javascript">
SOURCE = SOURCE || {};
SOURCE.pdp = SOURCE.pdp || {};
SOURCE.pdp.propertyJSON = {
  "neighborhood": "Westwood",
  "neighborhoodId": 7187,
  "zipCode": "90024",
  "city": "Los Angeles",
  "county": "Los Angeles",
  "countyFIPS": "06037",
  "stateCode": "CA",
  "stateName": "California",
  "type": "CONDO",
  "typeDisplay": "Condo",
  "numBedrooms": "2",
  "numBathrooms": 2,
  "numFullBathrooms": 2,
  "numBeds": 2,
  "indexSource": "Assessor",
  "isForeclosure": false,
  "isOpaqueBAL": false,
  "foreclosureStatus": "",
  "isSrpFeatured": false,
  "price": null,
  "sqft": 1321,
  "formattedBedAndBath": "2bd, 2 full ba",
  "formattedSqft": "1,321 sqft"
}
pdp_location_data = {
  "neighborhood": {
    "locationId": "87308",
    "name": "Westwood",
    "locationType": "neighborhood",
    "altId": "7187"
  },
  "state": {
    "locationId": "5",
    "name": "California",
    "locationType": "state",
    "altId": "CA"
  },
  "county": {
    "locationId": "57",
    "name": "Los Angeles County",
    "locationType": "county",
    "altId": "06037"
  },
  "city": {
    "locationId": "22637",
    "name": "Los Angeles",
    "locationType": "city",
    "altId": "4396"
  },
  "zipCode": {
    "locationId": "76090",
    "name": "90024",
    "locationType": "zipCode",
    "altName": "90024",
    "altId": "90024"
  }
};
SOURCE.pdp.isCountySupportsValuation = true;
SOURCE.pdp.isInHighDemandRegion = false;
var _SPANLONG = pdp_location_data.longitude;
var _SPANLAT = pdp_location_data.latitude;
var _CENLONG = pdp_location_data.longitude;
var _CENLAT = pdp_location_data.latitude;
</script>

小心丑陋的蟒蛇！

  from bs4 import BeautifulSoup as bsoup
  import requests as rq

  url = 'https://www.SOURCE.com'
  source_code = rq.get(url).text
  soupcon = bsoup(source_code,"html.parser")
  souper = soupcon.find_all('script', {'type': 'text/javascript'})

  for line in souper:
      if format(line).find('SOURCE.pdp.propertyJSON') != -1:
            parts = format(line).split(',')

  for var in parts:
        if var.find('zipCode') != -1:
            zipCode = var.split(':')[1].strip('"')
        elif var.find('numBathrooms') != -1:
            numBathrooms = var.split(':')[1].strip('"')

如您所见，我目前正在访问我想要的 JS 对象，方法是查找所有 text/javascript 类型的脚本元素，遍历它们以找到包含我想要的对象的脚本，然后拆分整个通过 JS 分隔符 ',' 编写脚本，并通过搜索我的关键词来识别 JS 对象的元素。不是一个理想的解决方案。

Answer 1

您可以使用 json.loads 将数据作为字典加载：

from bs4 import BeautifulSoup as bsoup
import re
from json import loads
source = """<script type="text/javascript">  SOURCE = SOURCE || {};
  SOURCE.pdp = SOURCE.pdp || {};
  SOURCE.pdp.propertyJSON = {    "neighborhood": "Westwood",    "neighborhoodId": 7187,    "zipCode": "90024",    "city": "Los Angeles",    "county": "Los Angeles",    "countyFIPS": "06037",    "stateCode": "CA",    "stateName": "California",    "type": "CONDO",    "typeDisplay": "Condo",    "numBedrooms": "2",    "numBathrooms": 2,    "numFullBathrooms": 2,    "numBeds": 2,    "indexSource": "Assessor",    "isForeclosure": false,    "isOpaqueBAL": false,    "foreclosureStatus": "",    "isSrpFeatured": false,    "price": null,    "sqft": 1321,    "formattedBedAndBath": "2bd, 2 full ba",    "formattedSqft": "1,321 sqft"  }  pdp_location_data = {    "neighborhood": {      "locationId": "87308",      "name": "Westwood",      "locationType": "neighborhood",      "altId": "7187"    },    "state": {      "locationId": "5",      "name": "California",      "locationType": "state",      "altId": "CA"    },    "county": {      "locationId": "57",      "name": "Los Angeles County",      "locationType": "county",      "altId": "06037"    },    "city": {      "locationId": "22637",      "name": "Los Angeles",      "locationType": "city",      "altId": "4396"    },    "zipCode": {      "locationId": "76090",      "name": "90024",      "locationType": "zipCode",      "altName": "90024",      "altId": "90024"    }  };
   SOURCE.pdp.isCountySupportsValuation = true;
    SOURCE.pdp.isInHighDemandRegion = false;
    var _SPANLONG = pdp_location_data.longitude;
    var _SPANLAT = pdp_location_data.latitude;
    var _CENLONG = pdp_location_data.longitude;
   var _CENLAT = pdp_location_data.latitude;  </script>"""
soup = bsoup(source,"html.parser")


json_re = re.compile("SOURCE\.pdp\.propertyJSON\s+=\s+(\{.*\})\s+pdp_location_data")
scr = soup.find("script", text=re.compile("SOURCE.pdp.propertyJSON")).text
js_raw = json_re.search(scr).group(1)
json_dict = loads(js_raw)

这会给你：

{u'numBeds': 2, u'neighborhood': u'Westwood', u'stateName': u'California', u'numFullBathrooms': 2, u'indexSource': u'Assessor', u'countyFIPS': u'06037', u'city': u'Los Angeles', u'isSrpFeatured': False, u'type': u'CONDO', u'formattedSqft': u'1,321 sqft', u'isOpaqueBAL': False, u'price': None, u'zipCode': u'90024', u'numBedrooms': u'2', u'neighborhoodId': 7187, u'county': u'Los Angeles', u'formattedBedAndBath': u'2bd, 2 full ba', u'sqft': 1321, u'numBathrooms': 2, u'stateCode': u'CA', u'isForeclosure': False, u'typeDisplay': u'Condo', u'foreclosureStatus': u''}

如果您想要 pdp_location_data json，只需应用完全相同的逻辑。