如何使用 Python 从网站获取脚本标签变量答案

【问题标题】：How to Get Script Tag Variables From a Website using Python如何使用 Python 从网站获取脚本标签变量
【发布时间】：2019-01-17 15:44:51
【问题描述】：

我正在尝试使用 Python 在脚本标签中提取一个名为 meta 的变量。我以前用过硒来做这件事，但是硒对于我想要完成的事情来说太慢了。有没有其他方法可以做到这一点。

我尝试过使用 BeautifulSoup，但我卡住了...代码如下

这是我试图从中获取元变量的脚本标签：

<script>window.ShopifyAnalytics = window.ShopifyAnalytics || {};
window.ShopifyAnalytics.meta = window.ShopifyAnalytics.meta || {};
window.ShopifyAnalytics.meta.currency = 'USD';
var meta = {"product"{"id":2006141861957,"vendor":"Nike","type":"Sneakers","variants": [{"id":19039563677765,"price":16000,"name":"Nike React Element '87 - Light Orewood Brown \/ Laser Orange \/ Volt Glow - 4","public_title":"4","sku":"191888228157"}, {"id":19039563710533,"price":16000,"name":"Nike React Element '87 - Light Orewood Brown \/ Laser Orange \/ Volt Glow - 4.5","public_title":"4.5","sku":"191888228164"},{"id":19039563743301,"price":16000,"name":"Nike React Element '87 - Light Orewood Brown \/ Laser Orange \/ Volt Glow - 5","public_title":"5","sku":"191888228171"},{"id":19039563776069,"price":16000,"name":"Nike React Element '87 - Light Orewood Brown \/ Laser Orange \/ Volt Glow - 5.5","public_title":"5.5","sku":"191888228188"},{"id":19039563808837,"price":16000,"name":"Nike React Element '87 - Light Orewood Brown \/ Laser Orange \/ Volt Glow - 6","public_title":"6","sku":"886059750741"},{"id":19039563841605,"price":16000,"name":"Nike React Element '87 - Light Orewood Brown \/ Laser Orange \/ Volt Glow - 6.5","public_title":"6.5","sku":"886059750758"},{"id":19039563874373,"price":16000,"name":"Nike React Element '87 - Light Orewood Brown \/ Laser Orange \/ Volt Glow - 7","public_title":"7","sku":"886059750765"},{"id":19039563907141,"price":16000,"name":"Nike React Element '87 - Light Orewood Brown \/ Laser Orange \/ Volt Glow - 7.5","public_title":"7.5","sku":"886059750772"},{"id":19039563939909,"price":16000,"name":"Nike React Element '87 - Light Orewood Brown \/ Laser Orange \/ Volt Glow - 8","public_title":"8","sku":"886059750789"},{"id":19039563972677,"price":16000,"name":"Nike React Element '87 - Light Orewood Brown \/ Laser Orange \/ Volt Glow - 8.5","public_title":"8.5","sku":"886059750796"},{"id":19039564005445,"price":16000,"name":"Nike React Element '87 - Light Orewood Brown \/ Laser Orange \/ Volt Glow - 9","public_title":"9","sku":"886059750802"},{"id":19039564038213,"price":16000,"name":"Nike React Element '87 - Light Orewood Brown \/ Laser Orange \/ Volt Glow - 9.5","public_title":"9.5","sku":"886059750819"},{"id":19039564070981,"price":16000,"name":"Nike React Element '87 - Light Orewood Brown \/ Laser Orange \/ Volt Glow - 10","public_title":"10","sku":"886059750826"},{"id":19039564103749,"price":16000,"name":"Nike React Element '87 - Light Orewood Brown \/ Laser Orange \/ Volt Glow - 10.5","public_title":"10.5","sku":"886059751038"},{"id":19039564136517,"price":16000,"name":"Nike React Element '87 - Light Orewood Brown \/ Laser Orange \/ Volt Glow - 11","public_title":"11","sku":"886059751045"},{"id":19039564169285,"price":16000,"name":"Nike React Element '87 - Light Orewood Brown \/ Laser Orange \/ Volt Glow - 11.5","public_title":"11.5","sku":"886059751052"},{"id":19039564202053,"price":16000,"name":"Nike React Element '87 - Light Orewood Brown \/ Laser Orange \/ Volt Glow - 12","public_title":"12","sku":"886059751069"},{"id":19039564234821,"price":16000,"name":"Nike React Element '87 - Light Orewood Brown \/ Laser Orange \/ Volt Glow - 12.5","public_title":"12.5","sku":"886059751076"},{"id":19039564267589,"price":16000,"name":"Nike React Element '87 - Light Orewood Brown \/ Laser Orange \/ Volt Glow - 13","public_title":"13","sku":"886059752448"}]},"page":{"pageType":"product","resourceType":"product","resourceId":2006141861957}};
for (var attr in meta) {
  window.ShopifyAnalytics.meta[attr] = meta[attr];
}</script>

这是我尝试过的：

bs = soup(r.text, "html.parser")
scripts = bs.findAll('script')
for s in scripts:
    if 'var meta' in s.text:
        print(s)

这会像字符串一样返回：

<script>window.ShopifyAnalytics = window.ShopifyAnalytics || {};
window.ShopifyAnalytics.meta = window.ShopifyAnalytics.meta || {};
window.ShopifyAnalytics.meta.currency = 'USD';
var meta = {"product":{"id":2008798101573,"vendor":"Adidas","type":"Sneakers","variants":[{"id":19054898249797,"price":13000,"name":"adidas Originals WMNS Falcon - Ash Pearl \/ Off White - 3","public_title":"3","sku":"98197426"},{"id":19054898282565,"price":13000,"name":"adidas Originals WMNS Falcon - Ash Pearl \/ Off White - 3.5","public_title":"3.5","sku":"98197427"},{"id":19054898315333,"price":13000,"name":"adidas Originals WMNS Falcon - Ash Pearl \/ Off White - 4","public_title":"4","sku":"98197428"},{"id":19054898348101,"price":13000,"name":"adidas Originals WMNS Falcon - Ash Pearl \/ Off White - 4.5","public_title":"4.5","sku":"98197429"},{"id":19054898380869,"price":13000,"name":"adidas Originals WMNS Falcon - Ash Pearl \/ Off White - 5","public_title":"5","sku":"191525030983"},{"id":19054898413637,"price":13000,"name":"adidas Originals WMNS Falcon - Ash Pearl \/ Off White - 5.5","public_title":"5.5","sku":"191525030952"},{"id":19054898446405,"price":13000,"name":"adidas Originals WMNS Falcon - Ash Pearl \/ Off White - 6","public_title":"6","sku":"191525030938"},{"id":19054898479173,"price":13000,"name":"adidas Originals WMNS Falcon - Ash Pearl \/ Off White - 6.5","public_title":"6.5","sku":"191525030914"},{"id":19054898511941,"price":13000,"name":"adidas Originals WMNS Falcon - Ash Pearl \/ Off White - 7","public_title":"7","sku":"191525030907"},{"id":19054898544709,"price":13000,"name":"adidas Originals WMNS Falcon - Ash Pearl \/ Off White - 7.5","public_title":"7.5","sku":"191525030976"},{"id":19054898577477,"price":13000,"name":"adidas Originals WMNS Falcon - Ash Pearl \/ Off White - 8","public_title":"8","sku":"191525031010"},{"id":19054898610245,"price":13000,"name":"adidas Originals WMNS Falcon - Ash Pearl \/ Off White - 8.5","public_title":"8.5","sku":"191525030990"},{"id":19054898643013,"price":13000,"name":"adidas Originals WMNS Falcon - Ash Pearl \/ Off White - 9","public_title":"9","sku":"191525031027"},{"id":19054898675781,"price":13000,"name":"adidas Originals WMNS Falcon - Ash Pearl \/ Off White - 9.5","public_title":"9.5","sku":"191525030921"},{"id":19054898708549,"price":13000,"name":"adidas Originals WMNS Falcon - Ash Pearl \/ Off White - 10","public_title":"10","sku":"191525030969"},{"id":19054898741317,"price":13000,"name":"adidas Originals WMNS Falcon - Ash Pearl \/ Off White - 10.5","public_title":"10.5","sku":"191525030945"},{"id":19054898774085,"price":13000,"name":"adidas Originals WMNS Falcon - Ash Pearl \/ Off White - 11","public_title":"11","sku":"191525031003"},{"id":19054898806853,"price":13000,"name":"adidas Originals WMNS Falcon - Ash Pearl \/ Off White - 11.5","public_title":"11.5","sku":"98197443"},{"id":19054898839621,"price":13000,"name":"adidas Originals WMNS Falcon - Ash Pearl \/ Off White - 12","public_title":"12","sku":"98197444"},{"id":19054898872389,"price":13000,"name":"adidas Originals WMNS Falcon - Ash Pearl \/ Off White - 12.5","public_title":"12.5","sku":"98197445"},{"id":19054898905157,"price":13000,"name":"adidas Originals WMNS Falcon - Ash Pearl \/ Off White - 13","public_title":"13","sku":"98197446"},{"id":19054898937925,"price":13000,"name":"adidas Originals WMNS Falcon - Ash Pearl \/ Off White - 13.5","public_title":"13.5","sku":"98197447"},{"id":19054898970693,"price":13000,"name":"adidas Originals WMNS Falcon - Ash Pearl \/ Off White - 14","public_title":"14","sku":"98197448"},{"id":19054899003461,"price":13000,"name":"adidas Originals WMNS Falcon - Ash Pearl \/ Off White - 14.5","public_title":"14.5","sku":"98197449"},{"id":19054899036229,"price":13000,"name":"adidas Originals WMNS Falcon - Ash Pearl \/ Off White - 15","public_title":"15","sku":"98197450"}]},"page":{"pageType":"product","resourceType":"product","resourceId":2008798101573}};
for (var attr in meta) {
 window.ShopifyAnalytics.meta[attr] = meta[attr];
}</script>

我想要做的是返回元变量，以便我能够从元变量中提取数据，例如，“产品”的所有“id”

【问题讨论】：

标签： python web-scraping beautifulsoup html-parsing

【解决方案1】：

没有完整的代码来获得该输出，我在这里猜测了一下。但是如果你可以抓取文本，那么只要使用json，你应该可以得到那个数据。

因此，我将使用您之前的一个问题的示例，其格式基本相同：

实际上并没有什么不同，除了我们将提取字符串中可以利用json.loads() 的部分。然后你有一个很好的 json 类型的字典和列表，你可以提取产品的 id：

import requests
import bs4
import json

url = 'https://packershoes.com/products/copy-of-adidas-predator-accelerator-trainer'
r = requests.get(url)

bs = bs4.BeautifulSoup(r.text, "html.parser")
scripts = bs.find_all('script')
jsonObj = None

for s in scripts:
    if 'var meta' in s.text:
        script = s.text
        script = script.split('var meta = ')[1]
        script = script.split(';\nfor (var attr in meta)')[0]

        jsonStr = script
        jsonObj = json.loads(jsonStr)

for value in jsonObj['product']['variants']:
    print ('ID: '+ str(value['id']))

输出：

ID: 14189113049177
ID: 14189122912345
ID: 14139452129369
ID: 14139452194905
ID: 14139452227673
ID: 14139452293209
ID: 14139452325977
ID: 14139452391513
ID: 14139452424281
ID: 14189321715801
ID: 14139452457049
ID: 14139909505113

【讨论】：