通过多个 url 抓取网页

Question

我有我想要的内容的代码，但是我想运行到目前为止已经玩过的所有 gameId，而不仅仅是 URL 中的那个。我想更改 2017020001 并使其通过 2017021272 或直到赛季结束，我相信大约是 1272。如何使用下面的代码来完成？

import csv
import requests
import os

req = requests.get('https://statsapi.web.nhl.com/api/v1/game/2017020001/feed/live?site=en_nhl')
data = req.json()

my_data = []
pk = data['gameData']['game']['pk']
for item in data['liveData']['plays']['allPlays']:
    players = item.get('players')
    if players:
        player_a = players[0]['player']['fullName'] if len(players) > 0 else None
        player_b = players[1]['player']['fullName'] if len(players) > 1 else None
    else:
        player_a, player_b = None, None
    event = item['result']['event']
    time = item['about']['periodTime']
    triCode = item.get('team', {}).get('triCode')
    coordinates_x, coordinates_y = item['coordinates'].get('x'), item['coordinates'].get('y')
    my_data.append([pk, player_a, player_b, event, time, triCode, coordinates_x, coordinates_y])

headers = ["pk", "player_a", "player_b", "event", "time", "triCode", "coordinates_x", "coordinates_y"]

with open("NHL_2017020001.csv", "a", newline='') as f:
    writer = csv.writer(f)
    writer.writerow(headers)
    writer.writerows(my_data)
f.close()

Answer 1

如果游戏 id 是按顺序编号的，那么就像将所有代码嵌套在一个循环中一样简单部分会改变：

import csv
import requests
import os

for i in range(1, 1273):
    url = 'https://statsapi.web.nhl.com/api/v1/game/201702{:04d}/feed/live?site=en_nhl'.format(i)
    req = requests.get(url)
    req.raise_for_status()
    data = req.json()
    my_data = []
    pk = data['gameData']['game']['pk']
    for item in data['liveData']['plays']['allPlays']:
        players = item.get('players')
        if players:
            player_a = players[0]['player']['fullName'] if len(players) > 0 else None
            player_b = players[1]['player']['fullName'] if len(players) > 1 else None
        else:
            player_a, player_b = None, None
            event = item['result']['event']
            time = item['about']['periodTime']
            triCode = item.get('team', {}).get('triCode')
        coordinates_x, coordinates_y = item['coordinates'].get('x'), item['coordinates'].get('y')
        my_data.append([pk, player_a, player_b, event, time, triCode, coordinates_x, coordinates_y])

        headers = ["pk", "player_a", "player_b", "event", "time", "triCode", "coordinates_x", "coordinates_y"]

    with open("NHL_201702{:04d}.csv".format(i), "a", newline='') as f:
        writer = csv.writer(f)
        writer.writerow(headers)
        writer.writerows(my_data)

最后一个更正是使用with ... as 使您无需显式关闭文件。 您可以找到有关使用 str.format() here的更多信息

Answer 2

您应该使用 for 循环遍历您的代码

这样的事情应该可以工作：

import csv
import requests
import os

for x in range(2017020001, 2017021273):
    req = requests.get('https://statsapi.web.nhl.com/api/v1/game/%s/feed/live?site=en_nhl' % x)
    data = req.json()

    my_data = []
    pk = data['gameData']['game']['pk']
    for item in data['liveData']['plays']['allPlays']:
        players = item.get('players')
        if players:
            player_a = players[0]['player']['fullName'] if len(players) > 0 else None
            player_b = players[1]['player']['fullName'] if len(players) > 1 else None
        else:
            player_a, player_b = None, None
        event = item['result']['event']
        time = item['about']['periodTime']
        triCode = item.get('team', {}).get('triCode')
        coordinates_x, coordinates_y = item['coordinates'].get('x'), item['coordinates'].get('y')
        my_data.append([pk, player_a, player_b, event, time, triCode, coordinates_x, coordinates_y])

    headers = ["pk", "player_a", "player_b", "event", "time", "triCode", "coordinates_x", "coordinates_y"]

    with open("NHL_2017020001.csv", "a", newline='') as f:
        writer = csv.writer(f)
        writer.writerow(headers)
        writer.writerows(my_data)
    f.close()