从 api 获取数据并将数据插入 mysql 耗时太长

Question

我有这段代码，我想用 github api 提取 github 用户信息并将所有信息插入到 mysql 表中，但是对于单个用户来说，插入到 mysql 中需要花费近 30-40 分钟的时间来获取数据，而且仅用于有关我想要的回购的信息，所以想添加有关可能需要一天时间的提交的信息，请建议我应该怎么做才能提高代码的速度和插入表中代码很大，很抱歉，但我想展示整个代码，所以我可以得到有关整体代码的建议。

connection = mysql.connector.connect(host='host',
                             user='root',
                             password='pas',
                             database='db',)

cursor = connection.cursor()

#user name--------------------------------------------------------------------------
username = 'waldekmastykarz'
#-----------------------------------------------------------------------------------
token = os.getenv('GITHUB_TOKEN', '-----------githubtoken---------')
params = {}
headers = {'Authorization': f'token {token}'}
data = requests.get('https://api.github.com/users/' +
                    username, headers=headers, params=params)
data = data.json()


#initial details ----------------------------------------------------------------
initial_details = {}
initial_details['username'] = username
initial_details['name'] = data['name']
initial_details['email'] = data['email']
initial_details['id'] = data['id']
initial_details['location'] = data['location']
initial_details['followers'] = data['followers']
initial_details['following'] = data['following']


#total details------------------------------------------------------------------
Total_commits_of_user = 0
total_repos = 0
total_stars = 0


#total repos -------------------------------------------------------------------
url = data['repos_url']
page_no = 1
repos_data = []
while (True):
    response = requests.get(url, headers=headers, params={"per_page": 100})
    response = response.json()
    repos_data = repos_data + response
    total_repos = len(response)+total_repos
    repos_fetched = len(response)
    if (repos_fetched == 100):
        page_no = page_no + 1
        url = data['repos_url'] + '?page=' + str(page_no)
    else:
        break

print(total_repos)

#get all repos info ---------------------------------------------------------------
all_repo_data = {}
for i, repo in enumerate(repos_data):
    all_repo_data[(repo['name'])] = {}
    all_repo_data[(repo['name'])]['name'] = (repo['name'])
    all_repo_data[(repo['name'])]['id'] = (repo['id'])
    all_repo_data[(repo['name'])]['description'] = repo['description']
    all_repo_data[(repo['name'])]['created_at'] = repo['created_at']
    all_repo_data[(repo['name'])]['updated_at'] = repo['updated_at']
    all_repo_data[(repo['name'])]['owner'] = repo['owner']['login']
    all_repo_data[(repo['name'])]['stars'] = repo['stargazers_count']
    all_repo_data[(repo['name'])]['url'] = repo['url']
    all_repo_data[(repo['name'])
                  ]['commits_url'] = repo['commits_url'].split("{")[0]
    all_repo_data[(repo['name'])]['languages'] = (repo['url'] + '/languages')
    all_repo_data[(repo['name'])]['contributors'] = repo['contributors_url']

    #total stars---------------------------------------------------------------------------------------
    total_stars = all_repo_data[(repo['name'])]['stars']+total_stars

    #corect format for date time ----------------------------------------------------------------------
    d1 = datetime.datetime.strptime(
        all_repo_data[(repo['name'])]['updated_at'], "%Y-%m-%dT%H:%M:%SZ")
    d2 = datetime.datetime.strptime(
        all_repo_data[(repo['name'])]['created_at'], "%Y-%m-%dT%H:%M:%SZ")
    new_format = "%Y-%m-%d"
    d1.strftime(new_format)
    d2.strftime(new_format)
    d1.date()
    d2.date()
    all_repo_data[(repo['name'])]['created_at'] = d2
    all_repo_data[(repo['name'])]['updated_at'] = d1


#total commits----------------------------------------------------------------------------------------------
    total_commit = 0
    url = all_repo_data[(repo['name'])]['commits_url']
    page_no = 1
    while (True):
        response = requests.get(url, headers=headers, params={"per_page": 100})
        response = response.json()
        total_commit = len(response)+total_commit
        commit_fetched = len(response)
        if (commit_fetched == 100):
            page_no = page_no + 1
            url = all_repo_data[(repo['name'])]['commits_url'] + \
                '?page=' + str(page_no)
        else:
            break
    #total commits -------------------------------------------------------------------------------------
    all_repo_data[(repo['name'])]['total_commits'] = total_commit


# commit_by_user ----------------------------------------------------------------------------------------
    commit_by_user = 0
    url = all_repo_data[(repo['name'])]['commits_url']
    page_no = 1
    while (True):
        response = requests.get(url, headers=headers, params={
                                "author": username, "per_page": 100})
        response = response.json()
        commit_by_user = len(response)+commit_by_user
        commit_by_user_fetched = len(response)
        if (commit_by_user_fetched == 100):
            page_no = page_no + 1
            url = all_repo_data[(repo['name'])]['commits_url'] + \
                '?page=' + str(page_no)
        else:
            break
    #total commits by user---------------------------------------------------------------------------------
    all_repo_data[(repo['name'])]['commit_by_user'] = commit_by_user
    Total_commits_of_user = commit_by_user+Total_commits_of_user


#contributors----------------------------------------------------------------------------------------------
    total_contributors = 0
    url = all_repo_data[(repo['name'])]['contributors']
    page_no = 1
    while (True):
        response = requests.get(url, headers=headers, params={"per_page": 100})
        response = response.json()
        total_contributors = len(response)+total_contributors
        contributors_fetched = len(response)
        if (contributors_fetched == 100):
            page_no = page_no + 1
            url = all_repo_data[(repo['name'])]['contributors'] + \
                '?page=' + str(page_no)
        else:
            break
    #total contributors---------------------------------------------------------------------------------------
    all_repo_data[(repo['name'])]['contributors'] = total_contributors


##commit dates-------------------------------------------------------------------------------------------------
    total_commit = 0
    commit_data = {}
    url = all_repo_data[(repo['name'])]['commits_url']
    page_no = 1
    while (True):
        response = requests.get(url, headers=headers, params={"per_page": 100})
        response = response.json()
        for commits in response:
            d1 = datetime.datetime.strptime(
                commits['commit']['author']['date'], "%Y-%m-%dT%H:%M:%SZ")
            new_format = "%Y-%m-%d"
            d1.strftime(new_format)
            d1.date()
            # print(all_repo_data[(repo['name'])]['name'],d1 )
            commit_fetched = len(response)
        if (commit_fetched == 100):
            page_no = page_no + 1
            url = all_repo_data[(repo['name'])]['commits_url'] + \
                '?page=' + str(page_no)
        else:
            break

    sql = "INSERT INTO user_repo (user_id, github_id, repo_id, user_connection, repo_name, contributors, total_commits, commit_by_user, created_date, last_commit) VALUES (  %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
    cursor.execute(sql, (
                initial_details['id'],
                username,
                all_repo_data[(repo['name'])]['id'],
                all_repo_data[(repo['name'])]['owner'],
                all_repo_data[(repo['name'])]['name'],
                all_repo_data[(repo['name'])]['contributors'],
                all_repo_data[(repo['name'])]['total_commits'],
                all_repo_data[(repo['name'])]['commit_by_user'],
                all_repo_data[(repo['name'])]['created_at'],
                all_repo_data[(repo['name'])]['updated_at'],
            ))
        
connection.commit()

Answer 1

可以有两种解决方案：

transcations

MySQL 有一个自动提交查询的标志。意味着当您发送插入/更新/删除查询时，它将自动提交它们。它减慢了速度。你应该做的是取消设置autocommit。

SET autocommit = 0;

现在你发送所有inserts 然后在最后提交它们，当你的所有查询都这样完成时：

START TRANSACTION
INSERT INTO `the_table`(field_1, filed_2, ..., field_n) VALUES (value_1, value_2, ..., value_n);
INSERT INTO `the_table`(field_1, filed_2, ..., field_n) VALUES (value_1, value_2, ..., value_n);
...
INSERT INTO `the_table`(field_1, filed_2, ..., field_n) VALUES (value_1, value_2, ..., value_n);
COMMIT

见：https://stackoverflow.com/a/7978511/2681662

多个inserts

一次插入多行是另一种选择。

INSERT INTO `the_table`
    (field_1, field_2, ..., field_n)
VALUES
    (value_1, value_2, ..., value_n),
    (value_1, value_2, ..., value_n),
    ...
    (value_1, value_2, ..., value_n);

见：https://stackoverflow.com/a/6889087/2681662

但是这有其他限制，例如服务器接受的最大允许数据大小。