Python DataFrame 到 MYSQL：TypeError：格式字符串的参数不足

Question

玩了 14 个小时（我是初学者）

从一个数据库表中提取数据以在 yahoo 上搜索该股票代码上的所有数据，然后“打算”上传它。

我最初将它作为 panda df 但得到“模棱两可的错误”，所以我现在再次将它作为 [] 。新错误。我绞尽脑汁:(但是，如果我把它留空，它确实有效。

    from __future__ import print_function
import yfinance as yf
import pandas as pd
import datetime
import warnings
import MySQLdb as mdb
import requests
import numpy as np
import MySQLdb as mdb
import requests


# Obtain a database connection to the MySQL instance
con = mdb.connect("localhost","sec_user","","securities_master")


def obtain_list_of_db_tickers():
    """
    Obtains a list of the ticker symbols in the database.
    """
    with con:
        cur = con.cursor()
        cur.execute("SELECT id, ticker FROM symbol")
        data = cur.fetchall()
        print(data)
        return [(d[0], d[1]) for d in data]

def get_daily_historic_data_yahoo(ticker):
    blow = yf.download(ticker)
    data = []
    data.append(yf.download(ticker).reset_index())
    return data

def insert_daily_data_into_db(data_vendor_id, symbol_id, daily_data):
    '''
    Takes a list of tuples of daily data and adds it to the MySQL database.
    Appends the vendor ID and symbol ID to the data.

    daily_data: List of tuples of the OHLC data (with adj_close and volume)
    '''

    # Create the time now
    now = datetime.datetime.utcnow()

    df = pd.DataFrame(data=daily_data[0])
    df.insert(0, 'data_vendor_id', data_vendor_id)
    df.insert(1, 'symbol_id', symbol_id)
    df.insert(3, 'created_date', now)
    df.insert(4, 'last_updated_date', now)
    daily_data = []
    daily_data.append(df)

    #df = daily_data

       # Amend the data to include the vendor ID and symbol ID


    # Connect to the MySQL instance
    db_host = 'localhost'
    db_user = ''
    db_pass = ''
    db_name = 'securities_master'
    con = mdb.connect("localhost", "sec_user", "", "securities_master"
                      # host=db_host, user=db_user, passwd=db_pass, db=db_name
                      )

    try:
        mdb.connect
    # If connection is not successful
    except:
        print("Can't connect to database")
        return 0

    # If Connection Is Successful
    print("Connected")


    final_str = """INSERT INTO daily_price (data_vendor_id, symbol_id, price_date, created_date,
    last_updated_date, open_price, high_price, low_price, close_price, volume, adj_close_price) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"""

    with con:
        cur = con.cursor()
        cur.executemany(final_str, daily_data)
        con.commit()

if __name__ == "__main__":
    # This ignores the warnings regarding Data Truncation
    # from the Yahoo precision to Decimal(19,4) datatypes
    warnings.filterwarnings('ignore')

    # Loop over the tickers and insert the daily historical
    # data into the database
    tickers = obtain_list_of_db_tickers()
    lentickers = len(tickers)
    for i, t in enumerate(tickers):
        print(
            "Adding data for %s: %s out of %s" %
            (t[1], i+1, lentickers)
        )
        yf_data = get_daily_historic_data_yahoo(t[1])
        insert_daily_data_into_db('1', t[0], yf_data)
    print("Successfully added Yahoo Finance pricing data to DB.")

错误

    Traceback (most recent call last):
  File "/home/quant/price_retrieval.py", line 106, in <module>
    insert_daily_data_into_db('1', t[0], yf_data)
  File "/home/quant/price_retrieval.py", line 88, in insert_daily_data_into_db

        cur.executemany(final_str, daily_data)
      File "/home/quant/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 230, in executemany
        return self._do_execute_many(
      File "/home/quant/.local/lib/python3.8/site-packages/MySQLdb/cursors.py", line 255, in _do_execute_many
        v = values % escape(next(args), conn)
    TypeError: not enough arguments for format string

Answer 1

我不是数据科学家，所以可能有一种更优雅的方法可以直接使用 pandas 修复它。但我通常使用 MySQL（以及任何 SQL 驱动程序）的方式是给它提供 python 元组列表。

如果您使用 for row in df.itertuples(): 解析 pandas 数据帧的每一行并仔细制作每个元组 - 确保类型与 SQL 表匹配，那么一切都应该有效；）

例子：

def insert_daily_data_into_db(data_vendor_id, symbol_id, daily_data):
    '''
    Takes a list of tuples of daily data and adds it to the MySQL database.
    Appends the vendor ID and symbol ID to the data.

    daily_data: List of tuples of the OHLC data (with adj_close and volume)
    '''

    # Create the time now
    now = datetime.datetime.utcnow()

    df = pd.DataFrame(data=daily_data[0])
    daily_data = []
    created_date = now
    last_updated_date = now
    for row in df.itertuples():
        _index = row[0]  # discard
        date = row[1]
        open = row[2]
        high = row[3]
        low = row[4]
        close = row[5]
        adj_close_price = row[6]
        volume = row[7]
        daily_data.append((int(data_vendor_id), symbol_id, date, created_date, last_updated_date, open, high, low, close, volume, adj_close_price))

    # Connect to the MySQL instance
    con = mdb.connect(host="localhost", user="user", password="yourpassword",
        db="yourdbname", port=3306)

    final_str = """
        INSERT INTO daily_price (data_vendor_id, symbol_id, price_date, created_date,
        last_updated_date, open_price, high_price, low_price, close_price, volume, adj_close_price) 
        VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
    """

    with con:
        cur = con.cursor()
        cur.executemany(final_str, daily_data)
        con.commit()

我尽量不要过多地篡改您现有的代码。足以让它发挥作用。

我认为那里发生的事情是，从技术上讲，您向它传递了一个熊猫数据框列表，列表中只有一个熊猫数据框。相反，您想要的是一个包含 11 个字段的元组列表，每个元组要解包。

也许您的意思是直接传递数据框，即不包含在列表中，但我仍然认为这不正确，因为 1）数据框中有一个“索引”列会给出错误的结果 2）您d 需要在数据帧上调用一些方法来仅检索值（而不是列的标题）并将其转换为正确的元组列表。这可能非常可行，但我会留给你去发现。

我还假设您的表架构是这样的：

CREATE TABLE IF NOT EXISTS daily_price (
    data_vendor_id INT,
    symbol_id INT,
    price_date DATETIME,
    created_date DATETIME,
    last_updated_date TIMESTAMP,
    open_price VARCHAR(256),
    high_price VARCHAR(256),
    low_price VARCHAR(256),
    close_price VARCHAR(256),
    volume INT,
    adj_close_price VARCHAR(256)
);