【发布时间】：2020-03-27 22:13:22
【问题描述】：
我已经创建了具有此列数据类型的表..
create_table_sql = "create table "+table_name+" (\
event_date varchar(100),\
event_timestamp BIGINT,\
event_name varchar(65535),\
event_params varchar(65535),\
event_previous_timestamp BIGINT,\
event_value_in_usd real,\
event_bundle_sequence_id BIGINT,\
event_server_timestamp_offset BIGINT,\
user_id varchar(65535),\
user_pseudo_id varchar(65535),\
user_properties varchar(65535),\
user_first_touch_timestamp BIGINT,\
user_ltv varchar(65535),\
device varchar(65535),\
geo varchar(65535),\
app_info varchar(65535),\
traffic_source varchar(65535),\
stream_id varchar(65535),\
platform varchar(65535) )"
当我尝试使用 psycopg2 在 redshift 数据库中插入这些下面的 json 数据时，它会给出错误。
data_array = ['{"event_date":"20180625","event_timestamp":"1529948530245005","event_name":"firebase_campaign","event_params":[{"key":"medium","value":{"string_value":"(not set)"}},{"key":"firebase_event_origin","value":{"string_value":"auto"}},{"key":"source","value":{"string_value":"(not set)"}}],"event_previous_timestamp":"1529948527702005","user_pseudo_id":"ffb01b060c860317c9baba657ffe402d","user_properties":[{"key":"first_open_time","value":{"int_value":"1529949600000","set_timestamp_micros":"1529948488186000"}}],"user_first_touch_timestamp":"1529948488186000","device":{"category":"mobile","mobile_brand_name":"Samsung","mobile_model_name":"SM-A500G","mobile_marketing_name":"Galaxy A5","mobile_os_hardware_model":"SM-A500G","operating_system":"ANDROID","operating_system_version":"6.0.1","advertising_id":"44f5dd05-1f78-461a-be16-f6c4b1665920","language":"en-gb","is_limited_ad_tracking":"No","time_zone_offset_seconds":"19800"},"geo":{"continent":"Asia","country":"India","region":"Gujarat","city":"Ahmedabad"},"app_info":{"id":"app.trell","version":"3.3.8","firebase_app_id":"1:808772988865:android:8db93cca16602100","install_source":"com.android.vending"},"traffic_source":{"name":"(direct)","medium":"(none)","source":"(direct)"},"stream_id":"1051639513","platform":"ANDROID"}', '{"event_date":"20180625","event_timestamp":"1529947540672001","event_name":"firebase_campaign","event_params":[{"key":"medium","value":{"string_value":"(not set)"}},{"key":"firebase_event_origin","value":{"string_value":"auto"}},{"key":"source","value":{"string_value":"(not set)"}}],"event_previous_timestamp":"1529947540484001","user_pseudo_id":"4696ce1fc56aae5c026b2d2408bde2cd","user_properties":[{"key":"first_open_time","value":{"int_value":"1529949600000","set_timestamp_micros":"1529947525415000"}}],"user_first_touch_timestamp":"1529947525415000","device":{"category":"mobile","mobile_brand_name":"Samsung","mobile_model_name":"SM-G935F","mobile_marketing_name":"Galaxy S7 Edge","mobile_os_hardware_model":"SM-G935F","operating_system":"ANDROID","operating_system_version":"8.0.0","advertising_id":"a952abd5-952f-45de-af1f-721805044eb0","language":"en-gb","is_limited_ad_tracking":"No","time_zone_offset_seconds":"19800"},"geo":{"continent":"Asia","country":"India","region":"Maharashtra","city":"Mumbai"},"app_info":{"id":"app.trell","version":"3.3.8","firebase_app_id":"1:808772988865:android:8db93cca16602100","install_source":"com.android.vending"},"traffic_source":{"name":"(direct)","medium":"(none)","source":"(direct)"},"stream_id":"1051639513","platform":"ANDROID"}']
尝试使用下面的代码插入。
df = pd.DataFrame(data_array)
df.to_sql(table_name, conn, index=False, if_exists='replace')
这是错误的堆栈跟踪。
Traceback (most recent call last):
File "/home/ubuntu/trell/env/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1226, in _execute_context
cursor, statement, parameters, context
File "/home/ubuntu/trell/env/lib/python3.6/site-packages/sqlalchemy/dialects/postgresql/psycopg2.py", line 854, in do_executemany
cursor.executemany(statement, parameters)
psycopg2.errors.StringDataRightTruncation: value too long for type character varying(256)
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "bq_to_s3_migration.py", line 232, in <module>
import_json_from_s3_to_rs(table_name, s3_bucket)
File "bq_to_s3_migration.py", line 180, in import_json_from_s3_to_rs
df.to_sql(table_name, conn, index=False, if_exists='replace')
File "/home/ubuntu/trell/env/lib/python3.6/site-packages/pandas/core/generic.py", line 2712, in to_sql
method=method,
File "/home/ubuntu/trell/env/lib/python3.6/site-packages/pandas/io/sql.py", line 518, in to_sql
method=method,
File "/home/ubuntu/trell/env/lib/python3.6/site-packages/pandas/io/sql.py", line 1320, in to_sql
table.insert(chunksize, method=method)
File "/home/ubuntu/trell/env/lib/python3.6/site-packages/pandas/io/sql.py", line 756, in insert
exec_insert(conn, keys, chunk_iter)
File "/home/ubuntu/trell/env/lib/python3.6/site-packages/pandas/io/sql.py", line 670, in _execute_insert
conn.execute(self.table.insert(), data)
File "/home/ubuntu/trell/env/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 982, in execute
return meth(self, multiparams, params)
File "/home/ubuntu/trell/env/lib/python3.6/site-packages/sqlalchemy/sql/elements.py", line 287, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/home/ubuntu/trell/env/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1101, in _execute_clauseelement
distilled_params,
File "/home/ubuntu/trell/env/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1250, in _execute_context
e, statement, parameters, cursor, context
File "/home/ubuntu/trell/env/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1476, in _handle_dbapi_exception
util.raise_from_cause(sqlalchemy_exception, exc_info)
File "/home/ubuntu/trell/env/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 398, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/home/ubuntu/trell/env/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 152, in reraise
raise value.with_traceback(tb)
File "/home/ubuntu/trell/env/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1226, in _execute_context
cursor, statement, parameters, context
File "/home/ubuntu/trell/env/lib/python3.6/site-packages/sqlalchemy/dialects/postgresql/psycopg2.py", line 854, in do_executemany
cursor.executemany(statement, parameters)
sqlalchemy.exc.DataError: (psycopg2.errors.StringDataRightTruncation) value too long for type character varying(256)
[SQL: INSERT INTO events_20180625 ("0") VALUES (%(0)s)]
[parameters: ({'0': '{"event_date":"20180625","event_timestamp":"1529948530245005","event_name":"firebase_campaign","event_params":[{"key":"medium","value":{"string_value ... (979 characters truncated) ... rce":"com.android.vending"},"traffic_source":{"name":"(direct)","medium":"(none)","source":"(direct)"},"stream_id":"1051639513","platform":"ANDROID"}'}, {'0': '{"event_date":"20180625","event_timestamp":"1529947540672001","event_name":"firebase_campaign","event_params":[{"key":"medium","value":{"string_value ... (985 characters truncated) ... rce":"com.android.vending"},"traffic_source":{"name":"(direct)","medium":"(none)","source":"(direct)"},"stream_id":"1051639513","platform":"ANDROID"}'}, {'0': '{"event_date":"20180625","event_timestamp":"1529948527702004","event_name":"firebase_campaign","event_params":[{"key":"medium","value":{"string_value ... (997 characters truncated) ... rce":"com.android.vending"},"traffic_source":{"name":"(direct)","medium":"(none)","source":"(direct)"},"stream_id":"1051639513","platform":"ANDROID"}'}, {'0': '{"event_date":"20180625","event_timestamp":"1529947540484000","event_name":"firebase_campaign","event_params":[{"key":"medium","value":{"string_value ... (1003 characters truncated) ... rce":"com.android.vending"},"traffic_source":{"name":"(direct)","medium":"(none)","source":"(direct)"},"stream_id":"1051639513","platform":"ANDROID"}'}, {'0': '{"event_date":"20180625","event_timestamp":"1529939515508006","event_name":"firebase_campaign","event_params":[{"key":"medium","value":{"string_value ... (1090 characters truncated) ... rce":"com.android.vending"},"traffic_source":{"name":"(direct)","medium":"(none)","source":"(direct)"},"stream_id":"1051639513","platform":"ANDROID"}'}, {'0': '{"event_date":"20180625","event_timestamp":"1529939930985001","event_name":"CREATE_POST_SUCCESS","event_params":[{"key":"item_name","value":{"string_ ... (1113 characters truncated) ... 7aa","install_source":"iTunes"},"traffic_source":{"name":"(direct)","medium":"(none)","source":"(direct)"},"stream_id":"1051639514","platform":"IOS"}'}, {'0': '{"event_date":"20180625","event_timestamp":"1529939930986002","event_name":"CREATE_POST_FINISH","event_params":[{"key":"item_name","value":{"string_v ... (1111 characters truncated) ... 7aa","install_source":"iTunes"},"traffic_source":{"name":"(direct)","medium":"(none)","source":"(direct)"},"stream_id":"1051639514","platform":"IOS"}'}, {'0': '{"event_date":"20180625","event_timestamp":"1529939930987003","event_name":"EMOTIONTAG_COUNT","event_params":[{"key":"item_name","value":{"string_val ... (1107 characters truncated) ... 7aa","install_source":"iTunes"},"traffic_source":{"name":"(direct)","medium":"(none)","source":"(direct)"},"stream_id":"1051639514","platform":"IOS"}'} ... displaying 10 of 49081 total bound parameter sets ... {'0': '{"event_date":"20180625","event_timestamp":"1529950522044233","event_name":"NARRATION_CLICK","event_params":[{"key":"firebase_screen_class","value":{ ... (1058 characters truncated) ... rce":"com.android.vending"},"traffic_source":{"name":"(direct)","medium":"(none)","source":"(direct)"},"stream_id":"1051639513","platform":"ANDROID"}'}, {'0': '{"event_date":"20180625","event_timestamp":"1529950553109238","event_name":"EMOTION_NEXT_CLICK","event_params":[{"key":"firebase_screen_class","value ... (350 characters truncated) ... 1238","user_pseudo_id":"edbabf0bc47bad8008d6b9037c439563","user_properties":[{"key":"first_open_time","value":{"int_value":"1529172000000","set_times'})]
我检查了表格的每一列，没有一列的 varchar 长度为 256。正如您在 create_table_sql 中看到的，所有 varchar 都是 65535。还尝试了json.load() 和json.dumps()，然后使用pandas 数据框将数据插入红移，但没有成功。
高度赞赏任何潜在客户。谢谢。
【问题讨论】：
标签： python sql python-3.x postgresql amazon-redshift