python导入csv大文件

pip install pandas sqlalchemy pymysql

import pandas as pd
from sqlalchemy import create_engine

# MySQL数据库连接信息
username = 'your_username'
password = 'your_password'
host = 'your_host'
database = 'your_database'
table_name = 'your_table'

# 创建数据库连接
engine = create_engine(f'mysql+pymysql://{username}:{password}@{host}/{database}')

# 大文件的分块大小（根据内存情况调整）
chunk_size = 10000

# 读取CSV文件并分块导入
csv_file = 'path_to_your_large_csv_file.csv'
for chunk in pd.read_csv(csv_file, chunksize=chunk_size):
    chunk.to_sql(name=table_name, con=engine, if_exists='append', index=False)
    print(f'Inserted chunk of size {len(chunk)}')

print("All chunks inserted successfully.")