shenjianZ/agricultural-sock-amalysis
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

data-collector fixed

Browse Source
This commit is contained in:
shenjianZ
2025-06-21 08:46:45 +08:00
parent 2e6603e29a
commit b52a9befee
28 changed files with 4097 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

147
data-collector/stock_crawler.py
View File

@@ -41,16 +41,75 @@ class StockCrawler:
# 农业类股票代码列表（示例）
self.agricultural_stocks = [
'sz300630', # 普利制药
'sh600998', # 九州通
'sh600371', # 万向钱潮
'sz000876', # 新希望
'sz002714', # 牧原股份
'sh600519', # 贵州茅台
'sz000858', # 五粮液
'sh600887', # 伊利股份
'sz002304', # 洋河股份
'sh600036', # 招商银行
'sz300189', # 神农种业
'sz000713', # 丰乐钟业
'sh600313', # 农发种业
'bj837403', # 康农种业
'sz200505', # 京粮B
'sz300268', # st佳沃
'sz000930', # 中粮科技
'sz002299', # 圣农发展
'sh600371', # 万向德农
'sh600598', # 北大荒
'sh603609', # 禾丰股份
'bj831087',
'sh603363',
'sh603336',
'sh600354',
'sz002385',
'sz000048',
'sh600251',
'sz002321',
'sz000505',
'sz001366',
'sz002772',
'sz002041',
'sh600127',
'sz002548',
'sh605296',
'sh600359',
'sh601952',
'sh600975',
# 新增股票代码
'sz300505',
'sh600141',
'sz000893',
'sh600691',
'sh600470',
'sh000912',
'sz000408',
'sz000902',
'sz002539',
'sz002599',
'sz002545',
'sz300387',
'sz002274',
'sz002470',
'sz002538',
'sz000731',
'sh600078',
'sh600227',
'sh603395',
'sz002588',
'sz000422',
'sz002556',
'sz002312',
'sz002170',
'sz000792',
'sh600096',
# 农药兽药类股票
'sz001231',
'sz002731',
'sz200553',
'sh603810',
'sh603970',
'sz301035',
'sz002868',
'sz003042',
'sz002391',
'bj870866',
'sz301665',
'sh600486',
]
# 初始化数据库连接
@@ -212,25 +271,59 @@ class StockCrawler:
try:
cursor = self.db_connection.cursor()
# 构建插入SQL
insert_sql = """
INSERT INTO stock_data (
stock_code, stock_name, open_price, close_price, high_price, low_price,
volume, turnover, change_percent, change_amount, pe_ratio, pb_ratio,
market_cap, float_market_cap, trade_date, create_time
) VALUES (
%(stock_code)s, %(stock_name)s, %(open_price)s, %(current_price)s,
%(high_price)s, %(low_price)s, %(volume)s, %(turnover)s,
%(change_percent)s, %(change_amount)s, %(pe_ratio)s, %(pb_ratio)s,
%(market_cap)s, %(float_market_cap)s, NOW(), NOW()
)
"""
# 获取当前日期
today = datetime.now().strftime('%Y-%m-%d')
for stock_data in stock_data_list:
# 查询今天是否已有该股票数据
check_sql = """
SELECT COUNT(*) FROM stock_data
WHERE stock_code = %s AND DATE(trade_date) = %s
"""
cursor.execute(check_sql, (stock_data['stock_code'], today))
count = cursor.fetchone()[0]
if count > 0:
# 已有数据，使用UPDATE更新
update_sql = """
UPDATE stock_data SET
stock_name = %(stock_name)s,
open_price = %(open_price)s,
close_price = %(current_price)s,
high_price = %(high_price)s,
low_price = %(low_price)s,
volume = %(volume)s,
turnover = %(turnover)s,
change_percent = %(change_percent)s,
change_amount = %(change_amount)s,
pe_ratio = %(pe_ratio)s,
pb_ratio = %(pb_ratio)s,
market_cap = %(market_cap)s,
float_market_cap = %(float_market_cap)s,
update_time = NOW()
WHERE stock_code = %(stock_code)s AND DATE(trade_date) = %(date)s
"""
cursor.execute(update_sql, {**stock_data, 'date': today})
logger.info(f"更新股票 {stock_data['stock_code']} 数据")
else:
# 无数据，使用INSERT插入
insert_sql = """
INSERT INTO stock_data (
stock_code, stock_name, open_price, close_price, high_price, low_price,
volume, turnover, change_percent, change_amount, pe_ratio, pb_ratio,
market_cap, float_market_cap, trade_date, create_time
) VALUES (
%(stock_code)s, %(stock_name)s, %(open_price)s, %(current_price)s,
%(high_price)s, %(low_price)s, %(volume)s, %(turnover)s,
%(change_percent)s, %(change_amount)s, %(pe_ratio)s, %(pb_ratio)s,
%(market_cap)s, %(float_market_cap)s, NOW(), NOW()
)
"""
cursor.execute(insert_sql, stock_data)
logger.info(f"插入股票 {stock_data['stock_code']} 数据")
# 批量插入数据
cursor.executemany(insert_sql, stock_data_list)
self.db_connection.commit()
logger.info(f"成功保存 {len(stock_data_list)} 条股票数据到数据库")
logger.info(f"成功处理 {len(stock_data_list)} 条股票数据到数据库")
except Exception as e:
logger.error(f"保存数据到数据库失败: {e}")