import json import sqlite3 import requests import time from pathlib import Path DB_PATH = Path("data/quotes.db") def create_database(): DB_PATH.parent.mkdir(parents=True, exist_ok=True) conn = sqlite3.connect(DB_PATH) cursor = conn.cursor() cursor.execute(""" CREATE TABLE IF NOT EXISTS quotes ( id TEXT PRIMARY KEY, text TEXT NOT NULL, author TEXT, source TEXT, dynasty TEXT, type TEXT, tags TEXT, emotion TEXT ) """) cursor.execute("CREATE INDEX IF NOT EXISTS idx_type ON quotes(type)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_dynasty ON quotes(dynasty)") conn.commit() return conn def insert_quotes(conn, quotes): cursor = conn.cursor() inserted = 0 for quote in quotes: try: cursor.execute(""" INSERT OR IGNORE INTO quotes (id, text, author, source, dynasty, type, tags, emotion) VALUES (?, ?, ?, ?, ?, ?, ?, ?) """, ( quote['id'], quote['text'], quote.get('author'), quote.get('source'), quote.get('dynasty'), quote.get('type'), json.dumps(quote.get('tags', []), ensure_ascii=False), json.dumps(quote.get('emotion', []), ensure_ascii=False) )) if cursor.rowcount > 0: inserted += 1 except Exception as e: print(f"Error inserting {quote['id']}: {e}") conn.commit() return inserted def fetch_chinese_poetry(): poems = [] base_url = "https://raw.githubusercontent.com/chinese-poetry/chinese-poetry/master" print("Fetching Tang Dynasty poems...") for i in range(0, 58000, 1000): url = f"{base_url}/poet/poet_{i}.json" try: response = requests.get(url, timeout=15) if response.status_code == 200: data = response.json() for poem in data: text = "".join(poem.get("paragraphs", [])) if len(text) >= 10: poems.append({ "id": f"tang_{poem.get('id', i)}", "text": text, "author": poem.get("author", ""), "source": poem.get("title", ""), "dynasty": "唐", "type": "诗词", "tags": [], "emotion": [] }) print(f" Fetched {len(data)} poems from batch {i}") time.sleep(0.5) except Exception as e: print(f" Error fetching batch {i}: {e}") return poems def fetch_song_poetry(): poems = [] base_url = "https://raw.githubusercontent.com/chinese-poetry/chinese-poetry/master" print("Fetching Song Dynasty poems...") for i in range(0, 25000, 1000): url = f"{base_url}/poet/poet.song_{i}.json" try: response = requests.get(url, timeout=15) if response.status_code == 200: data = response.json() for poem in data: text = "".join(poem.get("paragraphs", [])) if len(text) >= 10: poems.append({ "id": f"song_{poem.get('id', i)}", "text": text, "author": poem.get("author", ""), "source": poem.get("title", ""), "dynasty": "宋", "type": "诗词", "tags": [], "emotion": [] }) print(f" Fetched {len(data)} poems from batch {i}") time.sleep(0.5) except Exception as e: print(f" Error fetching batch {i}: {e}") return poems def fetch_idioms(): idioms = [] url = "https://raw.githubusercontent.com/pwxcoo/chinese-xinhua/master/json/idioms.json" print("Fetching idioms...") try: response = requests.get(url, timeout=30) if response.status_code == 200: data = response.json() for item in data: word = item.get('word', '') if word and len(word) >= 3: idioms.append({ "id": f"idiom_{word}", "text": word, "author": None, "source": item.get('derivation', ''), "dynasty": None, "type": "成语", "tags": [], "emotion": [] }) print(f" Fetched {len(idioms)} idioms") except Exception as e: print(f" Error fetching idioms: {e}") return idioms def fetch_classic_quotes(): quotes = [] classic_texts = [ {"text": "学而时习之，不亦说乎", "author": "孔子", "source": "论语", "dynasty": "春秋"}, {"text": "温故而知新，可以为师矣", "author": "孔子", "source": "论语", "dynasty": "春秋"}, {"text": "学而不思则罔，思而不学则殆", "author": "孔子", "source": "论语", "dynasty": "春秋"}, {"text": "知之为知之，不知为不知，是知也", "author": "孔子", "source": "论语", "dynasty": "春秋"}, {"text": "己所不欲，勿施于人", "author": "孔子", "source": "论语", "dynasty": "春秋"}, {"text": "三人行，必有我师焉", "author": "孔子", "source": "论语", "dynasty": "春秋"}, {"text": "君子坦荡荡，小人长戚戚", "author": "孔子", "source": "论语", "dynasty": "春秋"}, {"text": "生于忧患，死于安乐", "author": "孟子", "source": "孟子", "dynasty": "战国"}, {"text": "得道多助，失道寡助", "author": "孟子", "source": "孟子", "dynasty": "战国"}, {"text": "富贵不能淫，贫贱不能移，威武不能屈", "author": "孟子", "source": "孟子", "dynasty": "战国"}, {"text": "天行健，君子以自强不息", "author": "佚名", "source": "周易", "dynasty": "先秦"}, {"text": "地势坤，君子以厚德载物", "author": "佚名", "source": "周易", "dynasty": "先秦"}, {"text": "路漫漫其修远兮，吾将上下而求索", "author": "屈原", "source": "离骚", "dynasty": "战国"}, {"text": "长太息以掩涕兮，哀民生之多艰", "author": "屈原", "source": "离骚", "dynasty": "战国"}, {"text": "亦余心之所善兮，虽九死其犹未悔", "author": "屈原", "source": "离骚", "dynasty": "战国"}, {"text": "老骥伏枥，志在千里", "author": "曹操", "source": "龟虽寿", "dynasty": "东汉"}, {"text": "烈士暮年，壮心不已", "author": "曹操", "source": "龟虽寿", "dynasty": "东汉"}, {"text": "山不厌高，海不厌深", "author": "曹操", "source": "短歌行", "dynasty": "东汉"}, {"text": "周公吐哺，天下归心", "author": "曹操", "source": "短歌行", "dynasty": "东汉"}, {"text": "捐躯赴国难，视死忽如归", "author": "曹植", "source": "白马篇", "dynasty": "三国"}, {"text": "本是同根生，相煎何太急", "author": "曹植", "source": "七步诗", "dynasty": "三国"}, {"text": "鞠躬尽瘁，死而后已", "author": "诸葛亮", "source": "后出师表", "dynasty": "三国"}, {"text": "非淡泊无以明志，非宁静无以致远", "author": "诸葛亮", "source": "诫子书", "dynasty": "三国"}, {"text": "采菊东篱下，悠然见南山", "author": "陶渊明", "source": "饮酒", "dynasty": "东晋"}, {"text": "羁鸟恋旧林，池鱼思故渊", "author": "陶渊明", "source": "归园田居", "dynasty": "东晋"}, {"text": "海内存知己，天涯若比邻", "author": "王勃", "source": "送杜少府之任蜀州", "dynasty": "唐"}, {"text": "落霞与孤鹜齐飞，秋水共长天一色", "author": "王勃", "source": "滕王阁序", "dynasty": "唐"}, {"text": "前不见古人，后不见来者", "author": "陈子昂", "source": "登幽州台歌", "dynasty": "唐"}, {"text": "念天地之悠悠，独怆然而涕下", "author": "陈子昂", "source": "登幽州台歌", "dynasty": "唐"}, {"text": "春江潮水连海平，海上明月共潮生", "author": "张若虚", "source": "春江花月夜", "dynasty": "唐"}, {"text": "人生代代无穷已，江月年年望相似", "author": "张若虚", "source": "春江花月夜", "dynasty": "唐"}, {"text": "欲穷千里目，更上一层楼", "author": "王之涣", "source": "登鹳雀楼", "dynasty": "唐"}, {"text": "黄河远上白云间，一片孤城万仞山", "author": "王之涣", "source": "凉州词", "dynasty": "唐"}, {"text": "独在异乡为异客，每逢佳节倍思亲", "author": "王维", "source": "九月九日忆山东兄弟", "dynasty": "唐"}, {"text": "劝君更尽一杯酒，西出阳关无故人", "author": "王维", "source": "送元二使安西", "dynasty": "唐"}, {"text": "大漠孤烟直，长河落日圆", "author": "王维", "source": "使至塞上", "dynasty": "唐"}, {"text": "明月松间照，清泉石上流", "author": "王维", "source": "山居秋暝", "dynasty": "唐"}, {"text": "天生我材必有用，千金散尽还复来", "author": "李白", "source": "将进酒", "dynasty": "唐"}, {"text": "长风破浪会有时，直挂云帆济沧海", "author": "李白", "source": "行路难", "dynasty": "唐"}, {"text": "举杯邀明月，对影成三人", "author": "李白", "source": "月下独酌", "dynasty": "唐"}, {"text": "抽刀断水水更流，举杯消愁愁更愁", "author": "李白", "source": "宣州谢朓楼饯别校书叔云", "dynasty": "唐"}, {"text": "安能摧眉折腰事权贵，使我不得开心颜", "author": "李白", "source": "梦游天姥吟留别", "dynasty": "唐"}, {"text": "会当凌绝顶，一览众山小", "author": "杜甫", "source": "望岳", "dynasty": "唐"}, {"text": "读书破万卷，下笔如有神", "author": "杜甫", "source": "奉赠韦左丞丈二十二韵", "dynasty": "唐"}, {"text": "出师未捷身先死，长使英雄泪满襟", "author": "杜甫", "source": "蜀相", "dynasty": "唐"}, {"text": "无边落木萧萧下，不尽长江滚滚来", "author": "杜甫", "source": "登高", "dynasty": "唐"}, {"text": "安得广厦千万间，大庇天下寒士俱欢颜", "author": "杜甫", "source": "茅屋为秋风所破歌", "dynasty": "唐"}, {"text": "忽如一夜春风来，千树万树梨花开", "author": "岑参", "source": "白雪歌送武判官归京", "dynasty": "唐"}, {"text": "沉舟侧畔千帆过，病树前头万木春", "author": "刘禹锡", "source": "酬乐天扬州初逢席上见赠", "dynasty": "唐"}, {"text": "旧时王谢堂前燕，飞入寻常百姓家", "author": "刘禹锡", "source": "乌衣巷", "dynasty": "唐"}, {"text": "千淘万漉虽辛苦，吹尽狂沙始到金", "author": "刘禹锡", "source": "浪淘沙", "dynasty": "唐"}, {"text": "野火烧不尽，春风吹又生", "author": "白居易", "source": "赋得古原草送别", "dynasty": "唐"}, {"text": "同是天涯沦落人，相逢何必曾相识", "author": "白居易", "source": "琵琶行", "dynasty": "唐"}, {"text": "在天愿作比翼鸟，在地愿为连理枝", "author": "白居易", "source": "长恨歌", "dynasty": "唐"}, {"text": "千呼万唤始出来，犹抱琵琶半遮面", "author": "白居易", "source": "琵琶行", "dynasty": "唐"}, {"text": "曾经沧海难为水，除却巫山不是云", "author": "元稹", "source": "离思", "dynasty": "唐"}, {"text": "黑云压城城欲摧，甲光向日金鳞开", "author": "李贺", "source": "雁门太守行", "dynasty": "唐"}, {"text": "男儿何不带吴钩，收取关山五十州", "author": "李贺", "source": "南园", "dynasty": "唐"}, {"text": "商女不知亡国恨，隔江犹唱后庭花", "author": "杜牧", "source": "泊秦淮", "dynasty": "唐"}, {"text": "停车坐爱枫林晚，霜叶红于二月花", "author": "杜牧", "source": "山行", "dynasty": "唐"}, {"text": "东风不与周郎便，铜雀春深锁二乔", "author": "杜牧", "source": "赤壁", "dynasty": "唐"}, {"text": "春蚕到死丝方尽，蜡炬成灰泪始干", "author": "李商隐", "source": "无题", "dynasty": "唐"}, {"text": "身无彩凤双飞翼，心有灵犀一点通", "author": "李商隐", "source": "无题", "dynasty": "唐"}, {"text": "夕阳无限好，只是近黄昏", "author": "李商隐", "source": "登乐游原", "dynasty": "唐"}, {"text": "先天下之忧而忧，后天下之乐而乐", "author": "范仲淹", "source": "岳阳楼记", "dynasty": "宋"}, {"text": "不以物喜，不以己悲", "author": "范仲淹", "source": "岳阳楼记", "dynasty": "宋"}, {"text": "醉翁之意不在酒，在乎山水之间也", "author": "欧阳修", "source": "醉翁亭记", "dynasty": "宋"}, {"text": "人生自是有情痴，此恨不关风与月", "author": "欧阳修", "source": "玉楼春", "dynasty": "宋"}, {"text": "衣带渐宽终不悔，为伊消得人憔悴", "author": "柳永", "source": "蝶恋花", "dynasty": "宋"}, {"text": "今宵酒醒何处？杨柳岸，晓风残月", "author": "柳永", "source": "雨霖铃", "dynasty": "宋"}, {"text": "但愿人长久，千里共婵娟", "author": "苏轼", "source": "水调歌头", "dynasty": "宋"}, {"text": "大江东去，浪淘尽，千古风流人物", "author": "苏轼", "source": "念奴娇·赤壁怀古", "dynasty": "宋"}, {"text": "竹杖芒鞋轻胜马，谁怕？一蓑烟雨任平生", "author": "苏轼", "source": "定风波", "dynasty": "宋"}, {"text": "回首向来萧瑟处，归去，也无风雨也无晴", "author": "苏轼", "source": "定风波", "dynasty": "宋"}, {"text": "十年生死两茫茫，不思量，自难忘", "author": "苏轼", "source": "江城子", "dynasty": "宋"}, {"text": "人生到处知何似，应似飞鸿踏雪泥", "author": "苏轼", "source": "和子由渑池怀旧", "dynasty": "宋"}, {"text": "不识庐山真面目，只缘身在此山中", "author": "苏轼", "source": "题西林壁", "dynasty": "宋"}, {"text": "两情若是久长时，又岂在朝朝暮暮", "author": "秦观", "source": "鹊桥仙", "dynasty": "宋"}, {"text": "此情无计可消除，才下眉头，却上心头", "author": "李清照", "source": "一剪梅", "dynasty": "宋"}, {"text": "寻寻觅觅，冷冷清清，凄凄惨惨戚戚", "author": "李清照", "source": "声声慢", "dynasty": "宋"}, {"text": "生当作人杰，死亦为鬼雄", "author": "李清照", "source": "夏日绝句", "dynasty": "宋"}, {"text": "三十功名尘与土，八千里路云和月", "author": "岳飞", "source": "满江红", "dynasty": "宋"}, {"text": "莫等闲，白了少年头，空悲切", "author": "岳飞", "source": "满江红", "dynasty": "宋"}, {"text": "壮志饥餐胡虏肉，笑谈渴饮匈奴血", "author": "岳飞", "source": "满江红", "dynasty": "宋"}, {"text": "山重水复疑无路，柳暗花明又一村", "author": "陆游", "source": "游山西村", "dynasty": "宋"}, {"text": "王师北定中原日，家祭无忘告乃翁", "author": "陆游", "source": "示儿", "dynasty": "宋"}, {"text": "小楼一夜听春雨，深巷明朝卖杏花", "author": "陆游", "source": "临安春雨初霁", "dynasty": "宋"}, {"text": "出师一表真名世，千载谁堪伯仲间", "author": "陆游", "source": "书愤", "dynasty": "宋"}, {"text": "人生自古谁无死，留取丹心照汗青", "author": "文天祥", "source": "过零丁洋", "dynasty": "宋"}, {"text": "臣心一片磁针石，不指南方不肯休", "author": "文天祥", "source": "扬子江", "dynasty": "宋"}, {"text": "问渠那得清如许，为有源头活水来", "author": "朱熹", "source": "观书有感", "dynasty": "宋"}, {"text": "等闲识得东风面，万紫千红总是春", "author": "朱熹", "source": "春日", "dynasty": "宋"}, {"text": "众里寻他千百度，蓦然回首，那人却在，灯火阑珊处", "author": "辛弃疾", "source": "青玉案·元夕", "dynasty": "宋"}, {"text": "想当年，金戈铁马，气吞万里如虎", "author": "辛弃疾", "source": "永遇乐·京口北固亭怀古", "dynasty": "宋"}, {"text": "醉里挑灯看剑，梦回吹角连营", "author": "辛弃疾", "source": "破阵子", "dynasty": "宋"}, {"text": "稻花香里说丰年，听取蛙声一片", "author": "辛弃疾", "source": "西江月·夜行黄沙道中", "dynasty": "宋"}, {"text": "春色满园关不住，一枝红杏出墙来", "author": "叶绍翁", "source": "游园不值", "dynasty": "宋"}, {"text": "落红不是无情物，化作春泥更护花", "author": "龚自珍", "source": "己亥杂诗", "dynasty": "清"}, {"text": "我劝天公重抖擞，不拘一格降人才", "author": "龚自珍", "source": "己亥杂诗", "dynasty": "清"}, {"text": "苟利国家生死以，岂因祸福避趋之", "author": "林则徐", "source": "赴戍登程口占示家人", "dynasty": "清"}, {"text": "海到无边天作岸，山登绝顶我为峰", "author": "林则徐", "source": "出老", "dynasty": "清"}, {"text": "我自横刀向天笑，去留肝胆两昆仑", "author": "谭嗣同", "source": "狱中题壁", "dynasty": "清"}, {"text": "横眉冷对千夫指，俯首甘为孺子牛", "author": "鲁迅", "source": "自嘲", "dynasty": "近代"}, {"text": "寄意寒星荃不察，我以我血荐轩辕", "author": "鲁迅", "source": "自题小像", "dynasty": "近代"}, {"text": "心事浩茫连广宇，于无声处听惊雷", "author": "鲁迅", "source": "无题", "dynasty": "近代"}, ] for i, item in enumerate(classic_texts): quotes.append({ "id": f"classic_{i}", "text": item["text"], "author": item.get("author"), "source": item.get("source"), "dynasty": item.get("dynasty"), "type": "名句", "tags": [], "emotion": [] }) return quotes def main(): print("=" * 60) print("名句数据收集") print("=" * 60) conn = create_database() total_inserted = 0 print("\n[1/4] Fetching Tang Dynasty poems...") tang_poems = fetch_chinese_poetry() inserted = insert_quotes(conn, tang_poems) total_inserted += inserted print(f" Inserted {inserted} Tang poems") print("\n[2/4] Fetching Song Dynasty poems...") song_poems = fetch_song_poetry() inserted = insert_quotes(conn, song_poems) total_inserted += inserted print(f" Inserted {inserted} Song poems") print("\n[3/4] Fetching idioms...") idioms = fetch_idioms() inserted = insert_quotes(conn, idioms) total_inserted += inserted print(f" Inserted {inserted} idioms") print("\n[4/4] Adding classic quotes...") classics = fetch_classic_quotes() inserted = insert_quotes(conn, classics) total_inserted += inserted print(f" Inserted {inserted} classic quotes") conn.close() print("\n" + "=" * 60) print(f"Total inserted: {total_inserted} quotes") print(f"Database saved to: {DB_PATH}") print("=" * 60) if __name__ == "__main__": main()