feat: 添加文本摘要功能和Git提交信息获取
Browse files新增文本摘要功能接口和Git提交信息获取工具
在飞书通知中添加Git提交信息展示
重构标题打分接口以接收JSON格式请求
- .gitignore +1 -0
- app.py +43 -8
- utils/git.py +27 -0
- utils/summarization.py +22 -0
.gitignore
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
__pycache__/
|
app.py
CHANGED
|
@@ -30,6 +30,8 @@ from utils.detection import (
|
|
| 30 |
from utils.visualization import visualize_detected_tables, plot_results
|
| 31 |
from utils.ocr import apply_ocr, save_csv
|
| 32 |
from utils.title_scoring import score_article_titles
|
|
|
|
|
|
|
| 33 |
|
| 34 |
|
| 35 |
# 设置日志
|
|
@@ -58,10 +60,18 @@ def check_hf_token(token):
|
|
| 58 |
# 向飞书 webhook 发送消息
|
| 59 |
if WEB_HOOK:
|
| 60 |
try:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
webhook_headers = {"Content-Type": "application/json"}
|
| 62 |
webhook_data = {
|
| 63 |
"msg_type": "text",
|
| 64 |
-
"content": {"text":
|
| 65 |
}
|
| 66 |
webhook_response = requests.post(
|
| 67 |
WEB_HOOK, json=webhook_data, headers=webhook_headers
|
|
@@ -505,32 +515,57 @@ async def text_to_speech(
|
|
| 505 |
|
| 506 |
|
| 507 |
@app.post("/text/score")
|
| 508 |
-
async def score_titles(
|
| 509 |
-
titles: List[str] = Body(..., description="List of article titles to score")
|
| 510 |
-
):
|
| 511 |
"""
|
| 512 |
为文章标题列表打分,返回标题和对应的分数。
|
| 513 |
|
| 514 |
- **titles**: 文章标题列表
|
| 515 |
"""
|
| 516 |
try:
|
|
|
|
|
|
|
|
|
|
| 517 |
# 调用打分函数
|
| 518 |
scores = score_article_titles(titles)
|
| 519 |
-
|
| 520 |
# 组合标题和分数
|
| 521 |
results = [
|
| 522 |
-
{"title": title, "score": score}
|
|
|
|
| 523 |
]
|
| 524 |
-
|
| 525 |
# 按分数倒序排序
|
| 526 |
results.sort(key=lambda x: x["score"], reverse=True)
|
| 527 |
-
|
| 528 |
return {"results": results}
|
| 529 |
except Exception as e:
|
| 530 |
logger.error(f"Title scoring error: {e}")
|
| 531 |
raise HTTPException(status_code=500, detail=f"服务器内部错误: {e}")
|
| 532 |
|
| 533 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 534 |
from fastapi.staticfiles import StaticFiles
|
| 535 |
|
| 536 |
app.mount("/", StaticFiles(directory="static", html=True), name="web")
|
|
|
|
| 30 |
from utils.visualization import visualize_detected_tables, plot_results
|
| 31 |
from utils.ocr import apply_ocr, save_csv
|
| 32 |
from utils.title_scoring import score_article_titles
|
| 33 |
+
from utils.summarization import summarize_text
|
| 34 |
+
from utils.git import get_git_commit_message
|
| 35 |
|
| 36 |
|
| 37 |
# 设置日志
|
|
|
|
| 60 |
# 向飞书 webhook 发送消息
|
| 61 |
if WEB_HOOK:
|
| 62 |
try:
|
| 63 |
+
# 获取Git commit message
|
| 64 |
+
commit_message = get_git_commit_message()
|
| 65 |
+
|
| 66 |
+
# 构建消息内容
|
| 67 |
+
message = f"你好,{user_info.get('name')},我已启动"
|
| 68 |
+
if commit_message:
|
| 69 |
+
message += f"\n\n最近的Git提交: {commit_message}"
|
| 70 |
+
|
| 71 |
webhook_headers = {"Content-Type": "application/json"}
|
| 72 |
webhook_data = {
|
| 73 |
"msg_type": "text",
|
| 74 |
+
"content": {"text": message},
|
| 75 |
}
|
| 76 |
webhook_response = requests.post(
|
| 77 |
WEB_HOOK, json=webhook_data, headers=webhook_headers
|
|
|
|
| 515 |
|
| 516 |
|
| 517 |
@app.post("/text/score")
|
| 518 |
+
async def score_titles(request_data: Dict[str, List[str]] = Body(..., description="Object containing titles list")):
|
|
|
|
|
|
|
| 519 |
"""
|
| 520 |
为文章标题列表打分,返回标题和对应的分数。
|
| 521 |
|
| 522 |
- **titles**: 文章标题列表
|
| 523 |
"""
|
| 524 |
try:
|
| 525 |
+
# 获取标题列表
|
| 526 |
+
titles = request_data.get("titles", [])
|
| 527 |
+
|
| 528 |
# 调用打分函数
|
| 529 |
scores = score_article_titles(titles)
|
| 530 |
+
|
| 531 |
# 组合标题和分数
|
| 532 |
results = [
|
| 533 |
+
{"title": title, "score": score}
|
| 534 |
+
for title, score in zip(titles, scores)
|
| 535 |
]
|
| 536 |
+
|
| 537 |
# 按分数倒序排序
|
| 538 |
results.sort(key=lambda x: x["score"], reverse=True)
|
| 539 |
+
|
| 540 |
return {"results": results}
|
| 541 |
except Exception as e:
|
| 542 |
logger.error(f"Title scoring error: {e}")
|
| 543 |
raise HTTPException(status_code=500, detail=f"服务器内部错误: {e}")
|
| 544 |
|
| 545 |
|
| 546 |
+
@app.post("/text/summary")
|
| 547 |
+
async def generate_summary(
|
| 548 |
+
text: str = Body(..., description="Text to summarize"),
|
| 549 |
+
max_length: int = Body(300, description="Maximum length of the summary"),
|
| 550 |
+
min_length: int = Body(50, description="Minimum length of the summary")
|
| 551 |
+
):
|
| 552 |
+
"""
|
| 553 |
+
生成文本摘要。
|
| 554 |
+
|
| 555 |
+
- **text**: 需要摘要的文本(建议为英文)
|
| 556 |
+
- **max_length**: 摘要最大长度(token数)
|
| 557 |
+
- **min_length**: 摘要最小长度(token数)
|
| 558 |
+
"""
|
| 559 |
+
try:
|
| 560 |
+
# 调用摘要函数
|
| 561 |
+
summary = summarize_text(text, max_length=max_length, min_length=min_length)
|
| 562 |
+
|
| 563 |
+
return {"summary": summary}
|
| 564 |
+
except Exception as e:
|
| 565 |
+
logger.error(f"Summarization error: {e}")
|
| 566 |
+
raise HTTPException(status_code=500, detail=f"服务器内部错误: {e}")
|
| 567 |
+
|
| 568 |
+
|
| 569 |
from fastapi.staticfiles import StaticFiles
|
| 570 |
|
| 571 |
app.mount("/", StaticFiles(directory="static", html=True), name="web")
|
utils/git.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import subprocess
|
| 3 |
+
import logging
|
| 4 |
+
|
| 5 |
+
logger = logging.getLogger(__name__)
|
| 6 |
+
|
| 7 |
+
def get_git_commit_message():
|
| 8 |
+
"""
|
| 9 |
+
获取当前Git commit message
|
| 10 |
+
"""
|
| 11 |
+
try:
|
| 12 |
+
# 执行git命令获取最近的commit message
|
| 13 |
+
result = subprocess.run(
|
| 14 |
+
['git', 'log', '-1', '--pretty=%B'],
|
| 15 |
+
capture_output=True,
|
| 16 |
+
text=True,
|
| 17 |
+
cwd=os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
| 18 |
+
)
|
| 19 |
+
if result.returncode == 0:
|
| 20 |
+
commit_message = result.stdout.strip()
|
| 21 |
+
return commit_message
|
| 22 |
+
else:
|
| 23 |
+
logger.error(f"获取Git commit message失败: {result.stderr}")
|
| 24 |
+
return None
|
| 25 |
+
except Exception as e:
|
| 26 |
+
logger.error(f"获取Git commit message出错: {e}")
|
| 27 |
+
return None
|
utils/summarization.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from transformers import pipeline
|
| 2 |
+
|
| 3 |
+
def summarize_text(text, max_length=300, min_length=50):
|
| 4 |
+
"""
|
| 5 |
+
使用 Falconsai/text_summarization 生成摘要
|
| 6 |
+
:param text: 输入的文章内容(建议为英文)
|
| 7 |
+
:param max_length: 摘要最大长度(token数,约等于汉字数*0.5)
|
| 8 |
+
:param min_length: 摘要最小长度
|
| 9 |
+
:return: 生成的摘要
|
| 10 |
+
"""
|
| 11 |
+
# 加载Pipeline
|
| 12 |
+
summarizer = pipeline("summarization", model="Falconsai/text_summarization")
|
| 13 |
+
|
| 14 |
+
# 生成摘要
|
| 15 |
+
result = summarizer(
|
| 16 |
+
text,
|
| 17 |
+
max_length=max_length,
|
| 18 |
+
min_length=min_length,
|
| 19 |
+
do_sample=False
|
| 20 |
+
)
|
| 21 |
+
|
| 22 |
+
return result[0]['summary_text']
|