Spaces:
Runtime error
Runtime error
chore: 添加开发辅助脚本用于测试和数据库检查
Browse files新增 run_test_backfill.py 用于测试巴西贸易数据连接器的回填任务,新增 check_db.py 用于检查并展示数据库贸易记录。本次提交还包含临时日志文件与Python缓存文件,此类文件通常不应纳入版本控制。
check_db.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
+
from packages.core.database import AsyncSessionLocal
|
| 3 |
+
from sqlalchemy import text
|
| 4 |
+
|
| 5 |
+
async def main():
|
| 6 |
+
async with AsyncSessionLocal() as session:
|
| 7 |
+
# 1. 检查入库总数
|
| 8 |
+
r = await session.execute(text("select count(1) from standard_trade_records"))
|
| 9 |
+
print(f"总记录数: {r.scalar()}")
|
| 10 |
+
|
| 11 |
+
# 2. 检查几条真实数据的翻译情况
|
| 12 |
+
query = """
|
| 13 |
+
select hs_code, product_name, origin_country, destination_country,
|
| 14 |
+
departure_port, arrival_port, transport_mode, amount, weight
|
| 15 |
+
from standard_trade_records
|
| 16 |
+
limit 5
|
| 17 |
+
"""
|
| 18 |
+
r = await session.execute(text(query))
|
| 19 |
+
print("\n数据样例:")
|
| 20 |
+
for row in r:
|
| 21 |
+
print(f"- HS: {row.hs_code}")
|
| 22 |
+
print(f" 商品名: {row.product_name}")
|
| 23 |
+
print(f" 原产国/目的国: {row.origin_country} -> {row.destination_country}")
|
| 24 |
+
print(f" 起运港/目的港: {row.departure_port} -> {row.arrival_port}")
|
| 25 |
+
print(f" 运输方式: {row.transport_mode}")
|
| 26 |
+
print(f" 金额/重量: ${row.amount} / {row.weight}KG")
|
| 27 |
+
print("---")
|
| 28 |
+
|
| 29 |
+
asyncio.run(main())
|
logs/app_2026-06-10.log
CHANGED
|
@@ -45,3 +45,14 @@
|
|
| 45 |
2026-06-10 22:57:13.171 | WARNING | packages.core.http_client:request:79 - Request Error: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1081) for https://balanca.economia.gov.br/balanca/bd/tabelas/VIA.csv
|
| 46 |
2026-06-10 22:57:13.172 | WARNING | packages.core.http_client:request:82 - SSL 证书校验失败,降级为 verify=False 重试: https://balanca.economia.gov.br/balanca/bd/tabelas/VIA.csv
|
| 47 |
2026-06-10 22:57:13.174 | DEBUG | packages.core.http_client:request:55 - Request: GET https://balanca.economia.gov.br/balanca/bd/tabelas/VIA.csv | Attempt: 2/3
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
2026-06-10 22:57:13.171 | WARNING | packages.core.http_client:request:79 - Request Error: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1081) for https://balanca.economia.gov.br/balanca/bd/tabelas/VIA.csv
|
| 46 |
2026-06-10 22:57:13.172 | WARNING | packages.core.http_client:request:82 - SSL 证书校验失败,降级为 verify=False 重试: https://balanca.economia.gov.br/balanca/bd/tabelas/VIA.csv
|
| 47 |
2026-06-10 22:57:13.174 | DEBUG | packages.core.http_client:request:55 - Request: GET https://balanca.economia.gov.br/balanca/bd/tabelas/VIA.csv | Attempt: 2/3
|
| 48 |
+
2026-06-10 23:00:09.588 | INFO | __main__:run_backfill:16 - --- Starting BACKFILL Job for BR from 2026-05-01 to 2026-05-31 ---
|
| 49 |
+
2026-06-10 23:00:11.742 | INFO | packages.connectors.base:run:128 - [BR] Starting job, batch_no: BACKFILL_BR_202605_202605
|
| 50 |
+
2026-06-10 23:00:12.412 | DEBUG | packages.core.http_client:request:55 - Request: GET https://balanca.economia.gov.br/balanca/bd/comexstat-bd/ncm/EXP_2026.csv | Attempt: 1/3
|
| 51 |
+
2026-06-10 23:00:14.140 | WARNING | packages.core.http_client:request:79 - Request Error: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1081) for https://balanca.economia.gov.br/balanca/bd/comexstat-bd/ncm/EXP_2026.csv
|
| 52 |
+
2026-06-10 23:00:14.142 | WARNING | packages.core.http_client:request:82 - SSL 证书校验失败,降级为 verify=False 重试: https://balanca.economia.gov.br/balanca/bd/comexstat-bd/ncm/EXP_2026.csv
|
| 53 |
+
2026-06-10 23:00:14.164 | DEBUG | packages.core.http_client:request:55 - Request: GET https://balanca.economia.gov.br/balanca/bd/comexstat-bd/ncm/EXP_2026.csv | Attempt: 2/3
|
| 54 |
+
2026-06-10 23:26:13.157 | INFO | packages.connectors.base:run:128 - [BR] Starting job, batch_no: TEST_BATCH_001
|
| 55 |
+
2026-06-10 23:26:13.430 | DEBUG | packages.core.http_client:request:55 - Request: GET https://balanca.economia.gov.br/balanca/bd/comexstat-bd/ncm/EXP_2026.csv | Attempt: 1/3
|
| 56 |
+
2026-06-10 23:26:15.300 | WARNING | packages.core.http_client:request:79 - Request Error: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1081) for https://balanca.economia.gov.br/balanca/bd/comexstat-bd/ncm/EXP_2026.csv
|
| 57 |
+
2026-06-10 23:26:15.300 | WARNING | packages.core.http_client:request:82 - SSL 证书校验失败,降级为 verify=False 重试: https://balanca.economia.gov.br/balanca/bd/comexstat-bd/ncm/EXP_2026.csv
|
| 58 |
+
2026-06-10 23:26:15.302 | DEBUG | packages.core.http_client:request:55 - Request: GET https://balanca.economia.gov.br/balanca/bd/comexstat-bd/ncm/EXP_2026.csv | Attempt: 2/3
|
packages/connectors/mock/__pycache__/extended_mock.cpython-314.pyc
ADDED
|
Binary file (6.58 kB). View file
|
|
|
packages/connectors/mock/__pycache__/us_mock.cpython-314.pyc
ADDED
|
Binary file (4.8 kB). View file
|
|
|
run_test_backfill.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
+
from packages.connectors.brazil import BrazilComexStatConnector
|
| 3 |
+
from packages.core.database import AsyncSessionLocal
|
| 4 |
+
|
| 5 |
+
async def main():
|
| 6 |
+
print("开始测试抓取...")
|
| 7 |
+
async with AsyncSessionLocal() as session:
|
| 8 |
+
connector = BrazilComexStatConnector(
|
| 9 |
+
session=session,
|
| 10 |
+
start_period="2026-05",
|
| 11 |
+
end_period="2026-05",
|
| 12 |
+
max_rows_per_slice=10 # 只抓 10 条
|
| 13 |
+
)
|
| 14 |
+
connector.batch_no = "TEST_BATCH_001"
|
| 15 |
+
await connector.run()
|
| 16 |
+
print("测试抓取结束.")
|
| 17 |
+
|
| 18 |
+
asyncio.run(main())
|