3v324v23 commited on
Commit
c3202cd
·
1 Parent(s): 98897be

chore: 添加开发辅助脚本用于测试和数据库检查

Browse files

新增 run_test_backfill.py 用于测试巴西贸易数据连接器的回填任务,新增 check_db.py 用于检查并展示数据库贸易记录。本次提交还包含临时日志文件与Python缓存文件,此类文件通常不应纳入版本控制。

check_db.py ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import asyncio
2
+ from packages.core.database import AsyncSessionLocal
3
+ from sqlalchemy import text
4
+
5
+ async def main():
6
+ async with AsyncSessionLocal() as session:
7
+ # 1. 检查入库总数
8
+ r = await session.execute(text("select count(1) from standard_trade_records"))
9
+ print(f"总记录数: {r.scalar()}")
10
+
11
+ # 2. 检查几条真实数据的翻译情况
12
+ query = """
13
+ select hs_code, product_name, origin_country, destination_country,
14
+ departure_port, arrival_port, transport_mode, amount, weight
15
+ from standard_trade_records
16
+ limit 5
17
+ """
18
+ r = await session.execute(text(query))
19
+ print("\n数据样例:")
20
+ for row in r:
21
+ print(f"- HS: {row.hs_code}")
22
+ print(f" 商品名: {row.product_name}")
23
+ print(f" 原产国/目的国: {row.origin_country} -> {row.destination_country}")
24
+ print(f" 起运港/目的港: {row.departure_port} -> {row.arrival_port}")
25
+ print(f" 运输方式: {row.transport_mode}")
26
+ print(f" 金额/重量: ${row.amount} / {row.weight}KG")
27
+ print("---")
28
+
29
+ asyncio.run(main())
logs/app_2026-06-10.log CHANGED
@@ -45,3 +45,14 @@
45
  2026-06-10 22:57:13.171 | WARNING | packages.core.http_client:request:79 - Request Error: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1081) for https://balanca.economia.gov.br/balanca/bd/tabelas/VIA.csv
46
  2026-06-10 22:57:13.172 | WARNING | packages.core.http_client:request:82 - SSL 证书校验失败,降级为 verify=False 重试: https://balanca.economia.gov.br/balanca/bd/tabelas/VIA.csv
47
  2026-06-10 22:57:13.174 | DEBUG | packages.core.http_client:request:55 - Request: GET https://balanca.economia.gov.br/balanca/bd/tabelas/VIA.csv | Attempt: 2/3
 
 
 
 
 
 
 
 
 
 
 
 
45
  2026-06-10 22:57:13.171 | WARNING | packages.core.http_client:request:79 - Request Error: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1081) for https://balanca.economia.gov.br/balanca/bd/tabelas/VIA.csv
46
  2026-06-10 22:57:13.172 | WARNING | packages.core.http_client:request:82 - SSL 证书校验失败,降级为 verify=False 重试: https://balanca.economia.gov.br/balanca/bd/tabelas/VIA.csv
47
  2026-06-10 22:57:13.174 | DEBUG | packages.core.http_client:request:55 - Request: GET https://balanca.economia.gov.br/balanca/bd/tabelas/VIA.csv | Attempt: 2/3
48
+ 2026-06-10 23:00:09.588 | INFO | __main__:run_backfill:16 - --- Starting BACKFILL Job for BR from 2026-05-01 to 2026-05-31 ---
49
+ 2026-06-10 23:00:11.742 | INFO | packages.connectors.base:run:128 - [BR] Starting job, batch_no: BACKFILL_BR_202605_202605
50
+ 2026-06-10 23:00:12.412 | DEBUG | packages.core.http_client:request:55 - Request: GET https://balanca.economia.gov.br/balanca/bd/comexstat-bd/ncm/EXP_2026.csv | Attempt: 1/3
51
+ 2026-06-10 23:00:14.140 | WARNING | packages.core.http_client:request:79 - Request Error: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1081) for https://balanca.economia.gov.br/balanca/bd/comexstat-bd/ncm/EXP_2026.csv
52
+ 2026-06-10 23:00:14.142 | WARNING | packages.core.http_client:request:82 - SSL 证书校验失败,降级为 verify=False 重试: https://balanca.economia.gov.br/balanca/bd/comexstat-bd/ncm/EXP_2026.csv
53
+ 2026-06-10 23:00:14.164 | DEBUG | packages.core.http_client:request:55 - Request: GET https://balanca.economia.gov.br/balanca/bd/comexstat-bd/ncm/EXP_2026.csv | Attempt: 2/3
54
+ 2026-06-10 23:26:13.157 | INFO | packages.connectors.base:run:128 - [BR] Starting job, batch_no: TEST_BATCH_001
55
+ 2026-06-10 23:26:13.430 | DEBUG | packages.core.http_client:request:55 - Request: GET https://balanca.economia.gov.br/balanca/bd/comexstat-bd/ncm/EXP_2026.csv | Attempt: 1/3
56
+ 2026-06-10 23:26:15.300 | WARNING | packages.core.http_client:request:79 - Request Error: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1081) for https://balanca.economia.gov.br/balanca/bd/comexstat-bd/ncm/EXP_2026.csv
57
+ 2026-06-10 23:26:15.300 | WARNING | packages.core.http_client:request:82 - SSL 证书校验失败,降级为 verify=False 重试: https://balanca.economia.gov.br/balanca/bd/comexstat-bd/ncm/EXP_2026.csv
58
+ 2026-06-10 23:26:15.302 | DEBUG | packages.core.http_client:request:55 - Request: GET https://balanca.economia.gov.br/balanca/bd/comexstat-bd/ncm/EXP_2026.csv | Attempt: 2/3
packages/connectors/mock/__pycache__/extended_mock.cpython-314.pyc ADDED
Binary file (6.58 kB). View file
 
packages/connectors/mock/__pycache__/us_mock.cpython-314.pyc ADDED
Binary file (4.8 kB). View file
 
run_test_backfill.py ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import asyncio
2
+ from packages.connectors.brazil import BrazilComexStatConnector
3
+ from packages.core.database import AsyncSessionLocal
4
+
5
+ async def main():
6
+ print("开始测试抓取...")
7
+ async with AsyncSessionLocal() as session:
8
+ connector = BrazilComexStatConnector(
9
+ session=session,
10
+ start_period="2026-05",
11
+ end_period="2026-05",
12
+ max_rows_per_slice=10 # 只抓 10 条
13
+ )
14
+ connector.batch_no = "TEST_BATCH_001"
15
+ await connector.run()
16
+ print("测试抓取结束.")
17
+
18
+ asyncio.run(main())