ogx786 commited on
Commit
8e7dfd2
·
verified ·
1 Parent(s): aee7439

Update example

Browse files
Files changed (1) hide show
  1. example +99 -172
example CHANGED
@@ -1,173 +1,100 @@
1
- import json
2
- import csv
3
- from pathlib import Path
4
-
5
-
6
- # ==========================
7
- # CONFIGURATION
8
- # ==========================
9
-
10
- INPUT_JSON = "input.json" # Change to your JSON filename
11
- OUTPUT_CSV = "hbl_pipe_dataset.csv"
12
-
13
- # Fields to extract in order
14
- FIELDS = [
15
- "name",
16
- "fatherName",
17
- "motherName",
18
- "address1",
19
- "address2",
20
- "placeOfBirth",
21
- "cnic"
22
- ]
23
-
24
-
25
- # ==========================
26
- # HELPER FUNCTION
27
- # ==========================
28
-
29
- def clean_value(value):
30
- """
31
- Clean extracted values.
32
- Keeps empty fields instead of removing them.
33
- """
34
-
35
- if value is None:
36
- return ""
37
-
38
- # Convert everything to string
39
- return str(value).strip()
40
-
41
-
42
- def join_fields(data):
43
- """
44
- Convert dictionary fields into pipe-separated string.
45
- """
46
-
47
- return "|".join(
48
- clean_value(data.get(field, ""))
49
- for field in FIELDS
50
- )
51
-
52
-
53
- # ==========================
54
- # MAIN PROCESSING
55
- # ==========================
56
-
57
- def process_json():
58
-
59
- input_path = Path(INPUT_JSON)
60
-
61
- if not input_path.exists():
62
- raise FileNotFoundError(
63
- f"JSON file not found: {INPUT_JSON}"
64
- )
65
-
66
- print("Loading JSON...")
67
-
68
- with open(
69
- INPUT_JSON,
70
- "r",
71
- encoding="utf-8"
72
- ) as f:
73
- json_data = json.load(f)
74
-
75
-
76
- rows = []
77
-
78
- total_records = 0
79
- skipped_records = 0
80
-
81
-
82
- # Option A:
83
- # Outer object contains response array
84
-
85
- response_list = json_data.get("response", [])
86
-
87
-
88
- for item in response_list:
89
-
90
- total_records += 1
91
-
92
- try:
93
-
94
- # Roman Urdu
95
- roman_data = item.get(
96
- "data",
97
- {}
98
- )
99
-
100
- # Urdu
101
- urdu_data = item.get(
102
- "REQUEST_BODY",
103
- {}
104
- )
105
-
106
-
107
- if not urdu_data or not roman_data:
108
- skipped_records += 1
109
- continue
110
-
111
-
112
- urdu_text = join_fields(
113
- urdu_data
114
- )
115
-
116
- roman_text = join_fields(
117
- roman_data
118
- )
119
-
120
-
121
- rows.append(
122
- {
123
- "urdu": urdu_text,
124
- "roman": roman_text
125
  }
126
- )
127
-
128
-
129
- except Exception as e:
130
-
131
- print(
132
- "Skipping record due to error:",
133
- e
134
- )
135
-
136
- skipped_records += 1
137
-
138
-
139
-
140
- print(f"Total records: {total_records}")
141
- print(f"Created pairs: {len(rows)}")
142
- print(f"Skipped: {skipped_records}")
143
-
144
-
145
- # Save CSV
146
-
147
- with open(
148
- OUTPUT_CSV,
149
- "w",
150
- encoding="utf-8",
151
- newline=""
152
- ) as f:
153
-
154
- writer = csv.DictWriter(
155
- f,
156
- fieldnames=[
157
- "urdu",
158
- "roman"
159
- ]
160
- )
161
-
162
- writer.writeheader()
163
-
164
- writer.writerows(rows)
165
-
166
-
167
- print(
168
- f"Saved dataset: {OUTPUT_CSV}"
169
- )
170
-
171
-
172
- if __name__ == "__main__":
173
- process_json()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_id": "6a30f84db2bafb037face9c6",
3
+ "hasMore": true,
4
+ "limitReached": false,
5
+ "pageSize": 20,
6
+ "requestID": "6a30f84db2bafb037face9c6",
7
+ "requestType": "find",
8
+ "response": [
9
+ {
10
+ "_id": {
11
+ "$oid": "6a30f849eb71684d09ee22ae"
12
+ },
13
+ "message": "Success",
14
+ "data": {
15
+ "name": "SHAMEELA QAYOOM",
16
+ "fatherName": "ABDUL QAYYUM SHAH",
17
+ "motherName": "ABDUL QAYYUM SHAH",
18
+ "address1": "POST OFFICE KHAS SAMAL TEHSIL GOPAS DISTRICT GHAZAR",
19
+ "address2": "POST OFFICE KHAS SAMAL TEHSIL GOPAS DISTRICT GHAZAR",
20
+ "placeOfBirth": "GOPAS GHAZER",
21
+ "cnic": "7140132783264",
22
+ "aksaTransId": "8723608"
23
+ },
24
+ "LOG_CODE": "3",
25
+ "LOG_DESC": "service_name: custDataTranToEnglish, type: success",
26
+ "LOG_DATE": {
27
+ "$date": {
28
+ "$numberLong": "1781594185116"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
29
  }
30
+ },
31
+ "x-req-id": "10009706040",
32
+ "statusCode": {
33
+ "$numberInt": "200"
34
+ },
35
+ "statusMessage": "OK",
36
+ "CITIZEN_NUMBER": "7140132783264",
37
+ "REQUEST_BODY": {
38
+ "name": "شمیلہ قیوم",
39
+ "fatherName": "عبدالقیوم شاہ",
40
+ "motherName": "عبدالقیوم شاہ",
41
+ "address1": "ڈاک خانہ خاص،سمال،تحصیل گوپس،ضلع غذر",
42
+ "address2": "ڈاک خانہ خاص،سمال،تحصیل گوپس،ضلع غذر",
43
+ "placeOfBirth": "گوپس,غذر",
44
+ "cnic": "7140132783264"
45
+ },
46
+ "URL": "URL: https://apigateway.hbl.com:8343/api/v1/urdu/translate/english",
47
+ "REQUEST_HEADERS": {
48
+ "Content-Type": "application/json",
49
+ "x-channel-id": "HBL",
50
+ "x-req-id": "10009706040",
51
+ "x-sub-channel-id": "HBL",
52
+ "x-country-code": "PK"
53
+ }
54
+ },
55
+ {
56
+ "_id": {
57
+ "$oid": "6a30f8475c58784070731a9c"
58
+ },
59
+ "message": "Success",
60
+ "data": {
61
+ "name": "MUJAHID IQBAL",
62
+ "fatherName": "NOOR ZAMAN",
63
+ "motherName": "NOOR ZAMAN",
64
+ "address1": "DHOK ROADA DHER POST OFFICE KHAS DAHLIAN TEHSIL PINDI GHAIB DISTRICT ATTOCK",
65
+ "address2": "DHOK ROADA DHER POST OFFICE KHAS DAHLIAN TEHSIL PINDI GHAIB DISTRICT ATTOCK",
66
+ "placeOfBirth": "PINDI GHAIB ATTOCK",
67
+ "cnic": "3710503011839",
68
+ "aksaTransId": "8723607"
69
+ },
70
+ "LOG_CODE": "3",
71
+ "LOG_DESC": "service_name: custDataTranToEnglish, type: success",
72
+ "LOG_DATE": {
73
+ "$date": {
74
+ "$numberLong": "1781594183326"
75
+ }
76
+ },
77
+ "x-req-id": "10009706039",
78
+ "statusCode": {
79
+ "$numberInt": "200"
80
+ },
81
+ "statusMessage": "OK",
82
+ "CITIZEN_NUMBER": "3710503011839",
83
+ "REQUEST_BODY": {
84
+ "name": "مجاہد اقبال",
85
+ "fatherName": "نور زمان",
86
+ "motherName": "نور زمان",
87
+ "address1": "ڈھوک روڈا ڈھیر،ڈاک خانہ خاص،ڈھلیاں،تحصیل پنڈی گھیب،ضلع اٹک",
88
+ "address2": "ڈھوک روڈا ڈھیر،ڈاک خانہ خاص،ڈھلیاں،تحصیل پنڈی گھیب،ضلع اٹک",
89
+ "placeOfBirth": "پنڈی گھیب,اٹک",
90
+ "cnic": "3710503011839"
91
+ },
92
+ "URL": "URL: https://apigateway.hbl.com:8343/api/v1/urdu/translate/english",
93
+ "REQUEST_HEADERS": {
94
+ "Content-Type": "application/json",
95
+ "x-channel-id": "HBL",
96
+ "x-req-id": "10009706039",
97
+ "x-sub-channel-id": "HBL",
98
+ "x-country-code": "PK"
99
+ }
100
+ },