codeShare commited on
Commit
7527a2f
·
verified ·
1 Parent(s): fa07899

Upload HAR_extract_to_json.ipynb

Browse files
Files changed (1) hide show
  1. HAR_extract_to_json.ipynb +1939 -3
HAR_extract_to_json.ipynb CHANGED
@@ -15,6 +15,356 @@
15
  },
16
  "language_info": {
17
  "name": "python"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
18
  }
19
  },
20
  "cells": [
@@ -34,10 +384,22 @@
34
  "drive.mount('/content/drive')"
35
  ],
36
  "metadata": {
37
- "id": "sXBKnyLI0_L2"
 
 
 
 
38
  },
39
- "execution_count": null,
40
- "outputs": []
 
 
 
 
 
 
 
 
41
  },
42
  {
43
  "cell_type": "code",
@@ -867,6 +1229,1580 @@
867
  },
868
  "execution_count": null,
869
  "outputs": []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
870
  }
871
  ]
872
  }
 
15
  },
16
  "language_info": {
17
  "name": "python"
18
+ },
19
+ "widgets": {
20
+ "application/vnd.jupyter.widget-state+json": {
21
+ "version_major": 2,
22
+ "version_minor": 0,
23
+ "state": {
24
+ "1e5a7bbb49764dd1abef511964d23a70": {
25
+ "model_module": "@jupyter-widgets/controls",
26
+ "model_name": "HBoxModel",
27
+ "model_module_version": "1.5.0",
28
+ "state": {
29
+ "_dom_classes": [],
30
+ "_model_module": "@jupyter-widgets/controls",
31
+ "_model_module_version": "1.5.0",
32
+ "_model_name": "HBoxModel",
33
+ "_view_count": null,
34
+ "_view_module": "@jupyter-widgets/controls",
35
+ "_view_module_version": "1.5.0",
36
+ "_view_name": "HBoxView",
37
+ "box_style": "",
38
+ "children": [
39
+ "IPY_MODEL_d8e77ce9f701423583e447ccb5daf0f2",
40
+ "IPY_MODEL_a96ca4a84fa941b08cff43fb6719072e",
41
+ "IPY_MODEL_f08ac9209f1a4c559d81db1c14c0b195"
42
+ ],
43
+ "layout": "IPY_MODEL_90b359c740814c52b3f0c20447a0b578"
44
+ }
45
+ },
46
+ "d8e77ce9f701423583e447ccb5daf0f2": {
47
+ "model_module": "@jupyter-widgets/controls",
48
+ "model_name": "HTMLModel",
49
+ "model_module_version": "1.5.0",
50
+ "state": {
51
+ "_dom_classes": [],
52
+ "_model_module": "@jupyter-widgets/controls",
53
+ "_model_module_version": "1.5.0",
54
+ "_model_name": "HTMLModel",
55
+ "_view_count": null,
56
+ "_view_module": "@jupyter-widgets/controls",
57
+ "_view_module_version": "1.5.0",
58
+ "_view_name": "HTMLView",
59
+ "description": "",
60
+ "description_tooltip": null,
61
+ "layout": "IPY_MODEL_3c6fe424b4b6498fb8f3423f8e53c544",
62
+ "placeholder": "​",
63
+ "style": "IPY_MODEL_af4e28043c8b4aeda06173bdcd7389a0",
64
+ "value": "HAR files: 100%"
65
+ }
66
+ },
67
+ "a96ca4a84fa941b08cff43fb6719072e": {
68
+ "model_module": "@jupyter-widgets/controls",
69
+ "model_name": "FloatProgressModel",
70
+ "model_module_version": "1.5.0",
71
+ "state": {
72
+ "_dom_classes": [],
73
+ "_model_module": "@jupyter-widgets/controls",
74
+ "_model_module_version": "1.5.0",
75
+ "_model_name": "FloatProgressModel",
76
+ "_view_count": null,
77
+ "_view_module": "@jupyter-widgets/controls",
78
+ "_view_module_version": "1.5.0",
79
+ "_view_name": "ProgressView",
80
+ "bar_style": "success",
81
+ "description": "",
82
+ "description_tooltip": null,
83
+ "layout": "IPY_MODEL_f60bd97a472042e2b8bb3dac3db40278",
84
+ "max": 8,
85
+ "min": 0,
86
+ "orientation": "horizontal",
87
+ "style": "IPY_MODEL_4896f44a58674d5b9414bcb695b4cc61",
88
+ "value": 8
89
+ }
90
+ },
91
+ "f08ac9209f1a4c559d81db1c14c0b195": {
92
+ "model_module": "@jupyter-widgets/controls",
93
+ "model_name": "HTMLModel",
94
+ "model_module_version": "1.5.0",
95
+ "state": {
96
+ "_dom_classes": [],
97
+ "_model_module": "@jupyter-widgets/controls",
98
+ "_model_module_version": "1.5.0",
99
+ "_model_name": "HTMLModel",
100
+ "_view_count": null,
101
+ "_view_module": "@jupyter-widgets/controls",
102
+ "_view_module_version": "1.5.0",
103
+ "_view_name": "HTMLView",
104
+ "description": "",
105
+ "description_tooltip": null,
106
+ "layout": "IPY_MODEL_40fe6d49d9ad4f8b8374a91b01610fa4",
107
+ "placeholder": "​",
108
+ "style": "IPY_MODEL_7c691ef800d6445ebb896559dcae8931",
109
+ "value": " 8/8 [00:27<00:00,  2.76s/it]"
110
+ }
111
+ },
112
+ "90b359c740814c52b3f0c20447a0b578": {
113
+ "model_module": "@jupyter-widgets/base",
114
+ "model_name": "LayoutModel",
115
+ "model_module_version": "1.2.0",
116
+ "state": {
117
+ "_model_module": "@jupyter-widgets/base",
118
+ "_model_module_version": "1.2.0",
119
+ "_model_name": "LayoutModel",
120
+ "_view_count": null,
121
+ "_view_module": "@jupyter-widgets/base",
122
+ "_view_module_version": "1.2.0",
123
+ "_view_name": "LayoutView",
124
+ "align_content": null,
125
+ "align_items": null,
126
+ "align_self": null,
127
+ "border": null,
128
+ "bottom": null,
129
+ "display": null,
130
+ "flex": null,
131
+ "flex_flow": null,
132
+ "grid_area": null,
133
+ "grid_auto_columns": null,
134
+ "grid_auto_flow": null,
135
+ "grid_auto_rows": null,
136
+ "grid_column": null,
137
+ "grid_gap": null,
138
+ "grid_row": null,
139
+ "grid_template_areas": null,
140
+ "grid_template_columns": null,
141
+ "grid_template_rows": null,
142
+ "height": null,
143
+ "justify_content": null,
144
+ "justify_items": null,
145
+ "left": null,
146
+ "margin": null,
147
+ "max_height": null,
148
+ "max_width": null,
149
+ "min_height": null,
150
+ "min_width": null,
151
+ "object_fit": null,
152
+ "object_position": null,
153
+ "order": null,
154
+ "overflow": null,
155
+ "overflow_x": null,
156
+ "overflow_y": null,
157
+ "padding": null,
158
+ "right": null,
159
+ "top": null,
160
+ "visibility": null,
161
+ "width": null
162
+ }
163
+ },
164
+ "3c6fe424b4b6498fb8f3423f8e53c544": {
165
+ "model_module": "@jupyter-widgets/base",
166
+ "model_name": "LayoutModel",
167
+ "model_module_version": "1.2.0",
168
+ "state": {
169
+ "_model_module": "@jupyter-widgets/base",
170
+ "_model_module_version": "1.2.0",
171
+ "_model_name": "LayoutModel",
172
+ "_view_count": null,
173
+ "_view_module": "@jupyter-widgets/base",
174
+ "_view_module_version": "1.2.0",
175
+ "_view_name": "LayoutView",
176
+ "align_content": null,
177
+ "align_items": null,
178
+ "align_self": null,
179
+ "border": null,
180
+ "bottom": null,
181
+ "display": null,
182
+ "flex": null,
183
+ "flex_flow": null,
184
+ "grid_area": null,
185
+ "grid_auto_columns": null,
186
+ "grid_auto_flow": null,
187
+ "grid_auto_rows": null,
188
+ "grid_column": null,
189
+ "grid_gap": null,
190
+ "grid_row": null,
191
+ "grid_template_areas": null,
192
+ "grid_template_columns": null,
193
+ "grid_template_rows": null,
194
+ "height": null,
195
+ "justify_content": null,
196
+ "justify_items": null,
197
+ "left": null,
198
+ "margin": null,
199
+ "max_height": null,
200
+ "max_width": null,
201
+ "min_height": null,
202
+ "min_width": null,
203
+ "object_fit": null,
204
+ "object_position": null,
205
+ "order": null,
206
+ "overflow": null,
207
+ "overflow_x": null,
208
+ "overflow_y": null,
209
+ "padding": null,
210
+ "right": null,
211
+ "top": null,
212
+ "visibility": null,
213
+ "width": null
214
+ }
215
+ },
216
+ "af4e28043c8b4aeda06173bdcd7389a0": {
217
+ "model_module": "@jupyter-widgets/controls",
218
+ "model_name": "DescriptionStyleModel",
219
+ "model_module_version": "1.5.0",
220
+ "state": {
221
+ "_model_module": "@jupyter-widgets/controls",
222
+ "_model_module_version": "1.5.0",
223
+ "_model_name": "DescriptionStyleModel",
224
+ "_view_count": null,
225
+ "_view_module": "@jupyter-widgets/base",
226
+ "_view_module_version": "1.2.0",
227
+ "_view_name": "StyleView",
228
+ "description_width": ""
229
+ }
230
+ },
231
+ "f60bd97a472042e2b8bb3dac3db40278": {
232
+ "model_module": "@jupyter-widgets/base",
233
+ "model_name": "LayoutModel",
234
+ "model_module_version": "1.2.0",
235
+ "state": {
236
+ "_model_module": "@jupyter-widgets/base",
237
+ "_model_module_version": "1.2.0",
238
+ "_model_name": "LayoutModel",
239
+ "_view_count": null,
240
+ "_view_module": "@jupyter-widgets/base",
241
+ "_view_module_version": "1.2.0",
242
+ "_view_name": "LayoutView",
243
+ "align_content": null,
244
+ "align_items": null,
245
+ "align_self": null,
246
+ "border": null,
247
+ "bottom": null,
248
+ "display": null,
249
+ "flex": null,
250
+ "flex_flow": null,
251
+ "grid_area": null,
252
+ "grid_auto_columns": null,
253
+ "grid_auto_flow": null,
254
+ "grid_auto_rows": null,
255
+ "grid_column": null,
256
+ "grid_gap": null,
257
+ "grid_row": null,
258
+ "grid_template_areas": null,
259
+ "grid_template_columns": null,
260
+ "grid_template_rows": null,
261
+ "height": null,
262
+ "justify_content": null,
263
+ "justify_items": null,
264
+ "left": null,
265
+ "margin": null,
266
+ "max_height": null,
267
+ "max_width": null,
268
+ "min_height": null,
269
+ "min_width": null,
270
+ "object_fit": null,
271
+ "object_position": null,
272
+ "order": null,
273
+ "overflow": null,
274
+ "overflow_x": null,
275
+ "overflow_y": null,
276
+ "padding": null,
277
+ "right": null,
278
+ "top": null,
279
+ "visibility": null,
280
+ "width": null
281
+ }
282
+ },
283
+ "4896f44a58674d5b9414bcb695b4cc61": {
284
+ "model_module": "@jupyter-widgets/controls",
285
+ "model_name": "ProgressStyleModel",
286
+ "model_module_version": "1.5.0",
287
+ "state": {
288
+ "_model_module": "@jupyter-widgets/controls",
289
+ "_model_module_version": "1.5.0",
290
+ "_model_name": "ProgressStyleModel",
291
+ "_view_count": null,
292
+ "_view_module": "@jupyter-widgets/base",
293
+ "_view_module_version": "1.2.0",
294
+ "_view_name": "StyleView",
295
+ "bar_color": null,
296
+ "description_width": ""
297
+ }
298
+ },
299
+ "40fe6d49d9ad4f8b8374a91b01610fa4": {
300
+ "model_module": "@jupyter-widgets/base",
301
+ "model_name": "LayoutModel",
302
+ "model_module_version": "1.2.0",
303
+ "state": {
304
+ "_model_module": "@jupyter-widgets/base",
305
+ "_model_module_version": "1.2.0",
306
+ "_model_name": "LayoutModel",
307
+ "_view_count": null,
308
+ "_view_module": "@jupyter-widgets/base",
309
+ "_view_module_version": "1.2.0",
310
+ "_view_name": "LayoutView",
311
+ "align_content": null,
312
+ "align_items": null,
313
+ "align_self": null,
314
+ "border": null,
315
+ "bottom": null,
316
+ "display": null,
317
+ "flex": null,
318
+ "flex_flow": null,
319
+ "grid_area": null,
320
+ "grid_auto_columns": null,
321
+ "grid_auto_flow": null,
322
+ "grid_auto_rows": null,
323
+ "grid_column": null,
324
+ "grid_gap": null,
325
+ "grid_row": null,
326
+ "grid_template_areas": null,
327
+ "grid_template_columns": null,
328
+ "grid_template_rows": null,
329
+ "height": null,
330
+ "justify_content": null,
331
+ "justify_items": null,
332
+ "left": null,
333
+ "margin": null,
334
+ "max_height": null,
335
+ "max_width": null,
336
+ "min_height": null,
337
+ "min_width": null,
338
+ "object_fit": null,
339
+ "object_position": null,
340
+ "order": null,
341
+ "overflow": null,
342
+ "overflow_x": null,
343
+ "overflow_y": null,
344
+ "padding": null,
345
+ "right": null,
346
+ "top": null,
347
+ "visibility": null,
348
+ "width": null
349
+ }
350
+ },
351
+ "7c691ef800d6445ebb896559dcae8931": {
352
+ "model_module": "@jupyter-widgets/controls",
353
+ "model_name": "DescriptionStyleModel",
354
+ "model_module_version": "1.5.0",
355
+ "state": {
356
+ "_model_module": "@jupyter-widgets/controls",
357
+ "_model_module_version": "1.5.0",
358
+ "_model_name": "DescriptionStyleModel",
359
+ "_view_count": null,
360
+ "_view_module": "@jupyter-widgets/base",
361
+ "_view_module_version": "1.2.0",
362
+ "_view_name": "StyleView",
363
+ "description_width": ""
364
+ }
365
+ }
366
+ }
367
+ }
368
  }
369
  },
370
  "cells": [
 
384
  "drive.mount('/content/drive')"
385
  ],
386
  "metadata": {
387
+ "id": "sXBKnyLI0_L2",
388
+ "colab": {
389
+ "base_uri": "https://localhost:8080/"
390
+ },
391
+ "outputId": "94b650c4-d5e3-4d44-d242-68507fa897ae"
392
  },
393
+ "execution_count": 2,
394
+ "outputs": [
395
+ {
396
+ "output_type": "stream",
397
+ "name": "stdout",
398
+ "text": [
399
+ "Mounted at /content/drive\n"
400
+ ]
401
+ }
402
+ ]
403
  },
404
  {
405
  "cell_type": "code",
 
1229
  },
1230
  "execution_count": null,
1231
  "outputs": []
1232
+ },
1233
+ {
1234
+ "cell_type": "markdown",
1235
+ "source": [
1236
+ "# CIVIT HAR extract"
1237
+ ],
1238
+ "metadata": {
1239
+ "id": "5XH_IsQ7RwpE"
1240
+ }
1241
+ },
1242
+ {
1243
+ "cell_type": "code",
1244
+ "source": [
1245
+ "# ================================================================\n",
1246
+ "# CIVITAI HAR -> REQUEST / IMAGE INDEX\n",
1247
+ "#\n",
1248
+ "# PURPOSE:\n",
1249
+ "#\n",
1250
+ "# Build a reusable JSON index where:\n",
1251
+ "#\n",
1252
+ "# KEY:\n",
1253
+ "# Information required to make an HTTP request to retrieve\n",
1254
+ "# the image's generation metadata / prompt.\n",
1255
+ "#\n",
1256
+ "# VALUE:\n",
1257
+ "# Information required to retrieve the corresponding image.\n",
1258
+ "#\n",
1259
+ "# INPUT:\n",
1260
+ "# All *.har files directly inside:\n",
1261
+ "# /content/drive/MyDrive/\n",
1262
+ "#\n",
1263
+ "# OUTPUT:\n",
1264
+ "# /content/drive/MyDrive/civitai_request_image_index.json\n",
1265
+ "#\n",
1266
+ "# FEATURES:\n",
1267
+ "# - Processes ALL top-level HAR files\n",
1268
+ "# - Merges gallery records from all HAR files\n",
1269
+ "# - Deduplicates by image ID\n",
1270
+ "# - Collects actual image request URLs from HAR entries\n",
1271
+ "# - Preserves Civitai URL tokens\n",
1272
+ "# - Builds exact tRPC request parameters\n",
1273
+ "# - No HTTP requests\n",
1274
+ "# - No image downloads\n",
1275
+ "# - No prompt fetching\n",
1276
+ "# ================================================================\n",
1277
+ "\n",
1278
+ "\n",
1279
+ "# ================================================================\n",
1280
+ "# 1. IMPORTS\n",
1281
+ "# ================================================================\n",
1282
+ "\n",
1283
+ "import os\n",
1284
+ "import re\n",
1285
+ "import json\n",
1286
+ "import base64\n",
1287
+ "import hashlib\n",
1288
+ "\n",
1289
+ "from pathlib import Path\n",
1290
+ "from urllib.parse import (\n",
1291
+ " urlparse,\n",
1292
+ " parse_qs,\n",
1293
+ " unquote\n",
1294
+ ")\n",
1295
+ "\n",
1296
+ "from datetime import datetime, timezone\n",
1297
+ "\n",
1298
+ "from tqdm.auto import tqdm\n",
1299
+ "\n",
1300
+ "\n",
1301
+ "# ================================================================\n",
1302
+ "# 2. GOOGLE DRIVE\n",
1303
+ "# ================================================================\n",
1304
+ "\n",
1305
+ "from google.colab import drive\n",
1306
+ "\n",
1307
+ "DRIVE_PATH = Path(\"/content/drive\")\n",
1308
+ "\n",
1309
+ "if not os.path.ismount(str(DRIVE_PATH / \"MyDrive\")):\n",
1310
+ "\n",
1311
+ " drive.mount(\n",
1312
+ " \"/content/drive\",\n",
1313
+ " force_remount=False\n",
1314
+ " )\n",
1315
+ "\n",
1316
+ "HAR_DIRECTORY = Path(\"/content/drive/MyDrive\")\n",
1317
+ "\n",
1318
+ "if not HAR_DIRECTORY.exists():\n",
1319
+ " raise FileNotFoundError(\n",
1320
+ " f\"Google Drive directory not found: {HAR_DIRECTORY}\"\n",
1321
+ " )\n",
1322
+ "\n",
1323
+ "\n",
1324
+ "# ================================================================\n",
1325
+ "# 3. CONFIGURATION\n",
1326
+ "# ================================================================\n",
1327
+ "\n",
1328
+ "# Only search the top level of MyDrive.\n",
1329
+ "# No recursive search.\n",
1330
+ "\n",
1331
+ "HAR_FILES = sorted(\n",
1332
+ " HAR_DIRECTORY.glob(\"*.har\")\n",
1333
+ ")\n",
1334
+ "\n",
1335
+ "OUTPUT_FILE = (\n",
1336
+ " HAR_DIRECTORY / \"civitai_request_image_index.json\"\n",
1337
+ ")\n",
1338
+ "\n",
1339
+ "# Generation endpoint.\n",
1340
+ "GENERATION_ENDPOINT_PATH = (\n",
1341
+ " \"/api/trpc/image.getGenerationData\"\n",
1342
+ ")\n",
1343
+ "\n",
1344
+ "# Parameter used by the previous working collector.\n",
1345
+ "SEND_AUTHED_PARAMETER = True\n",
1346
+ "\n",
1347
+ "# Include actual image URLs observed in HAR requests.\n",
1348
+ "COLLECT_IMAGE_REQUEST_URLS = True\n",
1349
+ "\n",
1350
+ "# Preserve all unique observed image URL variants.\n",
1351
+ "KEEP_ALL_IMAGE_URL_VARIANTS = True\n",
1352
+ "\n",
1353
+ "\n",
1354
+ "print(\"==============================================\")\n",
1355
+ "print(\"CIVITAI HAR INDEX BUILDER\")\n",
1356
+ "print(\"==============================================\")\n",
1357
+ "\n",
1358
+ "print(\"HAR directory:\", HAR_DIRECTORY)\n",
1359
+ "print(\"HAR files found:\", len(HAR_FILES))\n",
1360
+ "\n",
1361
+ "for file in HAR_FILES:\n",
1362
+ " print(\" -\", file.name)\n",
1363
+ "\n",
1364
+ "if not HAR_FILES:\n",
1365
+ " raise FileNotFoundError(\n",
1366
+ " \"No .har files found directly inside Google Drive MyDrive.\"\n",
1367
+ " )\n",
1368
+ "\n",
1369
+ "\n",
1370
+ "# ================================================================\n",
1371
+ "# 4. GENERAL HELPERS\n",
1372
+ "# ================================================================\n",
1373
+ "\n",
1374
+ "def utc_now():\n",
1375
+ "\n",
1376
+ " return datetime.now(\n",
1377
+ " timezone.utc\n",
1378
+ " ).isoformat()\n",
1379
+ "\n",
1380
+ "\n",
1381
+ "def decode_har_content(content):\n",
1382
+ "\n",
1383
+ " text = content.get(\"text\", \"\")\n",
1384
+ "\n",
1385
+ " if not text:\n",
1386
+ " return \"\"\n",
1387
+ "\n",
1388
+ " if content.get(\"encoding\", \"\").lower() == \"base64\":\n",
1389
+ "\n",
1390
+ " try:\n",
1391
+ "\n",
1392
+ " return base64.b64decode(\n",
1393
+ " text\n",
1394
+ " ).decode(\n",
1395
+ " \"utf-8\",\n",
1396
+ " errors=\"replace\"\n",
1397
+ " )\n",
1398
+ "\n",
1399
+ " except Exception:\n",
1400
+ " return \"\"\n",
1401
+ "\n",
1402
+ " return text\n",
1403
+ "\n",
1404
+ "\n",
1405
+ "def get_entry_url(entry):\n",
1406
+ "\n",
1407
+ " return (\n",
1408
+ " entry.get(\"request\", {})\n",
1409
+ " .get(\"url\", \"\")\n",
1410
+ " )\n",
1411
+ "\n",
1412
+ "\n",
1413
+ "def parse_entry_response(entry):\n",
1414
+ "\n",
1415
+ " content = (\n",
1416
+ " entry.get(\"response\", {})\n",
1417
+ " .get(\"content\", {})\n",
1418
+ " )\n",
1419
+ "\n",
1420
+ " text = decode_har_content(content)\n",
1421
+ "\n",
1422
+ " if not text:\n",
1423
+ " return None\n",
1424
+ "\n",
1425
+ " try:\n",
1426
+ " return json.loads(text)\n",
1427
+ "\n",
1428
+ " except Exception:\n",
1429
+ " return None\n",
1430
+ "\n",
1431
+ "\n",
1432
+ "def get_request_input(entry):\n",
1433
+ "\n",
1434
+ " url = get_entry_url(entry)\n",
1435
+ "\n",
1436
+ " try:\n",
1437
+ "\n",
1438
+ " parsed = urlparse(url)\n",
1439
+ "\n",
1440
+ " query = parse_qs(\n",
1441
+ " parsed.query\n",
1442
+ " )\n",
1443
+ "\n",
1444
+ " raw_input = query.get(\n",
1445
+ " \"input\",\n",
1446
+ " [None]\n",
1447
+ " )[0]\n",
1448
+ "\n",
1449
+ " if not raw_input:\n",
1450
+ " return {}\n",
1451
+ "\n",
1452
+ " return json.loads(\n",
1453
+ " raw_input\n",
1454
+ " )\n",
1455
+ "\n",
1456
+ " except Exception:\n",
1457
+ " return {}\n",
1458
+ "\n",
1459
+ "\n",
1460
+ "def get_image_id_from_request(entry):\n",
1461
+ "\n",
1462
+ " request_input = get_request_input(\n",
1463
+ " entry\n",
1464
+ " )\n",
1465
+ "\n",
1466
+ " try:\n",
1467
+ "\n",
1468
+ " return int(\n",
1469
+ " request_input[\"json\"][\"id\"]\n",
1470
+ " )\n",
1471
+ "\n",
1472
+ " except Exception:\n",
1473
+ " return None\n",
1474
+ "\n",
1475
+ "\n",
1476
+ "def is_http_url(value):\n",
1477
+ "\n",
1478
+ " if not isinstance(value, str):\n",
1479
+ " return False\n",
1480
+ "\n",
1481
+ " return value.startswith(\n",
1482
+ " (\"http://\", \"https://\")\n",
1483
+ " )\n",
1484
+ "\n",
1485
+ "\n",
1486
+ "def unique_append(target, value):\n",
1487
+ "\n",
1488
+ " if not value:\n",
1489
+ " return\n",
1490
+ "\n",
1491
+ " if value not in target:\n",
1492
+ " target.append(value)\n",
1493
+ "\n",
1494
+ "\n",
1495
+ "# ================================================================\n",
1496
+ "# 5. tRPC RESPONSE DECODER\n",
1497
+ "# ================================================================\n",
1498
+ "\n",
1499
+ "def get_trpc_data(payload):\n",
1500
+ "\n",
1501
+ " if not isinstance(payload, dict):\n",
1502
+ " return None\n",
1503
+ "\n",
1504
+ " result = payload.get(\"result\")\n",
1505
+ "\n",
1506
+ " if not isinstance(result, dict):\n",
1507
+ " return None\n",
1508
+ "\n",
1509
+ " data = result.get(\"data\")\n",
1510
+ "\n",
1511
+ " if isinstance(data, str):\n",
1512
+ "\n",
1513
+ " try:\n",
1514
+ " data = json.loads(data)\n",
1515
+ "\n",
1516
+ " except Exception:\n",
1517
+ " return None\n",
1518
+ "\n",
1519
+ " for _ in range(5):\n",
1520
+ "\n",
1521
+ " if not isinstance(data, dict):\n",
1522
+ " break\n",
1523
+ "\n",
1524
+ " if \"json\" in data:\n",
1525
+ "\n",
1526
+ " data = data[\"json\"]\n",
1527
+ "\n",
1528
+ " if isinstance(data, str):\n",
1529
+ "\n",
1530
+ " try:\n",
1531
+ " data = json.loads(data)\n",
1532
+ "\n",
1533
+ " except Exception:\n",
1534
+ " return None\n",
1535
+ "\n",
1536
+ " continue\n",
1537
+ "\n",
1538
+ " break\n",
1539
+ "\n",
1540
+ " return data\n",
1541
+ "\n",
1542
+ "\n",
1543
+ "# ================================================================\n",
1544
+ "# 6. COMPACT ARRAY DECODER\n",
1545
+ "# ================================================================\n",
1546
+ "\n",
1547
+ "def dereference_once(packed, value):\n",
1548
+ "\n",
1549
+ " if isinstance(value, bool):\n",
1550
+ " return value\n",
1551
+ "\n",
1552
+ " if not isinstance(value, int):\n",
1553
+ " return value\n",
1554
+ "\n",
1555
+ " if value == -1:\n",
1556
+ " return None\n",
1557
+ "\n",
1558
+ " if 0 <= value < len(packed):\n",
1559
+ " return packed[value]\n",
1560
+ "\n",
1561
+ " return value\n",
1562
+ "\n",
1563
+ "\n",
1564
+ "def decode_compact_node(\n",
1565
+ " packed,\n",
1566
+ " node,\n",
1567
+ " depth=0\n",
1568
+ "):\n",
1569
+ "\n",
1570
+ " if depth > 30:\n",
1571
+ " return node\n",
1572
+ "\n",
1573
+ " if isinstance(node, dict):\n",
1574
+ "\n",
1575
+ " result = {}\n",
1576
+ "\n",
1577
+ " for key, value in node.items():\n",
1578
+ "\n",
1579
+ " resolved = dereference_once(\n",
1580
+ " packed,\n",
1581
+ " value\n",
1582
+ " )\n",
1583
+ "\n",
1584
+ " if isinstance(\n",
1585
+ " resolved,\n",
1586
+ " (dict, list)\n",
1587
+ " ):\n",
1588
+ "\n",
1589
+ " resolved = decode_compact_node(\n",
1590
+ " packed,\n",
1591
+ " resolved,\n",
1592
+ " depth + 1\n",
1593
+ " )\n",
1594
+ "\n",
1595
+ " result[key] = resolved\n",
1596
+ "\n",
1597
+ " return result\n",
1598
+ "\n",
1599
+ " if isinstance(node, list):\n",
1600
+ "\n",
1601
+ " result = []\n",
1602
+ "\n",
1603
+ " for value in node:\n",
1604
+ "\n",
1605
+ " resolved = dereference_once(\n",
1606
+ " packed,\n",
1607
+ " value\n",
1608
+ " )\n",
1609
+ "\n",
1610
+ " if isinstance(\n",
1611
+ " resolved,\n",
1612
+ " (dict, list)\n",
1613
+ " ):\n",
1614
+ "\n",
1615
+ " resolved = decode_compact_node(\n",
1616
+ " packed,\n",
1617
+ " resolved,\n",
1618
+ " depth + 1\n",
1619
+ " )\n",
1620
+ "\n",
1621
+ " result.append(resolved)\n",
1622
+ "\n",
1623
+ " return result\n",
1624
+ "\n",
1625
+ " return node\n",
1626
+ "\n",
1627
+ "\n",
1628
+ "# ================================================================\n",
1629
+ "# 7. EXTRACT GALLERY RECORDS\n",
1630
+ "# ================================================================\n",
1631
+ "\n",
1632
+ "def extract_gallery_records(payload):\n",
1633
+ "\n",
1634
+ " data = get_trpc_data(payload)\n",
1635
+ "\n",
1636
+ " if not isinstance(data, list):\n",
1637
+ " return []\n",
1638
+ "\n",
1639
+ " if not data:\n",
1640
+ " return []\n",
1641
+ "\n",
1642
+ " root = data[0]\n",
1643
+ "\n",
1644
+ " if not isinstance(root, dict):\n",
1645
+ " return []\n",
1646
+ "\n",
1647
+ " decoded_root = decode_compact_node(\n",
1648
+ " data,\n",
1649
+ " root\n",
1650
+ " )\n",
1651
+ "\n",
1652
+ " if not isinstance(decoded_root, dict):\n",
1653
+ " return []\n",
1654
+ "\n",
1655
+ " items = decoded_root.get(\"items\")\n",
1656
+ "\n",
1657
+ " records = []\n",
1658
+ "\n",
1659
+ " if isinstance(items, list):\n",
1660
+ "\n",
1661
+ " for record in items:\n",
1662
+ "\n",
1663
+ " if not isinstance(record, dict):\n",
1664
+ " continue\n",
1665
+ "\n",
1666
+ " image_id = record.get(\"id\")\n",
1667
+ "\n",
1668
+ " if isinstance(\n",
1669
+ " image_id,\n",
1670
+ " (int, float)\n",
1671
+ " ):\n",
1672
+ "\n",
1673
+ " records.append(record)\n",
1674
+ "\n",
1675
+ " if records:\n",
1676
+ " return records\n",
1677
+ "\n",
1678
+ " # Fallback for unresolved item references.\n",
1679
+ " raw_items_index = root.get(\"items\")\n",
1680
+ "\n",
1681
+ " if not isinstance(raw_items_index, int):\n",
1682
+ " return []\n",
1683
+ "\n",
1684
+ " if not (\n",
1685
+ " 0 <= raw_items_index < len(data)\n",
1686
+ " ):\n",
1687
+ " return []\n",
1688
+ "\n",
1689
+ " raw_items = data[raw_items_index]\n",
1690
+ "\n",
1691
+ " if not isinstance(raw_items, list):\n",
1692
+ " return []\n",
1693
+ "\n",
1694
+ " for reference in raw_items:\n",
1695
+ "\n",
1696
+ " if not isinstance(reference, int):\n",
1697
+ " continue\n",
1698
+ "\n",
1699
+ " if not (\n",
1700
+ " 0 <= reference < len(data)\n",
1701
+ " ):\n",
1702
+ " continue\n",
1703
+ "\n",
1704
+ " raw_record = data[reference]\n",
1705
+ "\n",
1706
+ " if not isinstance(raw_record, dict):\n",
1707
+ " continue\n",
1708
+ "\n",
1709
+ " record = decode_compact_node(\n",
1710
+ " data,\n",
1711
+ " raw_record\n",
1712
+ " )\n",
1713
+ "\n",
1714
+ " if not isinstance(record, dict):\n",
1715
+ " continue\n",
1716
+ "\n",
1717
+ " image_id = record.get(\"id\")\n",
1718
+ "\n",
1719
+ " if isinstance(\n",
1720
+ " image_id,\n",
1721
+ " (int, float)\n",
1722
+ " ):\n",
1723
+ "\n",
1724
+ " records.append(record)\n",
1725
+ "\n",
1726
+ " return records\n",
1727
+ "\n",
1728
+ "\n",
1729
+ "# ================================================================\n",
1730
+ "# 8. NORMALIZE GALLERY RECORD\n",
1731
+ "# ================================================================\n",
1732
+ "\n",
1733
+ "def normalize_gallery_record(record):\n",
1734
+ "\n",
1735
+ " image_id = record.get(\"id\")\n",
1736
+ "\n",
1737
+ " user = record.get(\"user\")\n",
1738
+ "\n",
1739
+ " username = None\n",
1740
+ "\n",
1741
+ " if isinstance(user, dict):\n",
1742
+ " username = user.get(\"username\")\n",
1743
+ "\n",
1744
+ " url_value = record.get(\"url\")\n",
1745
+ "\n",
1746
+ " direct_url = (\n",
1747
+ " url_value\n",
1748
+ " if is_http_url(url_value)\n",
1749
+ " else None\n",
1750
+ " )\n",
1751
+ "\n",
1752
+ " url_token = (\n",
1753
+ " None\n",
1754
+ " if direct_url\n",
1755
+ " else url_value\n",
1756
+ " )\n",
1757
+ "\n",
1758
+ " return {\n",
1759
+ "\n",
1760
+ " \"image_id\": int(image_id),\n",
1761
+ "\n",
1762
+ " \"name\": record.get(\"name\"),\n",
1763
+ "\n",
1764
+ " \"image_url\": direct_url,\n",
1765
+ "\n",
1766
+ " \"url_token\": url_token,\n",
1767
+ "\n",
1768
+ " \"width\": record.get(\"width\"),\n",
1769
+ "\n",
1770
+ " \"height\": record.get(\"height\"),\n",
1771
+ "\n",
1772
+ " \"hash\": record.get(\"hash\"),\n",
1773
+ "\n",
1774
+ " \"nsfw_level\": record.get(\"nsfwLevel\"),\n",
1775
+ "\n",
1776
+ " \"has_meta\": record.get(\"hasMeta\"),\n",
1777
+ "\n",
1778
+ " \"has_positive_prompt\": record.get(\n",
1779
+ " \"hasPositivePrompt\"\n",
1780
+ " ),\n",
1781
+ "\n",
1782
+ " \"created_at\": record.get(\"createdAt\"),\n",
1783
+ "\n",
1784
+ " \"published_at\": record.get(\"publishedAt\"),\n",
1785
+ "\n",
1786
+ " \"username\": username,\n",
1787
+ "\n",
1788
+ " \"model_version_ids\": record.get(\n",
1789
+ " \"modelVersionIds\"\n",
1790
+ " ),\n",
1791
+ "\n",
1792
+ " \"raw_gallery_record\": record\n",
1793
+ " }\n",
1794
+ "\n",
1795
+ "\n",
1796
+ "# ================================================================\n",
1797
+ "# 9. COLLECT IMAGE URL REQUESTS FROM HAR\n",
1798
+ "# ================================================================\n",
1799
+ "\n",
1800
+ "def looks_like_image_request(entry):\n",
1801
+ "\n",
1802
+ " request_url = get_entry_url(entry)\n",
1803
+ "\n",
1804
+ " if not request_url:\n",
1805
+ " return False\n",
1806
+ "\n",
1807
+ " parsed = urlparse(request_url)\n",
1808
+ "\n",
1809
+ " response = entry.get(\n",
1810
+ " \"response\",\n",
1811
+ " {}\n",
1812
+ " )\n",
1813
+ "\n",
1814
+ " content = response.get(\n",
1815
+ " \"content\",\n",
1816
+ " {}\n",
1817
+ " )\n",
1818
+ "\n",
1819
+ " mime_type = (\n",
1820
+ " content.get(\"mimeType\", \"\")\n",
1821
+ " .lower()\n",
1822
+ " )\n",
1823
+ "\n",
1824
+ " # Image content type.\n",
1825
+ " if mime_type.startswith(\"image/\"):\n",
1826
+ " return True\n",
1827
+ "\n",
1828
+ " # Common image file extensions.\n",
1829
+ " path = parsed.path.lower()\n",
1830
+ "\n",
1831
+ " if re.search(\n",
1832
+ " r\"\\.(jpg|jpeg|png|webp|gif|avif|bmp)(?:$)\",\n",
1833
+ " path\n",
1834
+ " ):\n",
1835
+ " return True\n",
1836
+ "\n",
1837
+ " # Common Civitai image CDN host.\n",
1838
+ " if \"image.civitai.com\" in parsed.netloc.lower():\n",
1839
+ " return True\n",
1840
+ "\n",
1841
+ " return False\n",
1842
+ "\n",
1843
+ "\n",
1844
+ "def get_image_request_details(entry):\n",
1845
+ "\n",
1846
+ " request = entry.get(\n",
1847
+ " \"request\",\n",
1848
+ " {}\n",
1849
+ " )\n",
1850
+ "\n",
1851
+ " response = entry.get(\n",
1852
+ " \"response\",\n",
1853
+ " {}\n",
1854
+ " )\n",
1855
+ "\n",
1856
+ " content = response.get(\n",
1857
+ " \"content\",\n",
1858
+ " {}\n",
1859
+ " )\n",
1860
+ "\n",
1861
+ " request_url = request.get(\n",
1862
+ " \"url\"\n",
1863
+ " )\n",
1864
+ "\n",
1865
+ " if not request_url:\n",
1866
+ " return None\n",
1867
+ "\n",
1868
+ " return {\n",
1869
+ "\n",
1870
+ " \"url\": request_url,\n",
1871
+ "\n",
1872
+ " \"method\": request.get(\n",
1873
+ " \"method\",\n",
1874
+ " \"GET\"\n",
1875
+ " ),\n",
1876
+ "\n",
1877
+ " \"mime_type\": content.get(\n",
1878
+ " \"mimeType\"\n",
1879
+ " ),\n",
1880
+ "\n",
1881
+ " \"http_status\": response.get(\n",
1882
+ " \"status\"\n",
1883
+ " ),\n",
1884
+ "\n",
1885
+ " \"response_size\": content.get(\n",
1886
+ " \"size\"\n",
1887
+ " ),\n",
1888
+ "\n",
1889
+ " \"source\": \"HAR_image_request\"\n",
1890
+ " }\n",
1891
+ "\n",
1892
+ "\n",
1893
+ "# ================================================================\n",
1894
+ "# 10. BUILD GLOBAL IMAGE INDEX\n",
1895
+ "# ================================================================\n",
1896
+ "\n",
1897
+ "images_by_id = {}\n",
1898
+ "\n",
1899
+ "image_request_urls = {}\n",
1900
+ "\n",
1901
+ "origins_by_image_id = {}\n",
1902
+ "\n",
1903
+ "har_statistics = []\n",
1904
+ "\n",
1905
+ "all_civitai_origins = set()\n",
1906
+ "\n",
1907
+ "print(\"\\n==============================================\")\n",
1908
+ "print(\"PROCESSING HAR FILES\")\n",
1909
+ "print(\"==============================================\")\n",
1910
+ "\n",
1911
+ "\n",
1912
+ "for har_file in tqdm(\n",
1913
+ " HAR_FILES,\n",
1914
+ " desc=\"HAR files\"\n",
1915
+ "):\n",
1916
+ "\n",
1917
+ " print(\n",
1918
+ " f\"\\nProcessing: {har_file.name}\"\n",
1919
+ " )\n",
1920
+ "\n",
1921
+ " try:\n",
1922
+ "\n",
1923
+ " with open(\n",
1924
+ " har_file,\n",
1925
+ " \"r\",\n",
1926
+ " encoding=\"utf-8\"\n",
1927
+ " ) as f:\n",
1928
+ "\n",
1929
+ " har_data = json.load(f)\n",
1930
+ "\n",
1931
+ " except Exception as exc:\n",
1932
+ "\n",
1933
+ " print(\n",
1934
+ " \"Could not read HAR:\",\n",
1935
+ " exc\n",
1936
+ " )\n",
1937
+ "\n",
1938
+ " continue\n",
1939
+ "\n",
1940
+ " entries = (\n",
1941
+ " har_data.get(\"log\", {})\n",
1942
+ " .get(\"entries\", [])\n",
1943
+ " )\n",
1944
+ "\n",
1945
+ " gallery_records_found = 0\n",
1946
+ " image_requests_found = 0\n",
1947
+ "\n",
1948
+ " # ------------------------------------------------------------\n",
1949
+ " # First pass: collect all image request URLs.\n",
1950
+ " # ------------------------------------------------------------\n",
1951
+ "\n",
1952
+ " if COLLECT_IMAGE_REQUEST_URLS:\n",
1953
+ "\n",
1954
+ " for entry in entries:\n",
1955
+ "\n",
1956
+ " if not looks_like_image_request(entry):\n",
1957
+ " continue\n",
1958
+ "\n",
1959
+ " details = get_image_request_details(\n",
1960
+ " entry\n",
1961
+ " )\n",
1962
+ "\n",
1963
+ " if not details:\n",
1964
+ " continue\n",
1965
+ "\n",
1966
+ " image_requests_found += 1\n",
1967
+ "\n",
1968
+ " unique_append(\n",
1969
+ " image_request_urls.setdefault(\n",
1970
+ " \"__all__\",\n",
1971
+ " []\n",
1972
+ " ),\n",
1973
+ " details[\"url\"]\n",
1974
+ " )\n",
1975
+ "\n",
1976
+ " # ------------------------------------------------------------\n",
1977
+ " # Second pass: gallery records and origin detection.\n",
1978
+ " # ------------------------------------------------------------\n",
1979
+ "\n",
1980
+ " for entry in entries:\n",
1981
+ "\n",
1982
+ " request_url = get_entry_url(entry)\n",
1983
+ "\n",
1984
+ " if not request_url:\n",
1985
+ " continue\n",
1986
+ "\n",
1987
+ " parsed = urlparse(request_url)\n",
1988
+ "\n",
1989
+ " if (\n",
1990
+ " \"image.getInfinite\" in parsed.path\n",
1991
+ " or \"image.getGenerationData\" in parsed.path\n",
1992
+ " ):\n",
1993
+ "\n",
1994
+ " origin = (\n",
1995
+ " f\"{parsed.scheme}://{parsed.netloc}\"\n",
1996
+ " )\n",
1997
+ "\n",
1998
+ " all_civitai_origins.add(origin)\n",
1999
+ "\n",
2000
+ " # --------------------------------------------------------\n",
2001
+ " # Gallery records.\n",
2002
+ " # --------------------------------------------------------\n",
2003
+ "\n",
2004
+ " if \"image.getInfinite\" not in parsed.path:\n",
2005
+ " continue\n",
2006
+ "\n",
2007
+ " payload = parse_entry_response(entry)\n",
2008
+ "\n",
2009
+ " if payload is None:\n",
2010
+ " continue\n",
2011
+ "\n",
2012
+ " records = extract_gallery_records(\n",
2013
+ " payload\n",
2014
+ " )\n",
2015
+ "\n",
2016
+ " for record in records:\n",
2017
+ "\n",
2018
+ " normalized = normalize_gallery_record(\n",
2019
+ " record\n",
2020
+ " )\n",
2021
+ "\n",
2022
+ " image_id = normalized[\"image_id\"]\n",
2023
+ "\n",
2024
+ " gallery_records_found += 1\n",
2025
+ "\n",
2026
+ " # Store the origin used by this gallery response.\n",
2027
+ " origin = (\n",
2028
+ " f\"{parsed.scheme}://{parsed.netloc}\"\n",
2029
+ " )\n",
2030
+ "\n",
2031
+ " origins_by_image_id.setdefault(\n",
2032
+ " image_id,\n",
2033
+ " []\n",
2034
+ " )\n",
2035
+ "\n",
2036
+ " unique_append(\n",
2037
+ " origins_by_image_id[image_id],\n",
2038
+ " origin\n",
2039
+ " )\n",
2040
+ "\n",
2041
+ " # ----------------------------------------------------\n",
2042
+ " # Deduplicate by image ID.\n",
2043
+ " # ----------------------------------------------------\n",
2044
+ "\n",
2045
+ " if image_id not in images_by_id:\n",
2046
+ "\n",
2047
+ " images_by_id[image_id] = {\n",
2048
+ " **normalized,\n",
2049
+ "\n",
2050
+ " \"source_har_files\": [],\n",
2051
+ "\n",
2052
+ " \"observed_image_requests\": []\n",
2053
+ " }\n",
2054
+ "\n",
2055
+ " existing = images_by_id[image_id]\n",
2056
+ "\n",
2057
+ " # Merge useful metadata without replacing existing\n",
2058
+ " # non-empty values with missing ones.\n",
2059
+ "\n",
2060
+ " for key, value in normalized.items():\n",
2061
+ "\n",
2062
+ " if key == \"image_id\":\n",
2063
+ " continue\n",
2064
+ "\n",
2065
+ " if value is None:\n",
2066
+ " continue\n",
2067
+ "\n",
2068
+ " if existing.get(key) in (\n",
2069
+ " None,\n",
2070
+ " \"\",\n",
2071
+ " []\n",
2072
+ " ):\n",
2073
+ "\n",
2074
+ " existing[key] = value\n",
2075
+ "\n",
2076
+ " unique_append(\n",
2077
+ " existing[\"source_har_files\"],\n",
2078
+ " har_file.name\n",
2079
+ " )\n",
2080
+ "\n",
2081
+ " # ------------------------------------------------------------\n",
2082
+ " # Per-file statistics.\n",
2083
+ " # ------------------------------------------------------------\n",
2084
+ "\n",
2085
+ " har_statistics.append({\n",
2086
+ "\n",
2087
+ " \"filename\": har_file.name,\n",
2088
+ "\n",
2089
+ " \"entries\": len(entries),\n",
2090
+ "\n",
2091
+ " \"gallery_records\": gallery_records_found,\n",
2092
+ "\n",
2093
+ " \"image_requests\": image_requests_found\n",
2094
+ " })\n",
2095
+ "\n",
2096
+ "\n",
2097
+ "# ================================================================\n",
2098
+ "# 11. MATCH OBSERVED IMAGE URLS TO GALLERY RECORDS\n",
2099
+ "# ================================================================\n",
2100
+ "\n",
2101
+ "print(\"\\nMatching image requests to gallery records...\")\n",
2102
+ "\n",
2103
+ "all_image_urls = image_request_urls.get(\n",
2104
+ " \"__all__\",\n",
2105
+ " []\n",
2106
+ ")\n",
2107
+ "\n",
2108
+ "for image_id, image_record in images_by_id.items():\n",
2109
+ "\n",
2110
+ " token = image_record.get(\n",
2111
+ " \"url_token\"\n",
2112
+ " )\n",
2113
+ "\n",
2114
+ " direct_url = image_record.get(\n",
2115
+ " \"image_url\"\n",
2116
+ " )\n",
2117
+ "\n",
2118
+ " matched_urls = []\n",
2119
+ "\n",
2120
+ " # ------------------------------------------------------------\n",
2121
+ " # A. Direct image URL from the gallery record.\n",
2122
+ " # ------------------------------------------------------------\n",
2123
+ "\n",
2124
+ " if direct_url:\n",
2125
+ "\n",
2126
+ " unique_append(\n",
2127
+ " matched_urls,\n",
2128
+ " direct_url\n",
2129
+ " )\n",
2130
+ "\n",
2131
+ " # ------------------------------------------------------------\n",
2132
+ " # B. Match using the URL token.\n",
2133
+ " # ------------------------------------------------------------\n",
2134
+ "\n",
2135
+ " if token:\n",
2136
+ "\n",
2137
+ " token_string = str(token)\n",
2138
+ "\n",
2139
+ " for request_url in all_image_urls:\n",
2140
+ "\n",
2141
+ " if (\n",
2142
+ " token_string in request_url\n",
2143
+ " or unquote(token_string) in unquote(request_url)\n",
2144
+ " ):\n",
2145
+ "\n",
2146
+ " unique_append(\n",
2147
+ " matched_urls,\n",
2148
+ " request_url\n",
2149
+ " )\n",
2150
+ "\n",
2151
+ " # ------------------------------------------------------------\n",
2152
+ " # C. Match by image ID if present in an image request URL.\n",
2153
+ " # ------------------------------------------------------------\n",
2154
+ "\n",
2155
+ " for request_url in all_image_urls:\n",
2156
+ "\n",
2157
+ " parsed = urlparse(request_url)\n",
2158
+ "\n",
2159
+ " query_string = unquote(\n",
2160
+ " parsed.query\n",
2161
+ " )\n",
2162
+ "\n",
2163
+ " path_string = unquote(\n",
2164
+ " parsed.path\n",
2165
+ " )\n",
2166
+ "\n",
2167
+ " if (\n",
2168
+ " str(image_id) in query_string\n",
2169
+ " or str(image_id) in path_string\n",
2170
+ " ):\n",
2171
+ "\n",
2172
+ " unique_append(\n",
2173
+ " matched_urls,\n",
2174
+ " request_url\n",
2175
+ " )\n",
2176
+ "\n",
2177
+ " image_record[\"observed_image_requests\"] = (\n",
2178
+ " matched_urls\n",
2179
+ " )\n",
2180
+ "\n",
2181
+ " # Prefer an actual observed URL when available.\n",
2182
+ " if matched_urls:\n",
2183
+ "\n",
2184
+ " image_record[\"image_url\"] = (\n",
2185
+ " matched_urls[0]\n",
2186
+ " )\n",
2187
+ "\n",
2188
+ " image_record[\"image_url_status\"] = (\n",
2189
+ " \"observed_in_har\"\n",
2190
+ " )\n",
2191
+ "\n",
2192
+ " elif direct_url:\n",
2193
+ "\n",
2194
+ " image_record[\"image_url_status\"] = (\n",
2195
+ " \"gallery_record_url\"\n",
2196
+ " )\n",
2197
+ "\n",
2198
+ " elif token:\n",
2199
+ "\n",
2200
+ " image_record[\"image_url_status\"] = (\n",
2201
+ " \"url_token_only\"\n",
2202
+ " )\n",
2203
+ "\n",
2204
+ " else:\n",
2205
+ "\n",
2206
+ " image_record[\"image_url_status\"] = (\n",
2207
+ " \"no_image_url_found\"\n",
2208
+ " )\n",
2209
+ "\n",
2210
+ "\n",
2211
+ "# ================================================================\n",
2212
+ "# 12. BUILD REQUEST KEY\n",
2213
+ "# ================================================================\n",
2214
+ "\n",
2215
+ "def build_prompt_request_key(\n",
2216
+ " image_id,\n",
2217
+ " origin\n",
2218
+ "):\n",
2219
+ "\n",
2220
+ " request_json = {\n",
2221
+ " \"id\": int(image_id)\n",
2222
+ " }\n",
2223
+ "\n",
2224
+ " if SEND_AUTHED_PARAMETER:\n",
2225
+ " request_json[\"authed\"] = True\n",
2226
+ "\n",
2227
+ " return {\n",
2228
+ "\n",
2229
+ " \"method\": \"GET\",\n",
2230
+ "\n",
2231
+ " \"url\": (\n",
2232
+ " origin\n",
2233
+ " + GENERATION_ENDPOINT_PATH\n",
2234
+ " ),\n",
2235
+ "\n",
2236
+ " \"params\": {\n",
2237
+ "\n",
2238
+ " \"input\": json.dumps(\n",
2239
+ " {\n",
2240
+ " \"json\": request_json\n",
2241
+ " },\n",
2242
+ " separators=(\",\", \":\")\n",
2243
+ " )\n",
2244
+ " },\n",
2245
+ "\n",
2246
+ " \"headers\": {\n",
2247
+ "\n",
2248
+ " \"Referer\": (\n",
2249
+ " f\"{origin}/images/{image_id}\"\n",
2250
+ " ),\n",
2251
+ "\n",
2252
+ " \"Accept\": (\n",
2253
+ " \"application/json, text/plain, */*\"\n",
2254
+ " )\n",
2255
+ " }\n",
2256
+ " }\n",
2257
+ "\n",
2258
+ "\n",
2259
+ "# ================================================================\n",
2260
+ "# 13. BUILD FINAL KEY / VALUE RECORDS\n",
2261
+ "# ================================================================\n",
2262
+ "\n",
2263
+ "final_records = []\n",
2264
+ "\n",
2265
+ "seen_image_ids = set()\n",
2266
+ "seen_request_keys = set()\n",
2267
+ "\n",
2268
+ "duplicate_count = 0\n",
2269
+ "\n",
2270
+ "missing_image_url_count = 0\n",
2271
+ "observed_image_url_count = 0\n",
2272
+ "token_only_count = 0\n",
2273
+ "\n",
2274
+ "\n",
2275
+ "for image_id in sorted(images_by_id):\n",
2276
+ "\n",
2277
+ " if image_id in seen_image_ids:\n",
2278
+ "\n",
2279
+ " duplicate_count += 1\n",
2280
+ " continue\n",
2281
+ "\n",
2282
+ " seen_image_ids.add(image_id)\n",
2283
+ "\n",
2284
+ " image = images_by_id[image_id]\n",
2285
+ "\n",
2286
+ " origins = origins_by_image_id.get(\n",
2287
+ " image_id,\n",
2288
+ " []\n",
2289
+ " )\n",
2290
+ "\n",
2291
+ " if origins:\n",
2292
+ " origin = origins[0]\n",
2293
+ "\n",
2294
+ " elif all_civitai_origins:\n",
2295
+ " origin = sorted(all_civitai_origins)[0]\n",
2296
+ "\n",
2297
+ " else:\n",
2298
+ " origin = \"https://civitai.red\"\n",
2299
+ "\n",
2300
+ " # ------------------------------------------------------------\n",
2301
+ " # KEY: HTTP request needed to fetch the prompt.\n",
2302
+ " # ------------------------------------------------------------\n",
2303
+ "\n",
2304
+ " request_key = build_prompt_request_key(\n",
2305
+ " image_id,\n",
2306
+ " origin\n",
2307
+ " )\n",
2308
+ "\n",
2309
+ " # ------------------------------------------------------------\n",
2310
+ " # Deduplicate by exact request.\n",
2311
+ " # ------------------------------------------------------------\n",
2312
+ "\n",
2313
+ " request_signature = json.dumps(\n",
2314
+ " request_key,\n",
2315
+ " sort_keys=True\n",
2316
+ " )\n",
2317
+ "\n",
2318
+ " if request_signature in seen_request_keys:\n",
2319
+ "\n",
2320
+ " duplicate_count += 1\n",
2321
+ " continue\n",
2322
+ "\n",
2323
+ " seen_request_keys.add(\n",
2324
+ " request_signature\n",
2325
+ " )\n",
2326
+ "\n",
2327
+ " # ------------------------------------------------------------\n",
2328
+ " # VALUE: Information needed to fetch the image.\n",
2329
+ " # ------------------------------------------------------------\n",
2330
+ "\n",
2331
+ " image_value = {\n",
2332
+ "\n",
2333
+ " \"image_id\": image_id,\n",
2334
+ "\n",
2335
+ " \"image_url\": image.get(\n",
2336
+ " \"image_url\"\n",
2337
+ " ),\n",
2338
+ "\n",
2339
+ " \"image_url_candidates\": (\n",
2340
+ " image.get(\"observed_image_requests\", [])\n",
2341
+ " if KEEP_ALL_IMAGE_URL_VARIANTS\n",
2342
+ " else image.get(\"observed_image_requests\", [])[:1]\n",
2343
+ " ),\n",
2344
+ "\n",
2345
+ " \"image_url_status\": image.get(\n",
2346
+ " \"image_url_status\"\n",
2347
+ " ),\n",
2348
+ "\n",
2349
+ " \"url_token\": image.get(\n",
2350
+ " \"url_token\"\n",
2351
+ " ),\n",
2352
+ "\n",
2353
+ " \"width\": image.get(\n",
2354
+ " \"width\"\n",
2355
+ " ),\n",
2356
+ "\n",
2357
+ " \"height\": image.get(\n",
2358
+ " \"height\"\n",
2359
+ " ),\n",
2360
+ "\n",
2361
+ " \"hash\": image.get(\n",
2362
+ " \"hash\"\n",
2363
+ " ),\n",
2364
+ "\n",
2365
+ " \"name\": image.get(\n",
2366
+ " \"name\"\n",
2367
+ " ),\n",
2368
+ "\n",
2369
+ " \"username\": image.get(\n",
2370
+ " \"username\"\n",
2371
+ " ),\n",
2372
+ "\n",
2373
+ " \"nsfw_level\": image.get(\n",
2374
+ " \"nsfw_level\"\n",
2375
+ " ),\n",
2376
+ "\n",
2377
+ " \"created_at\": image.get(\n",
2378
+ " \"created_at\"\n",
2379
+ " ),\n",
2380
+ "\n",
2381
+ " \"published_at\": image.get(\n",
2382
+ " \"published_at\"\n",
2383
+ " ),\n",
2384
+ "\n",
2385
+ " \"has_meta\": image.get(\n",
2386
+ " \"has_meta\"\n",
2387
+ " ),\n",
2388
+ "\n",
2389
+ " \"has_positive_prompt\": image.get(\n",
2390
+ " \"has_positive_prompt\"\n",
2391
+ " ),\n",
2392
+ "\n",
2393
+ " \"model_version_ids\": image.get(\n",
2394
+ " \"model_version_ids\"\n",
2395
+ " ),\n",
2396
+ "\n",
2397
+ " \"civitai_page_url\": (\n",
2398
+ " f\"{origin}/images/{image_id}\"\n",
2399
+ " ),\n",
2400
+ "\n",
2401
+ " \"source_har_files\": image.get(\n",
2402
+ " \"source_har_files\",\n",
2403
+ " []\n",
2404
+ " ),\n",
2405
+ "\n",
2406
+ " \"gallery_origin\": origin\n",
2407
+ " }\n",
2408
+ "\n",
2409
+ " if image_value[\"image_url\"]:\n",
2410
+ " observed_image_url_count += 1\n",
2411
+ "\n",
2412
+ " elif image_value[\"url_token\"]:\n",
2413
+ " token_only_count += 1\n",
2414
+ "\n",
2415
+ " else:\n",
2416
+ " missing_image_url_count += 1\n",
2417
+ "\n",
2418
+ " # ------------------------------------------------------------\n",
2419
+ " # Final key/value entry.\n",
2420
+ " # ------------------------------------------------------------\n",
2421
+ "\n",
2422
+ " final_records.append({\n",
2423
+ "\n",
2424
+ " \"key\": request_key,\n",
2425
+ "\n",
2426
+ " \"value\": image_value\n",
2427
+ " })\n",
2428
+ "\n",
2429
+ "\n",
2430
+ "# ================================================================\n",
2431
+ "# 14. FINAL DEDUPLICATION CHECK\n",
2432
+ "# ================================================================\n",
2433
+ "\n",
2434
+ "final_image_ids = [\n",
2435
+ " item[\"value\"][\"image_id\"]\n",
2436
+ " for item in final_records\n",
2437
+ "]\n",
2438
+ "\n",
2439
+ "if len(final_image_ids) != len(set(final_image_ids)):\n",
2440
+ "\n",
2441
+ " raise RuntimeError(\n",
2442
+ " \"Duplicate image IDs remain in the final output.\"\n",
2443
+ " )\n",
2444
+ "\n",
2445
+ "if len(final_records) != len(seen_request_keys):\n",
2446
+ "\n",
2447
+ " raise RuntimeError(\n",
2448
+ " \"Request key deduplication check failed.\"\n",
2449
+ " )\n",
2450
+ "\n",
2451
+ "\n",
2452
+ "# ================================================================\n",
2453
+ "# 15. BUILD JSON DOCUMENT\n",
2454
+ "# ================================================================\n",
2455
+ "\n",
2456
+ "output_data = {\n",
2457
+ "\n",
2458
+ " \"schema_version\": 1,\n",
2459
+ "\n",
2460
+ " \"description\": (\n",
2461
+ " \"Civitai prompt-request to image-fetch index. \"\n",
2462
+ " \"Each record contains a request key and an image value.\"\n",
2463
+ " ),\n",
2464
+ "\n",
2465
+ " \"generated_at\": utc_now(),\n",
2466
+ "\n",
2467
+ " \"processing\": {\n",
2468
+ "\n",
2469
+ " \"input_directory\": str(HAR_DIRECTORY),\n",
2470
+ "\n",
2471
+ " \"recursive_search\": False,\n",
2472
+ "\n",
2473
+ " \"har_files_processed\": len(HAR_FILES),\n",
2474
+ "\n",
2475
+ " \"har_files\": [\n",
2476
+ " file.name\n",
2477
+ " for file in HAR_FILES\n",
2478
+ " ],\n",
2479
+ "\n",
2480
+ " \"http_requests_made\": 0,\n",
2481
+ "\n",
2482
+ " \"images_downloaded\": 0,\n",
2483
+ "\n",
2484
+ " \"prompts_fetched\": 0,\n",
2485
+ "\n",
2486
+ " \"deduplication_key\": \"image_id\"\n",
2487
+ " },\n",
2488
+ "\n",
2489
+ " \"statistics\": {\n",
2490
+ "\n",
2491
+ " \"unique_images\": len(final_records),\n",
2492
+ "\n",
2493
+ " \"duplicate_entries_removed\": duplicate_count,\n",
2494
+ "\n",
2495
+ " \"images_with_observed_image_urls\": (\n",
2496
+ " observed_image_url_count\n",
2497
+ " ),\n",
2498
+ "\n",
2499
+ " \"images_with_token_only\": (\n",
2500
+ " token_only_count\n",
2501
+ " ),\n",
2502
+ "\n",
2503
+ " \"images_without_image_url_or_token\": (\n",
2504
+ " missing_image_url_count\n",
2505
+ " ),\n",
2506
+ "\n",
2507
+ " \"unique_civitai_origins\": sorted(\n",
2508
+ " all_civitai_origins\n",
2509
+ " )\n",
2510
+ " },\n",
2511
+ "\n",
2512
+ " \"har_statistics\": har_statistics,\n",
2513
+ "\n",
2514
+ " \"records\": final_records\n",
2515
+ "}\n",
2516
+ "\n",
2517
+ "\n",
2518
+ "# ================================================================\n",
2519
+ "# 16. SAVE JSON TO GOOGLE DRIVE\n",
2520
+ "# ================================================================\n",
2521
+ "\n",
2522
+ "TEMP_OUTPUT_FILE = OUTPUT_FILE.with_suffix(\n",
2523
+ " \".tmp\"\n",
2524
+ ")\n",
2525
+ "\n",
2526
+ "with open(\n",
2527
+ " TEMP_OUTPUT_FILE,\n",
2528
+ " \"w\",\n",
2529
+ " encoding=\"utf-8\"\n",
2530
+ ") as f:\n",
2531
+ "\n",
2532
+ " json.dump(\n",
2533
+ " output_data,\n",
2534
+ " f,\n",
2535
+ " ensure_ascii=False,\n",
2536
+ " indent=2\n",
2537
+ " )\n",
2538
+ "\n",
2539
+ "TEMP_OUTPUT_FILE.replace(\n",
2540
+ " OUTPUT_FILE\n",
2541
+ ")\n",
2542
+ "\n",
2543
+ "\n",
2544
+ "# ================================================================\n",
2545
+ "# 17. VERIFY JSON FILE\n",
2546
+ "# ================================================================\n",
2547
+ "\n",
2548
+ "with open(\n",
2549
+ " OUTPUT_FILE,\n",
2550
+ " \"r\",\n",
2551
+ " encoding=\"utf-8\"\n",
2552
+ ") as f:\n",
2553
+ "\n",
2554
+ " verification = json.load(f)\n",
2555
+ "\n",
2556
+ "\n",
2557
+ "verified_records = verification.get(\n",
2558
+ " \"records\",\n",
2559
+ " []\n",
2560
+ ")\n",
2561
+ "\n",
2562
+ "verified_ids = [\n",
2563
+ " record[\"value\"][\"image_id\"]\n",
2564
+ " for record in verified_records\n",
2565
+ "]\n",
2566
+ "\n",
2567
+ "assert len(verified_ids) == len(\n",
2568
+ " set(verified_ids)\n",
2569
+ "), \"Duplicate IDs found after saving.\"\n",
2570
+ "\n",
2571
+ "assert len(verified_records) == len(\n",
2572
+ " final_records\n",
2573
+ "), \"Record count changed after saving.\"\n",
2574
+ "\n",
2575
+ "# Every key must contain a request URL and parameters.\n",
2576
+ "for record in verified_records:\n",
2577
+ "\n",
2578
+ " key = record[\"key\"]\n",
2579
+ " value = record[\"value\"]\n",
2580
+ "\n",
2581
+ " assert key[\"method\"] == \"GET\"\n",
2582
+ " assert key[\"url\"].startswith(\"https://\")\n",
2583
+ " assert \"input\" in key[\"params\"]\n",
2584
+ "\n",
2585
+ " assert isinstance(\n",
2586
+ " value[\"image_id\"],\n",
2587
+ " int\n",
2588
+ " )\n",
2589
+ "\n",
2590
+ "\n",
2591
+ "# ================================================================\n",
2592
+ "# 18. FINAL REPORT\n",
2593
+ "# ================================================================\n",
2594
+ "\n",
2595
+ "print(\"\\n==============================================\")\n",
2596
+ "print(\"INDEX BUILD COMPLETE\")\n",
2597
+ "print(\"==============================================\")\n",
2598
+ "\n",
2599
+ "print(\"HAR files processed:\", len(HAR_FILES))\n",
2600
+ "\n",
2601
+ "print(\"Unique image records:\", len(final_records))\n",
2602
+ "\n",
2603
+ "print(\"Duplicate entries removed:\", duplicate_count)\n",
2604
+ "\n",
2605
+ "print(\n",
2606
+ " \"Images with observed image URLs:\",\n",
2607
+ " observed_image_url_count\n",
2608
+ ")\n",
2609
+ "\n",
2610
+ "print(\n",
2611
+ " \"Images with URL token only:\",\n",
2612
+ " token_only_count\n",
2613
+ ")\n",
2614
+ "\n",
2615
+ "print(\n",
2616
+ " \"Images without URL or token:\",\n",
2617
+ " missing_image_url_count\n",
2618
+ ")\n",
2619
+ "\n",
2620
+ "print(\"\\nHTTP requests made: 0\")\n",
2621
+ "print(\"Images downloaded: 0\")\n",
2622
+ "print(\"Prompts fetched: 0\")\n",
2623
+ "\n",
2624
+ "print(\"\\nOutput JSON:\")\n",
2625
+ "print(OUTPUT_FILE)\n",
2626
+ "\n",
2627
+ "print(\n",
2628
+ " \"File size:\",\n",
2629
+ " round(\n",
2630
+ " OUTPUT_FILE.stat().st_size / 1024**2,\n",
2631
+ " 3\n",
2632
+ " ),\n",
2633
+ " \"MB\"\n",
2634
+ ")\n",
2635
+ "\n",
2636
+ "print(\"\\n==============================================\")\n",
2637
+ "print(\"SAMPLE RECORD\")\n",
2638
+ "print(\"==============================================\")\n",
2639
+ "\n",
2640
+ "if verified_records:\n",
2641
+ "\n",
2642
+ " print(\n",
2643
+ " json.dumps(\n",
2644
+ " verified_records[0],\n",
2645
+ " ensure_ascii=False,\n",
2646
+ " indent=2\n",
2647
+ " )[:5000]\n",
2648
+ " )\n",
2649
+ "\n",
2650
+ "print(\"\\nAll checks passed.\")"
2651
+ ],
2652
+ "metadata": {
2653
+ "colab": {
2654
+ "base_uri": "https://localhost:8080/",
2655
+ "height": 1000,
2656
+ "referenced_widgets": [
2657
+ "1e5a7bbb49764dd1abef511964d23a70",
2658
+ "d8e77ce9f701423583e447ccb5daf0f2",
2659
+ "a96ca4a84fa941b08cff43fb6719072e",
2660
+ "f08ac9209f1a4c559d81db1c14c0b195",
2661
+ "90b359c740814c52b3f0c20447a0b578",
2662
+ "3c6fe424b4b6498fb8f3423f8e53c544",
2663
+ "af4e28043c8b4aeda06173bdcd7389a0",
2664
+ "f60bd97a472042e2b8bb3dac3db40278",
2665
+ "4896f44a58674d5b9414bcb695b4cc61",
2666
+ "40fe6d49d9ad4f8b8374a91b01610fa4",
2667
+ "7c691ef800d6445ebb896559dcae8931"
2668
+ ]
2669
+ },
2670
+ "id": "84oTgBTVR1bg",
2671
+ "outputId": "01f356e5-c85b-4f43-9a77-acee0d9cfa5e"
2672
+ },
2673
+ "execution_count": 8,
2674
+ "outputs": [
2675
+ {
2676
+ "output_type": "stream",
2677
+ "name": "stdout",
2678
+ "text": [
2679
+ "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n",
2680
+ "==============================================\n",
2681
+ "CIVITAI HAR INDEX BUILDER\n",
2682
+ "==============================================\n",
2683
+ "HAR directory: /content/drive/MyDrive\n",
2684
+ "HAR files found: 8\n",
2685
+ " - part1.har\n",
2686
+ " - part2.har\n",
2687
+ " - part3.har\n",
2688
+ " - part4.har\n",
2689
+ " - part5.har\n",
2690
+ " - part6.har\n",
2691
+ " - part7.har\n",
2692
+ " - part8.har\n",
2693
+ "\n",
2694
+ "==============================================\n",
2695
+ "PROCESSING HAR FILES\n",
2696
+ "==============================================\n"
2697
+ ]
2698
+ },
2699
+ {
2700
+ "output_type": "display_data",
2701
+ "data": {
2702
+ "text/plain": [
2703
+ "HAR files: 0%| | 0/8 [00:00<?, ?it/s]"
2704
+ ],
2705
+ "application/vnd.jupyter.widget-view+json": {
2706
+ "version_major": 2,
2707
+ "version_minor": 0,
2708
+ "model_id": "1e5a7bbb49764dd1abef511964d23a70"
2709
+ }
2710
+ },
2711
+ "metadata": {}
2712
+ },
2713
+ {
2714
+ "output_type": "stream",
2715
+ "name": "stdout",
2716
+ "text": [
2717
+ "\n",
2718
+ "Processing: part1.har\n",
2719
+ "\n",
2720
+ "Processing: part2.har\n",
2721
+ "\n",
2722
+ "Processing: part3.har\n",
2723
+ "\n",
2724
+ "Processing: part4.har\n",
2725
+ "\n",
2726
+ "Processing: part5.har\n",
2727
+ "\n",
2728
+ "Processing: part6.har\n",
2729
+ "\n",
2730
+ "Processing: part7.har\n",
2731
+ "\n",
2732
+ "Processing: part8.har\n",
2733
+ "\n",
2734
+ "Matching image requests to gallery records...\n",
2735
+ "\n",
2736
+ "==============================================\n",
2737
+ "INDEX BUILD COMPLETE\n",
2738
+ "==============================================\n",
2739
+ "HAR files processed: 8\n",
2740
+ "Unique image records: 3900\n",
2741
+ "Duplicate entries removed: 0\n",
2742
+ "Images with observed image URLs: 3089\n",
2743
+ "Images with URL token only: 811\n",
2744
+ "Images without URL or token: 0\n",
2745
+ "\n",
2746
+ "HTTP requests made: 0\n",
2747
+ "Images downloaded: 0\n",
2748
+ "Prompts fetched: 0\n",
2749
+ "\n",
2750
+ "Output JSON:\n",
2751
+ "/content/drive/MyDrive/civitai_request_image_index.json\n",
2752
+ "File size: 5.748 MB\n",
2753
+ "\n",
2754
+ "==============================================\n",
2755
+ "SAMPLE RECORD\n",
2756
+ "==============================================\n",
2757
+ "{\n",
2758
+ " \"key\": {\n",
2759
+ " \"method\": \"GET\",\n",
2760
+ " \"url\": \"https://civitai.red/api/trpc/image.getGenerationData\",\n",
2761
+ " \"params\": {\n",
2762
+ " \"input\": \"{\\\"json\\\":{\\\"id\\\":134635479,\\\"authed\\\":true}}\"\n",
2763
+ " },\n",
2764
+ " \"headers\": {\n",
2765
+ " \"Referer\": \"https://civitai.red/images/134635479\",\n",
2766
+ " \"Accept\": \"application/json, text/plain, */*\"\n",
2767
+ " }\n",
2768
+ " },\n",
2769
+ " \"value\": {\n",
2770
+ " \"image_id\": 134635479,\n",
2771
+ " \"image_url\": \"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/67f1bac1-e23e-4450-a31a-df6f6034704b/anim=false,width=800,optimized=true/RLY-KREA2-Basic_Workflow-v001-Upscaled_00002_.jpeg\",\n",
2772
+ " \"image_url_candidates\": [\n",
2773
+ " \"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/67f1bac1-e23e-4450-a31a-df6f6034704b/anim=false,width=800,optimized=true/RLY-KREA2-Basic_Workflow-v001-Upscaled_00002_.jpeg\"\n",
2774
+ " ],\n",
2775
+ " \"image_url_status\": \"observed_in_har\",\n",
2776
+ " \"url_token\": \"67f1bac1-e23e-4450-a31a-df6f6034704b\",\n",
2777
+ " \"width\": 1288,\n",
2778
+ " \"height\": 1888,\n",
2779
+ " \"hash\": \"UMI;n?%gt-M|_Nbv%MxGIpIVobn$M{RjM{RP\",\n",
2780
+ " \"name\": \"RLY-KREA2-Basic_Workflow-v001-Upscaled_00002_.png\",\n",
2781
+ " \"username\": \"RaymondLuxuryYacht\",\n",
2782
+ " \"nsfw_level\": 8,\n",
2783
+ " \"created_at\": [\n",
2784
+ " \"Date\",\n",
2785
+ " \"2026-06-23T15:25:11.556Z\"\n",
2786
+ " ],\n",
2787
+ " \"published_at\": [\n",
2788
+ " \"Date\",\n",
2789
+ " \"2026-06-23T15:25:59.647Z\"\n",
2790
+ " ],\n",
2791
+ " \"has_meta\": true,\n",
2792
+ " \"has_positive_prompt\": true,\n",
2793
+ " \"model_version_ids\": [],\n",
2794
+ " \"civitai_page_url\": \"https://civitai.red/images/134635479\",\n",
2795
+ " \"source_har_files\": [\n",
2796
+ " \"part5.har\"\n",
2797
+ " ],\n",
2798
+ " \"gallery_origin\": \"https://civitai.red\"\n",
2799
+ " }\n",
2800
+ "}\n",
2801
+ "\n",
2802
+ "All checks passed.\n"
2803
+ ]
2804
+ }
2805
+ ]
2806
  }
2807
  ]
2808
  }