Spaces:
Running
Running
mike boone commited on
Commit ·
3d06bfd
1
Parent(s): cdded3f
chore: log table tml import calls
Browse files- thoughtspot_deployer.py +84 -27
thoughtspot_deployer.py
CHANGED
|
@@ -2235,21 +2235,40 @@ class ThoughtSpotDeployer:
|
|
| 2235 |
body_bytes = len(json.dumps(payload, default=str))
|
| 2236 |
start = time.time()
|
| 2237 |
log_progress(f" {phase_label}: importing {len(tmls)} table(s): {', '.join(names)}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2238 |
if _slog:
|
| 2239 |
-
_slog.
|
| 2240 |
"thoughtspot",
|
| 2241 |
-
"tml import started",
|
| 2242 |
-
|
| 2243 |
-
table_names=names,
|
| 2244 |
-
table_count=len(tmls),
|
| 2245 |
-
payload_bytes=body_bytes,
|
| 2246 |
)
|
| 2247 |
|
| 2248 |
-
|
| 2249 |
-
|
| 2250 |
-
|
| 2251 |
-
|
| 2252 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2253 |
elapsed = time.time() - start
|
| 2254 |
|
| 2255 |
if response.status_code == 401:
|
|
@@ -2264,17 +2283,54 @@ class ThoughtSpotDeployer:
|
|
| 2264 |
)
|
| 2265 |
if self.authenticate():
|
| 2266 |
retry_start = time.time()
|
| 2267 |
-
|
| 2268 |
-
|
| 2269 |
-
|
| 2270 |
-
|
| 2271 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2272 |
elapsed += time.time() - retry_start
|
| 2273 |
|
| 2274 |
if response.status_code == 200:
|
| 2275 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2276 |
if objects is None:
|
| 2277 |
-
error = f"{phase_label} failed: Unexpected response format: {type(
|
| 2278 |
log_progress(f" ❌ {error}")
|
| 2279 |
results['errors'].append(error)
|
| 2280 |
if _slog:
|
|
@@ -2287,15 +2343,6 @@ class ThoughtSpotDeployer:
|
|
| 2287 |
elapsed_s=round(elapsed, 1),
|
| 2288 |
)
|
| 2289 |
return None
|
| 2290 |
-
|
| 2291 |
-
if _slog:
|
| 2292 |
-
_slog.log_verbose(
|
| 2293 |
-
"thoughtspot",
|
| 2294 |
-
"tml import completed",
|
| 2295 |
-
phase=phase_label,
|
| 2296 |
-
table_names=names,
|
| 2297 |
-
elapsed_s=round(elapsed, 1),
|
| 2298 |
-
)
|
| 2299 |
return objects
|
| 2300 |
|
| 2301 |
error = f"{phase_label} HTTP error: {response.status_code} - {response.text}"
|
|
@@ -2311,6 +2358,16 @@ class ThoughtSpotDeployer:
|
|
| 2311 |
payload_bytes=body_bytes,
|
| 2312 |
elapsed_s=round(elapsed, 1),
|
| 2313 |
status_code=response.status_code,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2314 |
)
|
| 2315 |
return response
|
| 2316 |
|
|
|
|
| 2235 |
body_bytes = len(json.dumps(payload, default=str))
|
| 2236 |
start = time.time()
|
| 2237 |
log_progress(f" {phase_label}: importing {len(tmls)} table(s): {', '.join(names)}")
|
| 2238 |
+
import_meta = {
|
| 2239 |
+
"phase": phase_label,
|
| 2240 |
+
"table_names": names,
|
| 2241 |
+
"table_count": len(tmls),
|
| 2242 |
+
"payload_bytes": body_bytes,
|
| 2243 |
+
"create_new": create_new,
|
| 2244 |
+
}
|
| 2245 |
if _slog:
|
| 2246 |
+
_slog.log(
|
| 2247 |
"thoughtspot",
|
| 2248 |
+
"table tml import request started",
|
| 2249 |
+
**import_meta,
|
|
|
|
|
|
|
|
|
|
| 2250 |
)
|
| 2251 |
|
| 2252 |
+
try:
|
| 2253 |
+
response = self.session.post(
|
| 2254 |
+
f"{self.base_url}/api/rest/2.0/metadata/tml/import",
|
| 2255 |
+
json=payload,
|
| 2256 |
+
timeout=360,
|
| 2257 |
+
)
|
| 2258 |
+
except Exception as exc:
|
| 2259 |
+
elapsed = time.time() - start
|
| 2260 |
+
error = f"{phase_label} request exception: {exc}"
|
| 2261 |
+
log_progress(f" ❌ {error}")
|
| 2262 |
+
if _slog:
|
| 2263 |
+
_slog.log(
|
| 2264 |
+
"thoughtspot",
|
| 2265 |
+
"table tml import exception",
|
| 2266 |
+
error=error[:1000],
|
| 2267 |
+
**import_meta,
|
| 2268 |
+
elapsed_s=round(elapsed, 1),
|
| 2269 |
+
exception_type=type(exc).__name__,
|
| 2270 |
+
)
|
| 2271 |
+
return None
|
| 2272 |
elapsed = time.time() - start
|
| 2273 |
|
| 2274 |
if response.status_code == 401:
|
|
|
|
| 2283 |
)
|
| 2284 |
if self.authenticate():
|
| 2285 |
retry_start = time.time()
|
| 2286 |
+
try:
|
| 2287 |
+
response = self.session.post(
|
| 2288 |
+
f"{self.base_url}/api/rest/2.0/metadata/tml/import",
|
| 2289 |
+
json=payload,
|
| 2290 |
+
timeout=360,
|
| 2291 |
+
)
|
| 2292 |
+
except Exception as exc:
|
| 2293 |
+
elapsed += time.time() - retry_start
|
| 2294 |
+
error = f"{phase_label} request exception after re-auth: {exc}"
|
| 2295 |
+
log_progress(f" ❌ {error}")
|
| 2296 |
+
if _slog:
|
| 2297 |
+
_slog.log(
|
| 2298 |
+
"thoughtspot",
|
| 2299 |
+
"table tml import exception",
|
| 2300 |
+
error=error[:1000],
|
| 2301 |
+
**import_meta,
|
| 2302 |
+
elapsed_s=round(elapsed, 1),
|
| 2303 |
+
exception_type=type(exc).__name__,
|
| 2304 |
+
after_reauth=True,
|
| 2305 |
+
)
|
| 2306 |
+
return None
|
| 2307 |
elapsed += time.time() - retry_start
|
| 2308 |
|
| 2309 |
if response.status_code == 200:
|
| 2310 |
+
response_json = response.json()
|
| 2311 |
+
objects = _normalize_tml_import_objects(response_json)
|
| 2312 |
+
object_count = len(objects) if isinstance(objects, list) else 0
|
| 2313 |
+
object_statuses = []
|
| 2314 |
+
object_names = []
|
| 2315 |
+
for obj in objects or []:
|
| 2316 |
+
obj_response = obj.get('response', {}) if isinstance(obj, dict) else {}
|
| 2317 |
+
status = obj_response.get('status', {})
|
| 2318 |
+
header = obj_response.get('header', {})
|
| 2319 |
+
object_statuses.append(status.get('status_code'))
|
| 2320 |
+
object_names.append(header.get('name') or header.get('display_name'))
|
| 2321 |
+
if _slog:
|
| 2322 |
+
_slog.log(
|
| 2323 |
+
"thoughtspot",
|
| 2324 |
+
"table tml import response received",
|
| 2325 |
+
**import_meta,
|
| 2326 |
+
elapsed_s=round(elapsed, 1),
|
| 2327 |
+
status_code=response.status_code,
|
| 2328 |
+
object_count=object_count,
|
| 2329 |
+
object_statuses=object_statuses,
|
| 2330 |
+
object_names=[name for name in object_names if name],
|
| 2331 |
+
)
|
| 2332 |
if objects is None:
|
| 2333 |
+
error = f"{phase_label} failed: Unexpected response format: {type(response_json)}"
|
| 2334 |
log_progress(f" ❌ {error}")
|
| 2335 |
results['errors'].append(error)
|
| 2336 |
if _slog:
|
|
|
|
| 2343 |
elapsed_s=round(elapsed, 1),
|
| 2344 |
)
|
| 2345 |
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2346 |
return objects
|
| 2347 |
|
| 2348 |
error = f"{phase_label} HTTP error: {response.status_code} - {response.text}"
|
|
|
|
| 2358 |
payload_bytes=body_bytes,
|
| 2359 |
elapsed_s=round(elapsed, 1),
|
| 2360 |
status_code=response.status_code,
|
| 2361 |
+
response_text=response.text[:1000],
|
| 2362 |
+
)
|
| 2363 |
+
_slog.log(
|
| 2364 |
+
"thoughtspot",
|
| 2365 |
+
"table tml import response received",
|
| 2366 |
+
error=error[:1000],
|
| 2367 |
+
**import_meta,
|
| 2368 |
+
elapsed_s=round(elapsed, 1),
|
| 2369 |
+
status_code=response.status_code,
|
| 2370 |
+
response_text=response.text[:1000],
|
| 2371 |
)
|
| 2372 |
return response
|
| 2373 |
|