Auto upload zain 2026-08-14T21:30:04.912464 (part 4)

.gitattributes

@@ -61,3 +61,4 @@ zain/Activation/wandb/run-20260814_211407-i85cngas/run-i85cngas.wandb filter=lfs
 zain/Activation/wandb/run-20260814_211507-54wnyexz/run-54wnyexz.wandb filter=lfs diff=lfs merge=lfs -text
 zain/Activation/wandb/run-20260814_212208-n6fjpg9o/run-n6fjpg9o.wandb filter=lfs diff=lfs merge=lfs -text
 zain/Activation/out/glu-waleed10-100L_trash_run/training_log.jsonl filter=lfs diff=lfs merge=lfs -text
+zain/Activation/wandb/run-20260814_212835-gqskne0x/run-gqskne0x.wandb filter=lfs diff=lfs merge=lfs -text

zain/Activation/wandb/run-20260814_195151-ix6wcbwv/logs/debug-core.log

@@ -68,3 +68,10 @@
 {"time":"2026-08-14T21:22:08.718181031Z","level":"INFO","msg":"handleInformInit: received","streamId":"n6fjpg9o","id":"3(@)"}
 {"time":"2026-08-14T21:22:08.976795469Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"n6fjpg9o","id":"3(@)"}
 {"time":"2026-08-14T21:22:14.472169125Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"qlc6t75fdq0u"}
+{"time":"2026-08-14T21:28:18.727248774Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"qlc6t75fdq0u"}
+{"time":"2026-08-14T21:28:20.378913364Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"qlc6t75fdq0u"}
+{"time":"2026-08-14T21:28:20.400345521Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"n6fjpg9o","id":"3(@)"}
+{"time":"2026-08-14T21:28:20.400969576Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"n6fjpg9o","id":"3(@)"}
+{"time":"2026-08-14T21:28:35.344777056Z","level":"INFO","msg":"handleInformInit: received","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:28:35.603024157Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:28:41.078893683Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}

zain/Activation/wandb/run-20260814_200425-npumf6lz/logs/debug-core.log

@@ -47,3 +47,16 @@
 {"time":"2026-08-14T21:14:07.911109006Z","level":"INFO","msg":"handleInformInit: received","streamId":"i85cngas","id":"1(@)"}
 {"time":"2026-08-14T21:14:08.168219613Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"i85cngas","id":"1(@)"}
 {"time":"2026-08-14T21:14:13.672910923Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"d8p4y9v5n05m"}
+{"time":"2026-08-14T21:25:58.658206835Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"d8p4y9v5n05m"}
+{"time":"2026-08-14T21:25:59.189862536Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"d8p4y9v5n05m"}
+{"time":"2026-08-14T21:25:59.191285843Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"i85cngas","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.192218621Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"i85cngas","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492428474Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492473057Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492482505Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492489625Z","level":"INFO","msg":"server: is shutting down"}
+{"time":"2026-08-14T21:25:59.492505848Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492527496Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492557564Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492679293Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-409188-410559-1836801570/socket","Net":"unix"}}
+{"time":"2026-08-14T21:25:59.492751296Z","level":"INFO","msg":"server: all connections closed"}

zain/Activation/wandb/run-20260814_200503-bdhnr27j/logs/debug-core.log

@@ -68,3 +68,10 @@
 {"time":"2026-08-14T21:22:08.718181031Z","level":"INFO","msg":"handleInformInit: received","streamId":"n6fjpg9o","id":"3(@)"}
 {"time":"2026-08-14T21:22:08.976795469Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"n6fjpg9o","id":"3(@)"}
 {"time":"2026-08-14T21:22:14.472169125Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"qlc6t75fdq0u"}
+{"time":"2026-08-14T21:28:18.727248774Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"qlc6t75fdq0u"}
+{"time":"2026-08-14T21:28:20.378913364Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"qlc6t75fdq0u"}
+{"time":"2026-08-14T21:28:20.400345521Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"n6fjpg9o","id":"3(@)"}
+{"time":"2026-08-14T21:28:20.400969576Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"n6fjpg9o","id":"3(@)"}
+{"time":"2026-08-14T21:28:35.344777056Z","level":"INFO","msg":"handleInformInit: received","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:28:35.603024157Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:28:41.078893683Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}

zain/Activation/wandb/run-20260814_201615-w8mr4w6k/logs/debug-core.log

@@ -47,3 +47,16 @@
 {"time":"2026-08-14T21:14:07.911109006Z","level":"INFO","msg":"handleInformInit: received","streamId":"i85cngas","id":"1(@)"}
 {"time":"2026-08-14T21:14:08.168219613Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"i85cngas","id":"1(@)"}
 {"time":"2026-08-14T21:14:13.672910923Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"d8p4y9v5n05m"}
+{"time":"2026-08-14T21:25:58.658206835Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"d8p4y9v5n05m"}
+{"time":"2026-08-14T21:25:59.189862536Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"d8p4y9v5n05m"}
+{"time":"2026-08-14T21:25:59.191285843Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"i85cngas","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.192218621Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"i85cngas","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492428474Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492473057Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492482505Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492489625Z","level":"INFO","msg":"server: is shutting down"}
+{"time":"2026-08-14T21:25:59.492505848Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492527496Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492557564Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492679293Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-409188-410559-1836801570/socket","Net":"unix"}}
+{"time":"2026-08-14T21:25:59.492751296Z","level":"INFO","msg":"server: all connections closed"}

zain/Activation/wandb/run-20260814_202128-twgo1e89/logs/debug-core.log

@@ -68,3 +68,10 @@
 {"time":"2026-08-14T21:22:08.718181031Z","level":"INFO","msg":"handleInformInit: received","streamId":"n6fjpg9o","id":"3(@)"}
 {"time":"2026-08-14T21:22:08.976795469Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"n6fjpg9o","id":"3(@)"}
 {"time":"2026-08-14T21:22:14.472169125Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"qlc6t75fdq0u"}
+{"time":"2026-08-14T21:28:18.727248774Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"qlc6t75fdq0u"}
+{"time":"2026-08-14T21:28:20.378913364Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"qlc6t75fdq0u"}
+{"time":"2026-08-14T21:28:20.400345521Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"n6fjpg9o","id":"3(@)"}
+{"time":"2026-08-14T21:28:20.400969576Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"n6fjpg9o","id":"3(@)"}
+{"time":"2026-08-14T21:28:35.344777056Z","level":"INFO","msg":"handleInformInit: received","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:28:35.603024157Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:28:41.078893683Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}

zain/Activation/wandb/run-20260814_202806-g2ch84q3/logs/debug-core.log

@@ -47,3 +47,16 @@
 {"time":"2026-08-14T21:14:07.911109006Z","level":"INFO","msg":"handleInformInit: received","streamId":"i85cngas","id":"1(@)"}
 {"time":"2026-08-14T21:14:08.168219613Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"i85cngas","id":"1(@)"}
 {"time":"2026-08-14T21:14:13.672910923Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"d8p4y9v5n05m"}
+{"time":"2026-08-14T21:25:58.658206835Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"d8p4y9v5n05m"}
+{"time":"2026-08-14T21:25:59.189862536Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"d8p4y9v5n05m"}
+{"time":"2026-08-14T21:25:59.191285843Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"i85cngas","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.192218621Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"i85cngas","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492428474Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492473057Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492482505Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492489625Z","level":"INFO","msg":"server: is shutting down"}
+{"time":"2026-08-14T21:25:59.492505848Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492527496Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492557564Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492679293Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-409188-410559-1836801570/socket","Net":"unix"}}
+{"time":"2026-08-14T21:25:59.492751296Z","level":"INFO","msg":"server: all connections closed"}

zain/Activation/wandb/run-20260814_203803-7p5140bq/logs/debug-core.log

@@ -68,3 +68,10 @@
 {"time":"2026-08-14T21:22:08.718181031Z","level":"INFO","msg":"handleInformInit: received","streamId":"n6fjpg9o","id":"3(@)"}
 {"time":"2026-08-14T21:22:08.976795469Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"n6fjpg9o","id":"3(@)"}
 {"time":"2026-08-14T21:22:14.472169125Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"qlc6t75fdq0u"}
+{"time":"2026-08-14T21:28:18.727248774Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"qlc6t75fdq0u"}
+{"time":"2026-08-14T21:28:20.378913364Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"qlc6t75fdq0u"}
+{"time":"2026-08-14T21:28:20.400345521Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"n6fjpg9o","id":"3(@)"}
+{"time":"2026-08-14T21:28:20.400969576Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"n6fjpg9o","id":"3(@)"}
+{"time":"2026-08-14T21:28:35.344777056Z","level":"INFO","msg":"handleInformInit: received","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:28:35.603024157Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:28:41.078893683Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}

zain/Activation/wandb/run-20260814_203945-j86czemo/logs/debug-core.log

@@ -47,3 +47,16 @@
 {"time":"2026-08-14T21:14:07.911109006Z","level":"INFO","msg":"handleInformInit: received","streamId":"i85cngas","id":"1(@)"}
 {"time":"2026-08-14T21:14:08.168219613Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"i85cngas","id":"1(@)"}
 {"time":"2026-08-14T21:14:13.672910923Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"d8p4y9v5n05m"}
+{"time":"2026-08-14T21:25:58.658206835Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"d8p4y9v5n05m"}
+{"time":"2026-08-14T21:25:59.189862536Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"d8p4y9v5n05m"}
+{"time":"2026-08-14T21:25:59.191285843Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"i85cngas","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.192218621Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"i85cngas","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492428474Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492473057Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492482505Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492489625Z","level":"INFO","msg":"server: is shutting down"}
+{"time":"2026-08-14T21:25:59.492505848Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492527496Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492557564Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492679293Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-409188-410559-1836801570/socket","Net":"unix"}}
+{"time":"2026-08-14T21:25:59.492751296Z","level":"INFO","msg":"server: all connections closed"}

zain/Activation/wandb/run-20260814_205135-0vnbmyac/logs/debug-core.log

@@ -47,3 +47,16 @@
 {"time":"2026-08-14T21:14:07.911109006Z","level":"INFO","msg":"handleInformInit: received","streamId":"i85cngas","id":"1(@)"}
 {"time":"2026-08-14T21:14:08.168219613Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"i85cngas","id":"1(@)"}
 {"time":"2026-08-14T21:14:13.672910923Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"d8p4y9v5n05m"}
+{"time":"2026-08-14T21:25:58.658206835Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"d8p4y9v5n05m"}
+{"time":"2026-08-14T21:25:59.189862536Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"d8p4y9v5n05m"}
+{"time":"2026-08-14T21:25:59.191285843Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"i85cngas","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.192218621Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"i85cngas","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492428474Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492473057Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492482505Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492489625Z","level":"INFO","msg":"server: is shutting down"}
+{"time":"2026-08-14T21:25:59.492505848Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492527496Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492557564Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492679293Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-409188-410559-1836801570/socket","Net":"unix"}}
+{"time":"2026-08-14T21:25:59.492751296Z","level":"INFO","msg":"server: all connections closed"}

zain/Activation/wandb/run-20260814_210109-qauftiuh/logs/debug-core.log

@@ -68,3 +68,10 @@
 {"time":"2026-08-14T21:22:08.718181031Z","level":"INFO","msg":"handleInformInit: received","streamId":"n6fjpg9o","id":"3(@)"}
 {"time":"2026-08-14T21:22:08.976795469Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"n6fjpg9o","id":"3(@)"}
 {"time":"2026-08-14T21:22:14.472169125Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"qlc6t75fdq0u"}
+{"time":"2026-08-14T21:28:18.727248774Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"qlc6t75fdq0u"}
+{"time":"2026-08-14T21:28:20.378913364Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"qlc6t75fdq0u"}
+{"time":"2026-08-14T21:28:20.400345521Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"n6fjpg9o","id":"3(@)"}
+{"time":"2026-08-14T21:28:20.400969576Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"n6fjpg9o","id":"3(@)"}
+{"time":"2026-08-14T21:28:35.344777056Z","level":"INFO","msg":"handleInformInit: received","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:28:35.603024157Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:28:41.078893683Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}

zain/Activation/wandb/run-20260814_210155-rxpj34pj/logs/debug-core.log

@@ -47,3 +47,16 @@
 {"time":"2026-08-14T21:14:07.911109006Z","level":"INFO","msg":"handleInformInit: received","streamId":"i85cngas","id":"1(@)"}
 {"time":"2026-08-14T21:14:08.168219613Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"i85cngas","id":"1(@)"}
 {"time":"2026-08-14T21:14:13.672910923Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"d8p4y9v5n05m"}
+{"time":"2026-08-14T21:25:58.658206835Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"d8p4y9v5n05m"}
+{"time":"2026-08-14T21:25:59.189862536Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"d8p4y9v5n05m"}
+{"time":"2026-08-14T21:25:59.191285843Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"i85cngas","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.192218621Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"i85cngas","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492428474Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492473057Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492482505Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492489625Z","level":"INFO","msg":"server: is shutting down"}
+{"time":"2026-08-14T21:25:59.492505848Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492527496Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492557564Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492679293Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-409188-410559-1836801570/socket","Net":"unix"}}
+{"time":"2026-08-14T21:25:59.492751296Z","level":"INFO","msg":"server: all connections closed"}

zain/Activation/wandb/run-20260814_210809-a6574zx8/logs/debug-core.log

@@ -68,3 +68,10 @@
 {"time":"2026-08-14T21:22:08.718181031Z","level":"INFO","msg":"handleInformInit: received","streamId":"n6fjpg9o","id":"3(@)"}
 {"time":"2026-08-14T21:22:08.976795469Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"n6fjpg9o","id":"3(@)"}
 {"time":"2026-08-14T21:22:14.472169125Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"qlc6t75fdq0u"}
+{"time":"2026-08-14T21:28:18.727248774Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"qlc6t75fdq0u"}
+{"time":"2026-08-14T21:28:20.378913364Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"qlc6t75fdq0u"}
+{"time":"2026-08-14T21:28:20.400345521Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"n6fjpg9o","id":"3(@)"}
+{"time":"2026-08-14T21:28:20.400969576Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"n6fjpg9o","id":"3(@)"}
+{"time":"2026-08-14T21:28:35.344777056Z","level":"INFO","msg":"handleInformInit: received","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:28:35.603024157Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:28:41.078893683Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}

zain/Activation/wandb/run-20260814_211407-i85cngas/files/config.yaml

@@ -0,0 +1,439 @@
+_name_or_path:
+    value: ""
+_wandb:
+    value:
+        cli_version: 0.28.1
+        e:
+            unk3a3lapicd752rmq1uqlvx1kx26i7e:
+                args:
+                    - --config
+                    - /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/configs/baseline.yaml
+                    - --variants
+                    - glu-waleed10
+                    - glu-silu-waleed10
+                    - mlp-waleed10
+                    - mlp-silu-waleed10
+                    - glu-waleed
+                    - glu-situglu_low
+                    - glu-waleedglu_low
+                codePath: sweep.py
+                codePathLocal: sweep.py
+                cpu_count: 112
+                cpu_count_logical: 224
+                cudaVersion: "12.4"
+                disk:
+                    /:
+                        total: "1560765693952"
+                        used: "701346242560"
+                email: deepnevro@gmail.com
+                executable: /mnt/data/zainulabideen/zain-exp/notebooks/my_env/bin/python
+                git:
+                    commit: 3fb47c6943d6927eafc39bd399a3d3a520bae74a
+                    remote: https://github.com/w-ahmad1a10/Activation.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-39c684a5-fde6-83d7-1663-0859795881ae
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-68012e5a-38b6-b643-0ca6-62fb66720bf3
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-132944c4-b689-2b5f-89a4-d730401677ab
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-2df386cc-6d26-d0e2-7a2d-a057b0d95864
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bfa16575-1d94-1aa2-4537-2c93433f42ef
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bc6c3e3c-9b90-09ca-c034-774961847c54
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-00a441e1-7c95-e7d6-4c35-43d6b291aea9
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-1c4d29a2-4647-6fce-d8fc-0c5ecfbbd6ea
+                host: deeplens-k3s-node1
+                memory:
+                    total: "2164089937920"
+                os: Linux-5.15.0-126-generic-x86_64-with-glibc2.35
+                program: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py
+                python: CPython 3.11.15
+                root: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation
+                startedAt: "2026-08-14T21:14:07.908466Z"
+                writerId: unk3a3lapicd752rmq1uqlvx1kx26i7e
+        m:
+            - "1": train/global_step
+              "6":
+                - 3
+              "7": []
+            - "2": '*'
+              "5": 1
+              "6":
+                - 1
+              "7": []
+        python_version: 3.11.15
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 7
+                - 13
+                - 19
+                - 62
+                - 66
+            "4": 3.11.15
+            "5": 0.28.1
+            "6": 5.16.0.dev0
+            "9":
+                "1": transformers_trainer
+            "12": 0.28.1
+            "13": linux-x86_64
+accelerator_config:
+    value:
+        dispatch_batches: null
+        even_batches: true
+        gradient_accumulation_kwargs: null
+        non_blocking: false
+        split_batches: false
+        use_seedable_sampler: true
+activation:
+    value: waleedglu_low
+adam_beta1:
+    value: 0.9
+adam_beta2:
+    value: 0.999
+adam_epsilon:
+    value: 1e-08
+architectures:
+    value: null
+attention_bias:
+    value: false
+attention_dropout:
+    value: 0
+auto_find_batch_size:
+    value: false
+average_tokens_across_devices:
+    value: true
+batch_eval_metrics:
+    value: false
+bf16:
+    value: true
+bf16_full_eval:
+    value: false
+bos_token_id:
+    value: 1
+chunk_size_feed_forward:
+    value: 0
+data_seed:
+    value: 42
+dataloader_drop_last:
+    value: false
+dataloader_in_order:
+    value: true
+dataloader_multiprocessing_context:
+    value: null
+dataloader_num_workers:
+    value: 0
+dataloader_persistent_workers:
+    value: false
+dataloader_pin_memory:
+    value: true
+dataloader_prefetch_factor:
+    value: null
+ddp_backend:
+    value: null
+ddp_broadcast_buffers:
+    value: null
+ddp_bucket_cap_mb:
+    value: null
+ddp_find_unused_parameters:
+    value: null
+ddp_static_graph:
+    value: null
+ddp_timeout:
+    value: 1800
+debug:
+    value: []
+deepspeed:
+    value: null
+disable_tqdm:
+    value: false
+do_eval:
+    value: true
+do_predict:
+    value: false
+do_train:
+    value: false
+dtype:
+    value: null
+enable_jit_checkpoint:
+    value: false
+eos_token_id:
+    value: 2
+eval_accumulation_steps:
+    value: null
+eval_delay:
+    value: 0
+eval_do_concat_batches:
+    value: true
+eval_on_start:
+    value: false
+eval_steps:
+    value: 60000
+eval_strategy:
+    value: steps
+eval_use_gather_object:
+    value: false
+fp16:
+    value: false
+fp16_full_eval:
+    value: false
+fsdp:
+    value: null
+fsdp_config:
+    value: null
+full_determinism:
+    value: false
+gradient_accumulation_steps:
+    value: 1
+gradient_checkpointing:
+    value: false
+gradient_checkpointing_kwargs:
+    value: null
+greater_is_better:
+    value: null
+head_dim:
+    value: 32
+hidden_act:
+    value: silu
+hidden_size:
+    value: 128
+hub_always_push:
+    value: false
+hub_model_id:
+    value: w-ahmad/A-glu-waleedglu_low-9L
+hub_private_repo:
+    value: null
+hub_revision:
+    value: null
+hub_strategy:
+    value: every_save
+hub_token:
+    value: <HUB_TOKEN>
+id2label:
+    value:
+        "0": LABEL_0
+        "1": LABEL_1
+ignore_data_skip:
+    value: false
+include_for_metrics:
+    value: []
+include_num_input_tokens_seen:
+    value: "no"
+initializer_range:
+    value: 0.02
+intermediate_size:
+    value: 256
+is_encoder_decoder:
+    value: false
+label_names:
+    value: null
+label_smoothing_factor:
+    value: 0
+label2id:
+    value:
+        LABEL_0: 0
+        LABEL_1: 1
+learning_rate:
+    value: 0.001
+length_column_name:
+    value: length
+liger_kernel_config:
+    value: null
+load_best_model_at_end:
+    value: false
+local_rank:
+    value: -1
+log_level:
+    value: passive
+log_level_replica:
+    value: warning
+log_on_each_node:
+    value: true
+logging_first_step:
+    value: false
+logging_nan_inf_filter:
+    value: true
+logging_steps:
+    value: 20
+logging_strategy:
+    value: steps
+lr_scheduler_kwargs:
+    value: null
+lr_scheduler_type:
+    value: constant
+max_grad_norm:
+    value: 1
+max_position_embeddings:
+    value: 512
+max_steps:
+    value: 1000
+metric_for_best_model:
+    value: null
+mlp_bias:
+    value: false
+mlp_type:
+    value: glu
+model/num_parameters:
+    value: 2001280
+model_type:
+    value: tiny_llama
+neftune_noise_alpha:
+    value: null
+num_attention_heads:
+    value: 4
+num_hidden_layers:
+    value: 9
+num_key_value_heads:
+    value: 4
+num_train_epochs:
+    value: 1
+optim:
+    value: adamw_torch
+optim_args:
+    value: null
+optim_target_modules:
+    value: null
+output_attentions:
+    value: false
+output_dir:
+    value: out/glu-waleedglu_low-9L_run
+output_hidden_states:
+    value: false
+pad_token_id:
+    value: 0
+parallelism_config:
+    value: null
+per_device_eval_batch_size:
+    value: 512
+per_device_train_batch_size:
+    value: 512
+prediction_loss_only:
+    value: false
+pretraining_tp:
+    value: 1
+problem_type:
+    value: null
+project:
+    value: huggingface
+push_to_hub:
+    value: false
+remove_unused_columns:
+    value: false
+report_to:
+    value:
+        - wandb
+restore_callback_states_from_checkpoint:
+    value: false
+resume_from_checkpoint:
+    value: null
+return_dict:
+    value: true
+rms_norm_eps:
+    value: 1e-06
+rope_parameters:
+    value:
+        rope_theta: 10000
+        rope_type: default
+run_name:
+    value: LM-glu-waleedglu_low-9L-2.0M-20260814-211407
+save_on_each_node:
+    value: false
+save_only_model:
+    value: false
+save_steps:
+    value: 100
+save_strategy:
+    value: steps
+save_total_limit:
+    value: null
+seed:
+    value: 42
+skip_memory_metrics:
+    value: true
+tf32:
+    value: null
+tie_word_embeddings:
+    value: true
+tokenizer_name:
+    value: w-ahmad/tiny-stories-tokenizer
+torch_compile:
+    value: false
+torch_compile_backend:
+    value: null
+torch_compile_mode:
+    value: null
+torch_empty_cache_steps:
+    value: null
+trackio_bucket_id:
+    value: null
+trackio_space_id:
+    value: null
+trackio_static_space_id:
+    value: null
+train_sampling_strategy:
+    value: random
+transformers_version:
+    value: 5.16.0.dev0
+use_cache:
+    value: false
+use_cpu:
+    value: false
+use_liger_kernel:
+    value: false
+vocab_size:
+    value: 4096
+waleed_beta:
+    value: 10
+warmup_steps:
+    value: 0
+weight_decay:
+    value: 0.01

zain/Activation/wandb/run-20260814_211407-i85cngas/files/output.log

@@ -88,6 +88,23 @@ Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 155
   - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
   - If you are not the owner of the model architecture class, please contact the model code owner to update it.
 Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 156.87it/s]
- 95%|█████████▍| 949/1000 [10:54<00:34,  1.47it/s]?, ?it/s]
+100%|██████████| 1000/1000 [11:40<00:00,  1.33i[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
 {'loss': '2.268', 'grad_norm': '0.9023', 'learning_rate': '0.001', 'epoch': '0.496', 'train/total_time_seconds': '284.7', 'train/time_per_step_avg': '0.3259', 'train/epoch_time_elapsed': '634.1', 'train/estimated_remaining_minutes': '0.4126'}
 {'loss': '2.264', 'grad_norm': '0.6562', 'learning_rate': '0.001', 'epoch': '0.5067', 'train/total_time_seconds': '291.5', 'train/time_per_step_avg': '0.3278', 'train/epoch_time_elapsed': '648.6', 'train/estimated_remaining_minutes': '0.3102'}
+{'loss': '2.258', 'grad_norm': '0.6406', 'learning_rate': '0.001', 'epoch': '0.5175', 'train/total_time_seconds': '298', 'train/time_per_step_avg': '0.3264', 'train/epoch_time_elapsed': '662.8', 'train/estimated_remaining_minutes': '0.207'}
+{'loss': '2.251', 'grad_norm': '0.6484', 'learning_rate': '0.001', 'epoch': '0.5283', 'train/total_time_seconds': '304.5', 'train/time_per_step_avg': '0.3226', 'train/epoch_time_elapsed': '676.9', 'train/estimated_remaining_minutes': '0.1036'}
+{'loss': '2.245', 'grad_norm': '0.5703', 'learning_rate': '0.001', 'epoch': '0.5391', 'train/total_time_seconds': '311.1', 'train/time_per_step_avg': '0.3265', 'train/epoch_time_elapsed': '691.1', 'train/estimated_remaining_minutes': '0'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+{'eval_loss': '2.239', 'eval_runtime': '9.298', 'eval_samples_per_second': '1025', 'eval_steps_per_second': '2.044', 'epoch': '0.5391', 'train/total_time_seconds': '311.1', 'train/time_per_step_avg': '0.3265', 'train/epoch_time_elapsed': '700.4', 'train/estimated_remaining_minutes': '0'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 158.67it/s]
+100%|██████████| 1000/1000 [11:40<00:00,  1.43it/s], ?it/s]
+{'train_runtime': '701.3', 'train_samples_per_second': '730.1', 'train_steps_per_second': '1.426', 'train_loss': '2.97', 'epoch': '0.5391', 'train/total_time_seconds': '311.1', 'train/time_per_step_avg': '0.3265', 'train/epoch_time_elapsed': '700.4', 'train/estimated_remaining_minutes': '0'}
+100%|██████████| 19/19 [00:08<00:00,  2.17it/s]
+[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 154.24it/s]
+>>> FINISHED glu-waleedglu_low-9L successfully

zain/Activation/wandb/run-20260814_211407-i85cngas/files/wandb-summary.json

@@ -0,0 +1 @@
+{"total_flos":2.323107545088e+15,"eval/runtime":9.5433,"_timestamp":1.786742758641734e+09,"train_samples_per_second":730.125,"train_loss":2.969754108428955,"train/learning_rate":0.001,"train/epoch":0.5390835579514824,"_step":52,"_wandb":{"runtime":710},"train/grad_norm":0.5703125,"train/global_step":1000,"eval/steps_per_second":1.991,"train_steps_per_second":1.426,"eval/loss":2.23911452293396,"_runtime":710,"eval/samples_per_second":998.289,"train_runtime":701.2502,"train/loss":2.245043182373047}

zain/Activation/wandb/run-20260814_211407-i85cngas/logs/debug-core.log

@@ -47,3 +47,16 @@
 {"time":"2026-08-14T21:14:07.911109006Z","level":"INFO","msg":"handleInformInit: received","streamId":"i85cngas","id":"1(@)"}
 {"time":"2026-08-14T21:14:08.168219613Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"i85cngas","id":"1(@)"}
 {"time":"2026-08-14T21:14:13.672910923Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"d8p4y9v5n05m"}
+{"time":"2026-08-14T21:25:58.658206835Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"d8p4y9v5n05m"}
+{"time":"2026-08-14T21:25:59.189862536Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"d8p4y9v5n05m"}
+{"time":"2026-08-14T21:25:59.191285843Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"i85cngas","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.192218621Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"i85cngas","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492428474Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492473057Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492482505Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492489625Z","level":"INFO","msg":"server: is shutting down"}
+{"time":"2026-08-14T21:25:59.492505848Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492527496Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492557564Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-08-14T21:25:59.492679293Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-409188-410559-1836801570/socket","Net":"unix"}}
+{"time":"2026-08-14T21:25:59.492751296Z","level":"INFO","msg":"server: all connections closed"}

zain/Activation/wandb/run-20260814_211407-i85cngas/logs/debug-internal.log

@@ -93,3 +93,19 @@
 {"time":"2026-08-14T21:24:38.835784304Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
 {"time":"2026-08-14T21:24:53.676283225Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":45,"history_lines":1,"events_offset":83,"events_lines":2,"console_offset":82,"console_lines":10}
 {"time":"2026-08-14T21:24:53.807963308Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:25:08.67577254Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":46,"history_lines":1,"events_offset":85,"events_lines":2,"console_offset":90,"console_lines":1}
+{"time":"2026-08-14T21:25:08.813221459Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:25:23.675748872Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":47,"history_lines":1,"events_offset":87,"events_lines":2,"console_offset":90,"console_lines":1}
+{"time":"2026-08-14T21:25:23.862250657Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:25:38.675553902Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":48,"history_lines":1,"events_offset":89,"events_lines":2,"console_offset":90,"console_lines":1}
+{"time":"2026-08-14T21:25:38.897653817Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:25:53.676490134Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":49,"history_lines":3,"events_offset":91,"events_lines":2,"console_offset":90,"console_lines":1}
+{"time":"2026-08-14T21:25:53.925875438Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:25:59.040344747Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-08-14T21:25:59.040864542Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":52,"history_lines":1,"events_offset":93,"events_lines":1,"console_offset":92,"console_lines":18,"uploaded_len":3,"complete":true,"exit_code":0}
+{"time":"2026-08-14T21:25:59.187205462Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:25:59.188550421Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-08-14T21:25:59.191308472Z","level":"INFO","msg":"stream: finishing up"}
+{"time":"2026-08-14T21:25:59.191335194Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-08-14T21:25:59.191433704Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-08-14T21:25:59.191438728Z","level":"INFO","msg":"stream: all finished"}

zain/Activation/wandb/run-20260814_211407-i85cngas/logs/debug.log

@@ -17,3 +17,8 @@ config: {'_wandb': {}}
 2026-08-14 21:14:08,672 INFO    MainThread:409188 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 9, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'glu', 'activation': 'waleedglu_low', 'waleed_beta': 10.0, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/glu-waleedglu_low-9L_run', 'per_device_train_batch_size': 512, 'num_train_epochs': 1, 'max_steps': 1000, 'learning_rate': 0.001, 'lr_scheduler_type': 'constant', 'lr_scheduler_kwargs': None, 'warmup_steps': 0, 'optim': 'adamw_torch', 'optim_args': None, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 1.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-glu-waleedglu_low-9L-2.0M-20260814-211407', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 60000, 'eval_delay': 0, 'per_device_eval_batch_size': 512, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 100, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/A-glu-waleedglu_low-9L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
 2026-08-14 21:14:08,674 INFO    MainThread:409188 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 2001280 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x153f494c3450>>
 2026-08-14 21:14:08,674 INFO    MainThread:409188 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 2001280 None
+2026-08-14 21:25:58,657 INFO    MainThread:409188 [wandb_run.py:_finish():2383] finishing run deepnevro-deepnevro/research1/i85cngas
+2026-08-14 21:25:58,657 INFO    MainThread:409188 [wandb_run.py:_atexit_cleanup():2588] got exitcode: 0
+2026-08-14 21:25:58,658 INFO    MainThread:409188 [wandb_run.py:_restore():2570] restore
+2026-08-14 21:25:58,658 INFO    MainThread:409188 [wandb_run.py:_restore():2576] restore done
+2026-08-14 21:25:59,190 INFO    MainThread:409188 [wandb_run.py:_footer_sync_info():3993] logging synced files

zain/Activation/wandb/run-20260814_211407-i85cngas/run-i85cngas.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b0ebd0512cdc3282aa723f19ce33fc677dc5c04ef4430505d970ad61518c4800
-size 458752
+oid sha256:c4205d484f8c5c9946f91a69fae62658e7117137decdea8fec4aa2fb4748d2f1
+size 525167

zain/Activation/wandb/run-20260814_211507-54wnyexz/logs/debug-core.log

@@ -68,3 +68,10 @@
 {"time":"2026-08-14T21:22:08.718181031Z","level":"INFO","msg":"handleInformInit: received","streamId":"n6fjpg9o","id":"3(@)"}
 {"time":"2026-08-14T21:22:08.976795469Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"n6fjpg9o","id":"3(@)"}
 {"time":"2026-08-14T21:22:14.472169125Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"qlc6t75fdq0u"}
+{"time":"2026-08-14T21:28:18.727248774Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"qlc6t75fdq0u"}
+{"time":"2026-08-14T21:28:20.378913364Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"qlc6t75fdq0u"}
+{"time":"2026-08-14T21:28:20.400345521Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"n6fjpg9o","id":"3(@)"}
+{"time":"2026-08-14T21:28:20.400969576Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"n6fjpg9o","id":"3(@)"}
+{"time":"2026-08-14T21:28:35.344777056Z","level":"INFO","msg":"handleInformInit: received","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:28:35.603024157Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:28:41.078893683Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}

zain/Activation/wandb/run-20260814_212208-n6fjpg9o/files/config.yaml

@@ -0,0 +1,439 @@
+_name_or_path:
+    value: ""
+_wandb:
+    value:
+        cli_version: 0.28.1
+        e:
+            dwqr38wvfx2faq20nlxzk1aeem4hvapm:
+                args:
+                    - --config
+                    - /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/configs/baseline100l.yaml
+                    - --variants
+                    - glu-relu
+                    - glu-gelu
+                    - glu-sigmoid
+                    - glu-waleed10
+                    - glu-silu-waleed10
+                codePath: sweep.py
+                codePathLocal: sweep.py
+                cpu_count: 112
+                cpu_count_logical: 224
+                cudaVersion: "12.4"
+                disk:
+                    /:
+                        total: "1560765693952"
+                        used: "706493382656"
+                email: deepnevro@gmail.com
+                executable: /mnt/data/zainulabideen/zain-exp/notebooks/my_env/bin/python
+                git:
+                    commit: 3fb47c6943d6927eafc39bd399a3d3a520bae74a
+                    remote: https://github.com/w-ahmad1a10/Activation.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-39c684a5-fde6-83d7-1663-0859795881ae
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-68012e5a-38b6-b643-0ca6-62fb66720bf3
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-132944c4-b689-2b5f-89a4-d730401677ab
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-2df386cc-6d26-d0e2-7a2d-a057b0d95864
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bfa16575-1d94-1aa2-4537-2c93433f42ef
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bc6c3e3c-9b90-09ca-c034-774961847c54
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-00a441e1-7c95-e7d6-4c35-43d6b291aea9
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-1c4d29a2-4647-6fce-d8fc-0c5ecfbbd6ea
+                host: deeplens-k3s-node1
+                memory:
+                    total: "2164089937920"
+                os: Linux-5.15.0-126-generic-x86_64-with-glibc2.35
+                program: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py
+                python: CPython 3.11.15
+                root: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation
+                startedAt: "2026-08-14T21:22:08.715735Z"
+                writerId: dwqr38wvfx2faq20nlxzk1aeem4hvapm
+        m:
+            - "1": train/global_step
+              "6":
+                - 3
+              "7": []
+            - "2": '*'
+              "5": 1
+              "6":
+                - 1
+              "7": []
+        python_version: 3.11.15
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 7
+                - 13
+                - 19
+                - 41
+                - 61
+                - 62
+                - 66
+            "4": 3.11.15
+            "5": 0.28.1
+            "6": 5.16.0.dev0
+            "9":
+                "1": transformers_trainer
+            "12": 0.28.1
+            "13": linux-x86_64
+accelerator_config:
+    value:
+        dispatch_batches: null
+        even_batches: true
+        gradient_accumulation_kwargs: null
+        non_blocking: false
+        split_batches: false
+        use_seedable_sampler: true
+activation:
+    value: waleed10
+adam_beta1:
+    value: 0.9
+adam_beta2:
+    value: 0.999
+adam_epsilon:
+    value: 1e-08
+architectures:
+    value: null
+attention_bias:
+    value: false
+attention_dropout:
+    value: 0
+auto_find_batch_size:
+    value: false
+average_tokens_across_devices:
+    value: true
+batch_eval_metrics:
+    value: false
+bf16:
+    value: true
+bf16_full_eval:
+    value: false
+bos_token_id:
+    value: 1
+chunk_size_feed_forward:
+    value: 0
+data_seed:
+    value: 42
+dataloader_drop_last:
+    value: false
+dataloader_in_order:
+    value: true
+dataloader_multiprocessing_context:
+    value: null
+dataloader_num_workers:
+    value: 0
+dataloader_persistent_workers:
+    value: false
+dataloader_pin_memory:
+    value: true
+dataloader_prefetch_factor:
+    value: null
+ddp_backend:
+    value: null
+ddp_broadcast_buffers:
+    value: null
+ddp_bucket_cap_mb:
+    value: null
+ddp_find_unused_parameters:
+    value: null
+ddp_static_graph:
+    value: null
+ddp_timeout:
+    value: 1800
+debug:
+    value: []
+deepspeed:
+    value: null
+disable_tqdm:
+    value: false
+do_eval:
+    value: true
+do_predict:
+    value: false
+do_train:
+    value: false
+dtype:
+    value: null
+enable_jit_checkpoint:
+    value: false
+eos_token_id:
+    value: 2
+eval_accumulation_steps:
+    value: null
+eval_delay:
+    value: 0
+eval_do_concat_batches:
+    value: true
+eval_on_start:
+    value: false
+eval_steps:
+    value: 2498
+eval_strategy:
+    value: steps
+eval_use_gather_object:
+    value: false
+fp16:
+    value: false
+fp16_full_eval:
+    value: false
+fsdp:
+    value: null
+fsdp_config:
+    value: null
+full_determinism:
+    value: false
+gradient_accumulation_steps:
+    value: 1
+gradient_checkpointing:
+    value: false
+gradient_checkpointing_kwargs:
+    value: null
+greater_is_better:
+    value: null
+head_dim:
+    value: 32
+hidden_act:
+    value: silu
+hidden_size:
+    value: 128
+hub_always_push:
+    value: false
+hub_model_id:
+    value: w-ahmad/6L-glu-waleed10-100L
+hub_private_repo:
+    value: null
+hub_revision:
+    value: null
+hub_strategy:
+    value: every_save
+hub_token:
+    value: <HUB_TOKEN>
+id2label:
+    value:
+        "0": LABEL_0
+        "1": LABEL_1
+ignore_data_skip:
+    value: false
+include_for_metrics:
+    value: []
+include_num_input_tokens_seen:
+    value: "no"
+initializer_range:
+    value: 0.02
+intermediate_size:
+    value: 256
+is_encoder_decoder:
+    value: false
+label_names:
+    value: null
+label_smoothing_factor:
+    value: 0
+label2id:
+    value:
+        LABEL_0: 0
+        LABEL_1: 1
+learning_rate:
+    value: 9e-05
+length_column_name:
+    value: length
+liger_kernel_config:
+    value: null
+load_best_model_at_end:
+    value: false
+local_rank:
+    value: -1
+log_level:
+    value: passive
+log_level_replica:
+    value: warning
+log_on_each_node:
+    value: true
+logging_first_step:
+    value: false
+logging_nan_inf_filter:
+    value: true
+logging_steps:
+    value: 20
+logging_strategy:
+    value: steps
+lr_scheduler_kwargs:
+    value: null
+lr_scheduler_type:
+    value: constant_with_warmup
+max_grad_norm:
+    value: 1
+max_position_embeddings:
+    value: 512
+max_steps:
+    value: 3500
+metric_for_best_model:
+    value: null
+mlp_bias:
+    value: false
+mlp_type:
+    value: glu
+model/num_parameters:
+    value: 16934016
+model_type:
+    value: tiny_llama
+neftune_noise_alpha:
+    value: null
+num_attention_heads:
+    value: 4
+num_hidden_layers:
+    value: 100
+num_key_value_heads:
+    value: 4
+num_train_epochs:
+    value: 1
+optim:
+    value: adamw_torch_fused
+optim_args:
+    value: null
+optim_target_modules:
+    value: null
+output_attentions:
+    value: false
+output_dir:
+    value: out/glu-waleed10-100L_trash_run
+output_hidden_states:
+    value: false
+pad_token_id:
+    value: 0
+parallelism_config:
+    value: null
+per_device_eval_batch_size:
+    value: 1024
+per_device_train_batch_size:
+    value: 64
+prediction_loss_only:
+    value: false
+pretraining_tp:
+    value: 1
+problem_type:
+    value: null
+project:
+    value: huggingface
+push_to_hub:
+    value: false
+remove_unused_columns:
+    value: false
+report_to:
+    value:
+        - wandb
+restore_callback_states_from_checkpoint:
+    value: false
+resume_from_checkpoint:
+    value: null
+return_dict:
+    value: true
+rms_norm_eps:
+    value: 1e-06
+rope_parameters:
+    value:
+        rope_theta: 10000
+        rope_type: default
+run_name:
+    value: LM-glu-waleed10-100L-16.9M-20260814-212207
+save_on_each_node:
+    value: false
+save_only_model:
+    value: false
+save_steps:
+    value: 50
+save_strategy:
+    value: steps
+save_total_limit:
+    value: null
+seed:
+    value: 42
+skip_memory_metrics:
+    value: true
+tf32:
+    value: null
+tie_word_embeddings:
+    value: true
+tokenizer_name:
+    value: w-ahmad/tiny-stories-tokenizer
+torch_compile:
+    value: false
+torch_compile_backend:
+    value: null
+torch_compile_mode:
+    value: null
+torch_empty_cache_steps:
+    value: null
+trackio_bucket_id:
+    value: null
+trackio_space_id:
+    value: null
+trackio_static_space_id:
+    value: null
+train_sampling_strategy:
+    value: random
+transformers_version:
+    value: 5.16.0.dev0
+use_cache:
+    value: false
+use_cpu:
+    value: false
+use_liger_kernel:
+    value: false
+vocab_size:
+    value: 4096
+waleed_beta:
+    value: 4
+warmup_steps:
+    value: 50
+weight_decay:
+    value: 0

zain/Activation/wandb/run-20260814_212208-n6fjpg9o/files/output.log

@@ -39,6 +39,73 @@ Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 18.
   - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
   - If you are not the owner of the model architecture class, please contact the model code owner to update it.
 Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 14.39it/s]
- 84%|████████▍ | 2955/3500 [02:49<03:39,  2.48it/s], ?it/s]
+ 86%|████████▌ | 3000/3500 [03:05<02:55,  2.85it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
 {'loss': '7.945', 'grad_norm': '2.851e+06', 'learning_rate': '9e-05', 'epoch': '0.1968', 'train/total_time_seconds': '133.2', 'train/time_per_step_avg': '0.3165', 'train/epoch_time_elapsed': '156.7', 'train/estimated_remaining_minutes': '0.4411', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.3', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001747', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2715', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5469', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '188.6', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.001678', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09229', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3906', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3164', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3906', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.707', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '4447', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.009949', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.539', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '6.156', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-5.719', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '6.156', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '11.88', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '4448', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '-0.006439', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '1.539', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '5.438', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-5.375', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '5.438', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '10.81', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '6.545e+04', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '24.88', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '20', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '101', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0.6769', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '1.109', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '101', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '99.89', 'train/tensor_act_model_layers_0_mlp/norm': '7920', 'train/tensor_act_model_layers_0_mlp/mean': '3.859', 'train/tensor_act_model_layers_0_mlp/std': '0.3281', 'train/tensor_act_model_layers_0_mlp/max_abs': '4', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '1.078', 'train/tensor_act_model_layers_0_mlp/max': '4', 'train/tensor_act_model_layers_0_mlp/range': '2.922', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '7926', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '3.859', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.3418', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '4.312', 'train/tensor_act_model_lay
 {'loss': '7.946', 'grad_norm': '2.064e+06', 'learning_rate': '9e-05', 'epoch': '0.1982', 'train/total_time_seconds': '139.5', 'train/time_per_step_avg': '0.3138', 'train/epoch_time_elapsed': '164', 'train/estimated_remaining_minutes': '0.443'}
+{'loss': '7.938', 'grad_norm': '1.499e+06', 'learning_rate': '9e-05', 'epoch': '0.1995', 'train/total_time_seconds': '146', 'train/time_per_step_avg': '0.3136', 'train/epoch_time_elapsed': '171.6', 'train/estimated_remaining_minutes': '0.4438', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.3', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001625', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2715', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5469', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '188.7', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.001595', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09229', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3906', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3418', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3906', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.7324', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '4585', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.01001', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.586', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '6.312', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-5.719', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '6.312', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '12.03', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '4587', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '-0.005768', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '1.586', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '5.75', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-5.469', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '5.75', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '11.22', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '7.202e+04', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '28.38', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '20.75', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '107', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0.8107', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '1.93', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '107', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '105.1', 'train/tensor_act_model_layers_0_mlp/norm': '8099', 'train/tensor_act_model_layers_0_mlp/mean': '3.953', 'train/tensor_act_model_layers_0_mlp/std': '0.1328', 'train/tensor_act_model_layers_0_mlp/max_abs': '4', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '1.789', 'train/tensor_act_model_layers_0_mlp/max': '4', 'train/tensor_act_model_layers_0_mlp/range': '2.211', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '8104', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '3.953', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.1621', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '4.375', 'train/tensor_act_model_layer
+{'loss': '7.943', 'grad_norm': '1.057e+06', 'learning_rate': '9e-05', 'epoch': '0.2009', 'train/total_time_seconds': '152', 'train/time_per_step_avg': '0.3155', 'train/epoch_time_elapsed': '178.7', 'train/estimated_remaining_minutes': '0.442'}
+{'loss': '7.944', 'grad_norm': '8.069e+05', 'learning_rate': '9e-05', 'epoch': '0.2022', 'train/total_time_seconds': '158', 'train/time_per_step_avg': '0.3142', 'train/epoch_time_elapsed': '185.7', 'train/estimated_remaining_minutes': '0.4388'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 15.48it/s]
+ 89%|████████▊ | 3100/3500 [03:41<01:56,  3.44it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '7.943', 'grad_norm': '6.226e+05', 'learning_rate': '9e-05', 'epoch': '0.2036', 'train/total_time_seconds': '164.9', 'train/time_per_step_avg': '0.3162', 'train/epoch_time_elapsed': '193.9', 'train/estimated_remaining_minutes': '0.4367', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.5', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001663', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2715', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5469', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '188.7', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.00164', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09229', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3496', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3066', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3496', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.6562', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '4633', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.009521', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.602', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '6.531', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-5.75', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '6.531', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '12.28', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '4635', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '-0.004761', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '1.602', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '5.5', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-5.344', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '5.5', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '10.84', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '7.552e+04', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '30.25', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '21', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '108', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0.8997', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '2.891', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '108', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '105.1', 'train/tensor_act_model_layers_0_mlp/norm': '8153', 'train/tensor_act_model_layers_0_mlp/mean': '3.984', 'train/tensor_act_model_layers_0_mlp/std': '0.05884', 'train/tensor_act_model_layers_0_mlp/max_abs': '4', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '2.469', 'train/tensor_act_model_layers_0_mlp/max': '4', 'train/tensor_act_model_layers_0_mlp/range': '1.531', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '8159', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '3.984', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.1094', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '4.312', 'train/tensor_act_model_layers_
+{'loss': '7.948', 'grad_norm': '4.669e+05', 'learning_rate': '9e-05', 'epoch': '0.2049', 'train/total_time_seconds': '170.6', 'train/time_per_step_avg': '0.3105', 'train/epoch_time_elapsed': '200.7', 'train/estimated_remaining_minutes': '0.4302'}
+{'loss': '7.942', 'grad_norm': '3.645e+05', 'learning_rate': '9e-05', 'epoch': '0.2063', 'train/total_time_seconds': '177.6', 'train/time_per_step_avg': '0.316', 'train/epoch_time_elapsed': '208.8', 'train/estimated_remaining_minutes': '0.4256', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.2', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001671', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2715', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5469', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '188.5', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.00161', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.0918', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3301', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3281', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3301', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.6582', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '4695', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.009033', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.625', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '6.531', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-5.844', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '6.531', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '12.38', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '4694', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '-0.007019', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '1.617', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '5.625', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-5.406', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '5.625', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '11.03', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '7.853e+04', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '32', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '21.12', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '114.5', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0.9573', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '4.219', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '114.5', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '110.3', 'train/tensor_act_model_layers_0_mlp/norm': '8175', 'train/tensor_act_model_layers_0_mlp/mean': '3.984', 'train/tensor_act_model_layers_0_mlp/std': '0.02795', 'train/tensor_act_model_layers_0_mlp/max_abs': '4', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '3.141', 'train/tensor_act_model_layers_0_mlp/max': '4', 'train/tensor_act_model_layers_0_mlp/range': '0.8594', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '8180', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '4', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.09619', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '4.312', 'train/tensor_act_model_la
+{'loss': '7.942', 'grad_norm': '3.052e+05', 'learning_rate': '9e-05', 'epoch': '0.2076', 'train/total_time_seconds': '183', 'train/time_per_step_avg': '0.3101', 'train/epoch_time_elapsed': '215.2', 'train/estimated_remaining_minutes': '0.4158'}
+{'loss': '7.946', 'grad_norm': '2.437e+05', 'learning_rate': '9e-05', 'epoch': '0.209', 'train/total_time_seconds': '188', 'train/time_per_step_avg': '0.3008', 'train/epoch_time_elapsed': '221.3', 'train/estimated_remaining_minutes': '0.4044'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 18.72it/s]
+ 91%|█████████▏| 3200/3500 [04:10<01:21,  3.69it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '7.948', 'grad_norm': '1.976e+05', 'learning_rate': '9e-05', 'epoch': '0.2103', 'train/total_time_seconds': '193.7', 'train/time_per_step_avg': '0.2887', 'train/epoch_time_elapsed': '228.3', 'train/estimated_remaining_minutes': '0.3933', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.4', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001686', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2715', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5469', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '188.6', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.001694', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09229', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3262', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3262', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3203', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.6465', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '4683', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.008606', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.617', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '6.438', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-5.875', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '6.438', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '12.31', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '4678', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '-0.005646', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '1.617', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '5.656', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-5.344', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '5.656', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '11', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '7.929e+04', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '32.5', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '21', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '116', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0.9792', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '5.5', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '116', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '110.5', 'train/tensor_act_model_layers_0_mlp/norm': '8181', 'train/tensor_act_model_layers_0_mlp/mean': '4', 'train/tensor_act_model_layers_0_mlp/std': '0.01721', 'train/tensor_act_model_layers_0_mlp/max_abs': '4', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '3.516', 'train/tensor_act_model_layers_0_mlp/max': '4', 'train/tensor_act_model_layers_0_mlp/range': '0.4844', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '8187', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '4', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.09375', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '4.312', 'train/tensor_act_model_layers_0_resi
+{'loss': '7.943', 'grad_norm': '1.649e+05', 'learning_rate': '9e-05', 'epoch': '0.2117', 'train/total_time_seconds': '198.4', 'train/time_per_step_avg': '0.278', 'train/epoch_time_elapsed': '234', 'train/estimated_remaining_minutes': '0.3791'}
+{'loss': '7.944', 'grad_norm': '1.505e+05', 'learning_rate': '9e-05', 'epoch': '0.213', 'train/total_time_seconds': '203.3', 'train/time_per_step_avg': '0.2569', 'train/epoch_time_elapsed': '240', 'train/estimated_remaining_minutes': '0.3645', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.3', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001633', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2715', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5469', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '188.5', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.001633', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09229', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3418', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3418', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3301', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.6719', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '4703', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.009766', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.625', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '6.344', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-5.719', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '6.344', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '12.06', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '4697', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '-0.006012', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '1.625', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '5.625', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-5.406', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '5.625', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '11.03', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '8.064e+04', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '33.25', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '21', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '116.5', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0.9919', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '6.625', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '116.5', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '109.9', 'train/tensor_act_model_layers_0_mlp/norm': '8186', 'train/tensor_act_model_layers_0_mlp/mean': '4', 'train/tensor_act_model_layers_0_mlp/std': '0.01105', 'train/tensor_act_model_layers_0_mlp/max_abs': '4', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '3.719', 'train/tensor_act_model_layers_0_mlp/max': '4', 'train/tensor_act_model_layers_0_mlp/range': '0.2812', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '8191', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '4', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.09277', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '4.344', 'train/tensor_act_model_layers
+{'loss': '7.941', 'grad_norm': '1.249e+05', 'learning_rate': '9e-05', 'epoch': '0.2144', 'train/total_time_seconds': '207.7', 'train/time_per_step_avg': '0.2475', 'train/epoch_time_elapsed': '245.5', 'train/estimated_remaining_minutes': '0.3484'}
+{'loss': '7.943', 'grad_norm': '1.162e+05', 'learning_rate': '9e-05', 'epoch': '0.2157', 'train/total_time_seconds': '212.2', 'train/time_per_step_avg': '0.2413', 'train/epoch_time_elapsed': '251', 'train/estimated_remaining_minutes': '0.3315'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 17.05it/s]
+ 94%|█████████▍| 3300/3500 [04:39<00:54,  3.69it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '7.946', 'grad_norm': '9.216e+04', 'learning_rate': '9e-05', 'epoch': '0.2171', 'train/total_time_seconds': '217', 'train/time_per_step_avg': '0.2331', 'train/epoch_time_elapsed': '257.1', 'train/estimated_remaining_minutes': '0.3146', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.5', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001595', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2715', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5469', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '188.8', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.001617', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09229', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3418', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3418', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3281', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.6699', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '4696', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.009766', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.625', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '6.469', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-5.875', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '6.469', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '12.34', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '4690', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '-0.006653', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '1.617', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '5.594', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-5.5', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '5.594', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '11.09', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '8.094e+04', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '33.5', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '20.88', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '114.5', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0.9958', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '7.406', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '114.5', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '107.1', 'train/tensor_act_model_layers_0_mlp/norm': '8188', 'train/tensor_act_model_layers_0_mlp/mean': '4', 'train/tensor_act_model_layers_0_mlp/std': '0.007935', 'train/tensor_act_model_layers_0_mlp/max_abs': '4', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '3.812', 'train/tensor_act_model_layers_0_mlp/max': '4', 'train/tensor_act_model_layers_0_mlp/range': '0.1875', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '8193', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '4', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.09277', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '4.312', 'train/tensor_act_model_laye
+{'loss': '7.944', 'grad_norm': '8.96e+04', 'learning_rate': '9e-05', 'epoch': '0.2184', 'train/total_time_seconds': '221.5', 'train/time_per_step_avg': '0.2309', 'train/epoch_time_elapsed': '262.9', 'train/estimated_remaining_minutes': '0.2962'}
+{'loss': '7.945', 'grad_norm': '8.09e+04', 'learning_rate': '9e-05', 'epoch': '0.2198', 'train/total_time_seconds': '226.4', 'train/time_per_step_avg': '0.2313', 'train/epoch_time_elapsed': '269', 'train/estimated_remaining_minutes': '0.2778', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.5', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001648', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.0918', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2715', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5469', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '188.8', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.00164', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09229', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3262', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3242', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3262', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.6504', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '4682', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.00766', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.617', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '6.469', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-6.031', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '6.469', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '12.5', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '4675', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '-0.007874', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '1.617', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '5.688', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-5.344', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '5.688', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '11.03', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '8.103e+04', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '33.75', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '20.88', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '110.5', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0.9976', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '8', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '110.5', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '102.5', 'train/tensor_act_model_layers_0_mlp/norm': '8189', 'train/tensor_act_model_layers_0_mlp/mean': '4', 'train/tensor_act_model_layers_0_mlp/std': '0.006287', 'train/tensor_act_model_layers_0_mlp/max_abs': '4', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '3.859', 'train/tensor_act_model_layers_0_mlp/max': '4', 'train/tensor_act_model_layers_0_mlp/range': '0.1406', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '8194', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '4', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.09229', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '4.312', 'train/tensor_act_model_layers_0_r
+{'loss': '7.945', 'grad_norm': '7.117e+04', 'learning_rate': '9e-05', 'epoch': '0.2211', 'train/total_time_seconds': '230.8', 'train/time_per_step_avg': '0.2309', 'train/epoch_time_elapsed': '274.5', 'train/estimated_remaining_minutes': '0.258'}
+{'loss': '7.945', 'grad_norm': '6.605e+04', 'learning_rate': '9e-05', 'epoch': '0.2224', 'train/total_time_seconds': '235.3', 'train/time_per_step_avg': '0.2309', 'train/epoch_time_elapsed': '279.9', 'train/estimated_remaining_minutes': '0.2376'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 18.23it/s]
+ 97%|█████████▋| 3400/3500 [05:08<00:27,  3.68it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '7.941', 'grad_norm': '5.555e+04', 'learning_rate': '9e-05', 'epoch': '0.2238', 'train/total_time_seconds': '240.1', 'train/time_per_step_avg': '0.2306', 'train/epoch_time_elapsed': '286.1', 'train/estimated_remaining_minutes': '0.217', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.3', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001633', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2715', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5469', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '188.5', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.001617', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.0918', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3223', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3223', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3203', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.6426', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '4695', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.008484', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.617', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '6.219', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-5.906', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '6.219', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '12.12', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '4687', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '-0.006805', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '1.617', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '5.625', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-5.344', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '5.625', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '10.97', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '8.162e+04', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '34', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '20.75', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '111.5', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0.9984', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '8.062', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '111.5', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '103.4', 'train/tensor_act_model_layers_0_mlp/norm': '8190', 'train/tensor_act_model_layers_0_mlp/mean': '4', 'train/tensor_act_model_layers_0_mlp/std': '0.005157', 'train/tensor_act_model_layers_0_mlp/max_abs': '4', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '3.859', 'train/tensor_act_model_layers_0_mlp/max': '4', 'train/tensor_act_model_layers_0_mlp/range': '0.1406', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '8195', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '4', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.09229', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '4.312', 'train/tensor_act_model_laye
+{'loss': '7.943', 'grad_norm': '5.094e+04', 'learning_rate': '9e-05', 'epoch': '0.2251', 'train/total_time_seconds': '244.5', 'train/time_per_step_avg': '0.2306', 'train/epoch_time_elapsed': '291.5', 'train/estimated_remaining_minutes': '0.1952'}
+{'loss': '7.945', 'grad_norm': '4.506e+04', 'learning_rate': '9e-05', 'epoch': '0.2265', 'train/total_time_seconds': '249.4', 'train/time_per_step_avg': '0.2299', 'train/epoch_time_elapsed': '297.5', 'train/estimated_remaining_minutes': '0.1732', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.1', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001587', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2715', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5469', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '188.4', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.001564', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.0918', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3906', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3281', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3906', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.7188', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '4705', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.008057', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.625', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '6.375', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-5.812', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '6.375', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '12.19', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '4697', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '-0.008301', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '1.625', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '5.625', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-5.562', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '5.625', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '11.19', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '8.235e+04', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '34.5', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '20.75', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '112', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0.9994', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '8.75', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '112', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '103.2', 'train/tensor_act_model_layers_0_mlp/norm': '8190', 'train/tensor_act_model_layers_0_mlp/mean': '4', 'train/tensor_act_model_layers_0_mlp/std': '0.004181', 'train/tensor_act_model_layers_0_mlp/max_abs': '4', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '3.906', 'train/tensor_act_model_layers_0_mlp/max': '4', 'train/tensor_act_model_layers_0_mlp/range': '0.09375', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '8195', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '4', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.09229', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '4.375', 'train/tensor_act_model_layer
+{'loss': '7.945', 'grad_norm': '4.147e+04', 'learning_rate': '9e-05', 'epoch': '0.2278', 'train/total_time_seconds': '253.9', 'train/time_per_step_avg': '0.2303', 'train/epoch_time_elapsed': '303', 'train/estimated_remaining_minutes': '0.1502'}
+{'loss': '7.947', 'grad_norm': '3.61e+04', 'learning_rate': '9e-05', 'epoch': '0.2292', 'train/total_time_seconds': '258.3', 'train/time_per_step_avg': '0.2302', 'train/epoch_time_elapsed': '308.4', 'train/estimated_remaining_minutes': '0.1266'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 15.84it/s]
+100%|██████████| 3500/3500 [05:52<00:00,  3.66i[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '7.943', 'grad_norm': '3.098e+04', 'learning_rate': '9e-05', 'epoch': '0.2305', 'train/total_time_seconds': '263.1', 'train/time_per_step_avg': '0.2301', 'train/epoch_time_elapsed': '314.6', 'train/estimated_remaining_minutes': '0.1026', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.3', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001648', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2715', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5469', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '188.6', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.001602', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09229', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3125', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3125', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3105', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.623', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '4703', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.008362', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.625', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '6.469', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-5.938', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '6.469', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '12.41', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '4690', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '-0.00705', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '1.617', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '5.812', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-5.625', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '5.812', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '11.44', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '8.271e+04', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '34.5', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '20.88', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '112', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0.9999', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '9.375', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '112', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '102.6', 'train/tensor_act_model_layers_0_mlp/norm': '8191', 'train/tensor_act_model_layers_0_mlp/mean': '4', 'train/tensor_act_model_layers_0_mlp/std': '0.003555', 'train/tensor_act_model_layers_0_mlp/max_abs': '4', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '3.922', 'train/tensor_act_model_layers_0_mlp/max': '4', 'train/tensor_act_model_layers_0_mlp/range': '0.07812', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '8196', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '4', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.09229', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '4.312', 'train/tensor_act_model_layer
+{'loss': '7.943', 'grad_norm': '2.598e+04', 'learning_rate': '9e-05', 'epoch': '0.2319', 'train/total_time_seconds': '267.6', 'train/time_per_step_avg': '0.2306', 'train/epoch_time_elapsed': '320.1', 'train/estimated_remaining_minutes': '0.07779'}
+{'loss': '7.947', 'grad_norm': '2.125e+04', 'learning_rate': '9e-05', 'epoch': '0.2332', 'train/total_time_seconds': '272.5', 'train/time_per_step_avg': '0.2308', 'train/epoch_time_elapsed': '326.1', 'train/estimated_remaining_minutes': '0.0525', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.5', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001755', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2715', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5469', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '188.8', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.001747', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09229', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.373', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3418', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.373', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.7148', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '4689', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.007751', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.617', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '6.25', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-5.844', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '6.25', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '12.09', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '4675', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '-0.008545', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '1.617', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '5.625', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-5.5', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '5.625', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '11.12', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '8.314e+04', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '34.75', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '21', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '112', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '1', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '8.188', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '112', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '103.8', 'train/tensor_act_model_layers_0_mlp/norm': '8191', 'train/tensor_act_model_layers_0_mlp/mean': '4', 'train/tensor_act_model_layers_0_mlp/std': '0.002655', 'train/tensor_act_model_layers_0_mlp/max_abs': '4', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '3.875', 'train/tensor_act_model_layers_0_mlp/max': '4', 'train/tensor_act_model_layers_0_mlp/range': '0.125', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '8196', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '4', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.09229', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '4.375', 'train/tensor_act_model_layers_0_residual_
+{'loss': '7.947', 'grad_norm': '1.933e+04', 'learning_rate': '9e-05', 'epoch': '0.2346', 'train/total_time_seconds': '277', 'train/time_per_step_avg': '0.2314', 'train/epoch_time_elapsed': '331.7', 'train/estimated_remaining_minutes': '0.02653'}
+{'loss': '7.949', 'grad_norm': '1.389e+04', 'learning_rate': '9e-05', 'epoch': '0.2359', 'train/total_time_seconds': '281.5', 'train/time_per_step_avg': '0.232', 'train/epoch_time_elapsed': '337.2', 'train/estimated_remaining_minutes': '0'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+{'eval_loss': '7.945', 'eval_runtime': '15.6', 'eval_samples_per_second': '610.9', 'eval_steps_per_second': '0.641', 'epoch': '0.2359', 'train/total_time_seconds': '281.5', 'train/time_per_step_avg': '0.232', 'train/epoch_time_elapsed': '352.8', 'train/estimated_remaining_minutes': '0'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 14.20it/s]
+100%|██████████| 3500/3500 [05:53<00:00,  9.91it/s], ?it/s]
+{'train_runtime': '353.9', 'train_samples_per_second': '632.9', 'train_steps_per_second': '9.889', 'train_loss': '2.171', 'epoch': '0.2359', 'train/total_time_seconds': '281.5', 'train/time_per_step_avg': '0.232', 'train/epoch_time_elapsed': '353', 'train/estimated_remaining_minutes': '0'}
+100%|██████████| 10/10 [00:13<00:00,  1.40s/it]
+[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 15.65it/s]
+>>> FINISHED glu-waleed10-100L successfully

zain/Activation/wandb/run-20260814_212208-n6fjpg9o/logs/debug-core.log

@@ -68,3 +68,10 @@
 {"time":"2026-08-14T21:22:08.718181031Z","level":"INFO","msg":"handleInformInit: received","streamId":"n6fjpg9o","id":"3(@)"}
 {"time":"2026-08-14T21:22:08.976795469Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"n6fjpg9o","id":"3(@)"}
 {"time":"2026-08-14T21:22:14.472169125Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"qlc6t75fdq0u"}
+{"time":"2026-08-14T21:28:18.727248774Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"qlc6t75fdq0u"}
+{"time":"2026-08-14T21:28:20.378913364Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"qlc6t75fdq0u"}
+{"time":"2026-08-14T21:28:20.400345521Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"n6fjpg9o","id":"3(@)"}
+{"time":"2026-08-14T21:28:20.400969576Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"n6fjpg9o","id":"3(@)"}
+{"time":"2026-08-14T21:28:35.344777056Z","level":"INFO","msg":"handleInformInit: received","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:28:35.603024157Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:28:41.078893683Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}

zain/Activation/wandb/run-20260814_212208-n6fjpg9o/logs/debug-internal.log

@@ -37,3 +37,37 @@
 {"time":"2026-08-14T21:24:40.427030871Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
 {"time":"2026-08-14T21:24:54.460542104Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":20,"history_lines":2,"events_offset":19,"events_lines":2,"console_offset":35,"console_lines":9}
 {"time":"2026-08-14T21:24:55.385800453Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:25:09.459223532Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":22,"history_lines":2,"events_offset":21,"events_lines":2,"console_offset":41,"console_lines":1}
+{"time":"2026-08-14T21:25:10.338668442Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:25:24.454676952Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":24,"history_lines":2,"events_offset":23,"events_lines":2,"console_offset":41,"console_lines":1}
+{"time":"2026-08-14T21:25:25.245490507Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:25:39.467654857Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":26,"history_lines":2,"events_offset":25,"events_lines":2,"console_offset":44,"console_lines":11}
+{"time":"2026-08-14T21:25:40.389907102Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:25:54.463154808Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":28,"history_lines":2,"events_offset":27,"events_lines":2,"console_offset":51,"console_lines":1}
+{"time":"2026-08-14T21:25:55.24970084Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:26:09.464095349Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":30,"history_lines":2,"events_offset":29,"events_lines":2,"console_offset":55,"console_lines":9}
+{"time":"2026-08-14T21:26:10.391736659Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:26:24.458991962Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":32,"history_lines":3,"events_offset":31,"events_lines":2,"console_offset":61,"console_lines":1}
+{"time":"2026-08-14T21:26:25.382996244Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:26:39.461578547Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":35,"history_lines":3,"events_offset":33,"events_lines":2,"console_offset":64,"console_lines":11}
+{"time":"2026-08-14T21:26:40.376873392Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:26:54.461092479Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":38,"history_lines":2,"events_offset":35,"events_lines":2,"console_offset":71,"console_lines":1}
+{"time":"2026-08-14T21:26:55.499948157Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:27:09.464018745Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":40,"history_lines":3,"events_offset":37,"events_lines":2,"console_offset":75,"console_lines":10}
+{"time":"2026-08-14T21:27:10.348585635Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:27:24.46119055Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":43,"history_lines":3,"events_offset":39,"events_lines":2,"console_offset":81,"console_lines":1}
+{"time":"2026-08-14T21:27:25.337127716Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:27:39.459568913Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":46,"history_lines":2,"events_offset":41,"events_lines":2,"console_offset":85,"console_lines":10}
+{"time":"2026-08-14T21:27:40.420961456Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:27:54.460221415Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":48,"history_lines":2,"events_offset":43,"events_lines":2,"console_offset":91,"console_lines":1}
+{"time":"2026-08-14T21:27:55.378108654Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:28:09.454435714Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":50,"history_lines":2,"events_offset":45,"events_lines":2,"console_offset":91,"console_lines":1}
+{"time":"2026-08-14T21:28:10.354369244Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:28:19.516181311Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-08-14T21:28:19.529178819Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":52,"history_lines":1,"events_offset":47,"events_lines":1,"console_offset":95,"console_lines":16,"uploaded_len":3,"complete":true,"exit_code":0}
+{"time":"2026-08-14T21:28:20.359553197Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:28:20.360958219Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-08-14T21:28:20.40038758Z","level":"INFO","msg":"stream: finishing up"}
+{"time":"2026-08-14T21:28:20.400431303Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-08-14T21:28:20.400559672Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-08-14T21:28:20.400565215Z","level":"INFO","msg":"stream: all finished"}

zain/Activation/wandb/run-20260814_212208-n6fjpg9o/logs/debug.log

@@ -18,3 +18,8 @@ config: {'_wandb': {}}
 2026-08-14 21:22:09,429 INFO    MainThread:936813 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 100, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'glu', 'activation': 'waleed10', 'waleed_beta': 4, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/glu-waleed10-100L_trash_run', 'per_device_train_batch_size': 64, 'num_train_epochs': 1, 'max_steps': 3500, 'learning_rate': 9e-05, 'lr_scheduler_type': 'constant_with_warmup', 'lr_scheduler_kwargs': None, 'warmup_steps': 50, 'optim': 'adamw_torch_fused', 'optim_args': None, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 1.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-glu-waleed10-100L-16.9M-20260814-212207', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 2498, 'eval_delay': 0, 'per_device_eval_batch_size': 1024, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 50, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/6L-glu-waleed10-100L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
 2026-08-14 21:22:09,434 INFO    MainThread:936813 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 16934016 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x14ee39bf4050>>
 2026-08-14 21:22:09,434 INFO    MainThread:936813 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 16934016 None
+2026-08-14 21:28:18,726 INFO    MainThread:936813 [wandb_run.py:_finish():2383] finishing run deepnevro-deepnevro/research7/n6fjpg9o
+2026-08-14 21:28:18,726 INFO    MainThread:936813 [wandb_run.py:_atexit_cleanup():2588] got exitcode: 0
+2026-08-14 21:28:18,727 INFO    MainThread:936813 [wandb_run.py:_restore():2570] restore
+2026-08-14 21:28:18,727 INFO    MainThread:936813 [wandb_run.py:_restore():2576] restore done
+2026-08-14 21:28:20,399 INFO    MainThread:936813 [wandb_run.py:_footer_sync_info():3993] logging synced files

zain/Activation/wandb/run-20260814_212208-n6fjpg9o/run-n6fjpg9o.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e0184c598b27cc4154a7f934240743bb9d6d7505a18ca9a86ea5a3eeebfdaf07
-size 22315008
+oid sha256:270aeec28f07f1e18ac8153551f79ad41dae1453006422476b3eba2307f8afbc
+size 51292143

zain/Activation/wandb/run-20260814_212814-vtp6pf3m/files/config.yaml

@@ -0,0 +1,438 @@
+_name_or_path:
+    value: ""
+_wandb:
+    value:
+        cli_version: 0.28.1
+        e:
+            yu21vcw1fqgtt12osjsu76tw34s21eqd:
+                args:
+                    - --config
+                    - /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/configs/baseline.yaml
+                    - --variants
+                    - glu-waleed10
+                    - glu-silu-waleed10
+                    - mlp-waleed10
+                    - mlp-silu-waleed10
+                    - glu-waleed
+                    - glu-situglu_low
+                    - glu-waleedglu_low
+                codePath: sweep.py
+                codePathLocal: sweep.py
+                cpu_count: 112
+                cpu_count_logical: 224
+                cudaVersion: "12.4"
+                disk:
+                    /:
+                        total: "1560765693952"
+                        used: "706489905152"
+                email: deepnevro@gmail.com
+                executable: /mnt/data/zainulabideen/zain-exp/notebooks/my_env/bin/python
+                git:
+                    commit: 3fb47c6943d6927eafc39bd399a3d3a520bae74a
+                    remote: https://github.com/w-ahmad1a10/Activation.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-39c684a5-fde6-83d7-1663-0859795881ae
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-68012e5a-38b6-b643-0ca6-62fb66720bf3
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-132944c4-b689-2b5f-89a4-d730401677ab
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-2df386cc-6d26-d0e2-7a2d-a057b0d95864
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bfa16575-1d94-1aa2-4537-2c93433f42ef
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bc6c3e3c-9b90-09ca-c034-774961847c54
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-00a441e1-7c95-e7d6-4c35-43d6b291aea9
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-1c4d29a2-4647-6fce-d8fc-0c5ecfbbd6ea
+                host: deeplens-k3s-node1
+                memory:
+                    total: "2164089937920"
+                os: Linux-5.15.0-126-generic-x86_64-with-glibc2.35
+                program: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py
+                python: CPython 3.11.15
+                root: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation
+                startedAt: "2026-08-14T21:28:14.182237Z"
+                writerId: yu21vcw1fqgtt12osjsu76tw34s21eqd
+        m:
+            - "1": train/global_step
+              "6":
+                - 3
+              "7": []
+            - "2": '*'
+              "5": 1
+              "6":
+                - 1
+              "7": []
+        python_version: 3.11.15
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 7
+                - 13
+                - 19
+                - 66
+            "4": 3.11.15
+            "5": 0.28.1
+            "6": 5.16.0.dev0
+            "9":
+                "1": transformers_trainer
+            "12": 0.28.1
+            "13": linux-x86_64
+accelerator_config:
+    value:
+        dispatch_batches: null
+        even_batches: true
+        gradient_accumulation_kwargs: null
+        non_blocking: false
+        split_batches: false
+        use_seedable_sampler: true
+activation:
+    value: waleed10
+adam_beta1:
+    value: 0.9
+adam_beta2:
+    value: 0.999
+adam_epsilon:
+    value: 1e-08
+architectures:
+    value: null
+attention_bias:
+    value: false
+attention_dropout:
+    value: 0
+auto_find_batch_size:
+    value: false
+average_tokens_across_devices:
+    value: true
+batch_eval_metrics:
+    value: false
+bf16:
+    value: true
+bf16_full_eval:
+    value: false
+bos_token_id:
+    value: 1
+chunk_size_feed_forward:
+    value: 0
+data_seed:
+    value: 42
+dataloader_drop_last:
+    value: false
+dataloader_in_order:
+    value: true
+dataloader_multiprocessing_context:
+    value: null
+dataloader_num_workers:
+    value: 0
+dataloader_persistent_workers:
+    value: false
+dataloader_pin_memory:
+    value: true
+dataloader_prefetch_factor:
+    value: null
+ddp_backend:
+    value: null
+ddp_broadcast_buffers:
+    value: null
+ddp_bucket_cap_mb:
+    value: null
+ddp_find_unused_parameters:
+    value: null
+ddp_static_graph:
+    value: null
+ddp_timeout:
+    value: 1800
+debug:
+    value: []
+deepspeed:
+    value: null
+disable_tqdm:
+    value: false
+do_eval:
+    value: true
+do_predict:
+    value: false
+do_train:
+    value: false
+dtype:
+    value: null
+enable_jit_checkpoint:
+    value: false
+eos_token_id:
+    value: 2
+eval_accumulation_steps:
+    value: null
+eval_delay:
+    value: 0
+eval_do_concat_batches:
+    value: true
+eval_on_start:
+    value: false
+eval_steps:
+    value: 60000
+eval_strategy:
+    value: steps
+eval_use_gather_object:
+    value: false
+fp16:
+    value: false
+fp16_full_eval:
+    value: false
+fsdp:
+    value: null
+fsdp_config:
+    value: null
+full_determinism:
+    value: false
+gradient_accumulation_steps:
+    value: 1
+gradient_checkpointing:
+    value: false
+gradient_checkpointing_kwargs:
+    value: null
+greater_is_better:
+    value: null
+head_dim:
+    value: 32
+hidden_act:
+    value: silu
+hidden_size:
+    value: 128
+hub_always_push:
+    value: false
+hub_model_id:
+    value: w-ahmad/A-glu-waleed10-21L
+hub_private_repo:
+    value: null
+hub_revision:
+    value: null
+hub_strategy:
+    value: every_save
+hub_token:
+    value: <HUB_TOKEN>
+id2label:
+    value:
+        "0": LABEL_0
+        "1": LABEL_1
+ignore_data_skip:
+    value: false
+include_for_metrics:
+    value: []
+include_num_input_tokens_seen:
+    value: "no"
+initializer_range:
+    value: 0.02
+intermediate_size:
+    value: 256
+is_encoder_decoder:
+    value: false
+label_names:
+    value: null
+label_smoothing_factor:
+    value: 0
+label2id:
+    value:
+        LABEL_0: 0
+        LABEL_1: 1
+learning_rate:
+    value: 0.001
+length_column_name:
+    value: length
+liger_kernel_config:
+    value: null
+load_best_model_at_end:
+    value: false
+local_rank:
+    value: -1
+log_level:
+    value: passive
+log_level_replica:
+    value: warning
+log_on_each_node:
+    value: true
+logging_first_step:
+    value: false
+logging_nan_inf_filter:
+    value: true
+logging_steps:
+    value: 20
+logging_strategy:
+    value: steps
+lr_scheduler_kwargs:
+    value: null
+lr_scheduler_type:
+    value: constant
+max_grad_norm:
+    value: 1
+max_position_embeddings:
+    value: 512
+max_steps:
+    value: 1000
+metric_for_best_model:
+    value: null
+mlp_bias:
+    value: false
+mlp_type:
+    value: glu
+model/num_parameters:
+    value: 3970432
+model_type:
+    value: tiny_llama
+neftune_noise_alpha:
+    value: null
+num_attention_heads:
+    value: 4
+num_hidden_layers:
+    value: 21
+num_key_value_heads:
+    value: 4
+num_train_epochs:
+    value: 1
+optim:
+    value: adamw_torch
+optim_args:
+    value: null
+optim_target_modules:
+    value: null
+output_attentions:
+    value: false
+output_dir:
+    value: out/glu-waleed10-21L_run
+output_hidden_states:
+    value: false
+pad_token_id:
+    value: 0
+parallelism_config:
+    value: null
+per_device_eval_batch_size:
+    value: 512
+per_device_train_batch_size:
+    value: 512
+prediction_loss_only:
+    value: false
+pretraining_tp:
+    value: 1
+problem_type:
+    value: null
+project:
+    value: huggingface
+push_to_hub:
+    value: false
+remove_unused_columns:
+    value: false
+report_to:
+    value:
+        - wandb
+restore_callback_states_from_checkpoint:
+    value: false
+resume_from_checkpoint:
+    value: null
+return_dict:
+    value: true
+rms_norm_eps:
+    value: 1e-06
+rope_parameters:
+    value:
+        rope_theta: 10000
+        rope_type: default
+run_name:
+    value: LM-glu-waleed10-21L-4.0M-20260814-212813
+save_on_each_node:
+    value: false
+save_only_model:
+    value: false
+save_steps:
+    value: 100
+save_strategy:
+    value: steps
+save_total_limit:
+    value: null
+seed:
+    value: 42
+skip_memory_metrics:
+    value: true
+tf32:
+    value: null
+tie_word_embeddings:
+    value: true
+tokenizer_name:
+    value: w-ahmad/tiny-stories-tokenizer
+torch_compile:
+    value: false
+torch_compile_backend:
+    value: null
+torch_compile_mode:
+    value: null
+torch_empty_cache_steps:
+    value: null
+trackio_bucket_id:
+    value: null
+trackio_space_id:
+    value: null
+trackio_static_space_id:
+    value: null
+train_sampling_strategy:
+    value: random
+transformers_version:
+    value: 5.16.0.dev0
+use_cache:
+    value: false
+use_cpu:
+    value: false
+use_liger_kernel:
+    value: false
+vocab_size:
+    value: 4096
+waleed_beta:
+    value: 10
+warmup_steps:
+    value: 0
+weight_decay:
+    value: 0.01

zain/Activation/wandb/run-20260814_212814-vtp6pf3m/files/output.log

@@ -0,0 +1,51 @@
+  0%|          | 0/1000 [00:00<?, ?it/s][transformers] `use_return_dict` is deprecated! Use `return_dict` instead!
+[INFO] Causal mask (float with -inf) applied to all attention layers.
+!!! VARIANT glu-waleed10-21L FAILED with: CUDA out of memory. Tried to allocate 3.99 GiB. GPU 0 has a total capacity of 79.32 GiB of which 3.05 GiB is free. Process 936813 has 43.96 GiB memory in use. Including non-PyTorch memory, this process has 32.30 GiB memory in use. Of the allocated memory 31.63 GiB is allocated by PyTorch, and 11.41 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
+Traceback (most recent call last):
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py", line 171, in main
+    trainer.train(resume_from_checkpoint=checkpoint_path)
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/lib/python3.11/site-packages/transformers/trainer.py", line 1449, in train
+    return inner_training_loop(
+           ^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/lib/python3.11/site-packages/transformers/trainer.py", line 1531, in _inner_training_loop
+    self._run_epoch(
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/lib/python3.11/site-packages/transformers/trainer.py", line 1759, in _run_epoch
+    tr_loss_step = self.training_step(model, inputs, num_items_in_batch)
+                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/lib/python3.11/site-packages/transformers/trainer.py", line 1931, in training_step
+    loss = self.compute_loss(model, inputs, num_items_in_batch=num_items_in_batch)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/lib/python3.11/site-packages/transformers/trainer.py", line 2005, in compute_loss
+    outputs = model(**inputs)
+              ^^^^^^^^^^^^^^^
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/lib/python3.11/site-packages/accelerate/utils/operations.py", line 943, in forward
+    return model_forward(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/lib/python3.11/site-packages/accelerate/utils/operations.py", line 931, in __call__
+    return convert_to_fp32(self.model_forward(*args, **kwargs))
+                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/lib/python3.11/site-packages/torch/amp/autocast_mode.py", line 44, in decorate_autocast
+    return func(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/exp.py", line 320, in forward
+    loss = loss_fct(shift_logits.view(-1, self.config.vocab_size), shift_labels.view(-1))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/lib/python3.11/site-packages/torch/nn/modules/loss.py", line 1295, in forward
+    return F.cross_entropy(
+           ^^^^^^^^^^^^^^^^
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/lib/python3.11/site-packages/torch/nn/functional.py", line 3494, in cross_entropy
+    return torch._C._nn.cross_entropy_loss(
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 3.99 GiB. GPU 0 has a total capacity of 79.32 GiB of which 3.05 GiB is free. Process 936813 has 43.96 GiB memory in use. Including non-PyTorch memory, this process has 32.30 GiB memory in use. Of the allocated memory 31.63 GiB is allocated by PyTorch, and 11.41 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)

zain/Activation/wandb/run-20260814_212814-vtp6pf3m/files/requirements.txt

@@ -0,0 +1,149 @@
+asttokens==3.0.1
+comm==0.2.3
+debugpy==1.8.21
+decorator==5.3.1
+executing==2.2.1
+nest-asyncio==1.6.0
+parso==0.8.7
+platformdirs==4.11.0
+psutil==7.2.2
+ptyprocess==0.7.0
+pure_eval==0.2.3
+Pygments==2.20.0
+pyzmq==27.1.0
+setuptools==83.0.0
+six==1.17.0
+tornado==6.5.7
+traitlets==5.15.0
+fsspec==2026.4.0
+wcwidth==0.8.2
+ipython_pygments_lexers==1.1.1
+jedi==0.20.0
+jupyter_core==5.9.1
+matplotlib-inline==0.2.2
+pexpect==4.9.0
+prompt_toolkit==3.0.53
+python-dateutil==2.9.0.post0
+stack_data==0.6.3
+wheel==0.47.0
+jupyter_client==8.9.1
+pip==26.1.2
+ipython==9.15.0
+ipykernel==7.2.0
+threadpoolctl==3.6.0
+pyparsing==3.3.2
+typing_extensions==4.15.0
+Jinja2==3.1.6
+narwhals==2.24.0
+kiwisolver==1.5.0
+joblib==1.5.3
+fonttools==4.63.0
+cycler==0.12.1
+scipy==1.17.1
+pandas==3.0.5
+contourpy==1.3.3
+scikit-learn==1.9.0
+matplotlib==3.11.1
+urllib3==2.7.0
+tqdm==4.70.0
+idna==3.18
+charset-normalizer==3.4.9
+certifi==2026.7.22
+requests==2.34.2
+seaborn==0.13.2
+uv==0.12.0
+shellingham==1.5.4
+mpmath==1.3.0
+attrs==26.1.0
+hf-xet==1.5.2
+nvidia-nccl-cu12==2.21.5
+MarkupSafe==3.0.3
+regex==2026.7.19
+importlib_metadata==9.0.0
+httpcore==1.0.9
+annotated-doc==0.0.5
+multidict==6.7.1
+aiohttp==3.14.3
+aiosignal==1.4.0
+xxhash==3.8.1
+aiohappyeyeballs==2.7.1
+mdurl==0.1.2
+cuda-toolkit==13.0.3.0
+networkx==3.6.1
+PyYAML==6.0.3
+nvidia-cufile==1.15.1.6
+typer==0.27.0
+torchaudio==2.6.0+cu124
+rich==15.0.0
+nvidia-cufft-cu12==11.2.1.3
+h11==0.16.0
+dill==0.4.1
+cuda-pathfinder==1.6.0
+filelock==3.29.0
+nvidia-nvtx-cu12==12.4.127
+httpx==0.28.1
+anyio==4.14.2
+numpy==2.4.4
+yarl==1.24.5
+click==8.4.2
+triton==3.2.0
+frozenlist==1.8.0
+zipp==4.1.0
+propcache==0.5.2
+tokenizers==0.22.2
+markdown-it-py==4.2.0
+nvidia-cuda-runtime==13.0.96
+cuda-bindings==13.3.1
+nvidia-cuda-cupti==13.0.85
+torch==2.6.0+cu124
+multiprocess==0.70.19
+pillow==12.2.0
+transformers==5.16.0.dev0
+wandb==0.28.1
+nvidia-curand==10.4.0.35
+sympy==1.13.1
+nvidia-cusparse==12.6.3.3
+nvidia-cuda-nvrtc==13.0.88
+typing-inspection==0.4.2
+nvidia-cusolver==12.0.4.66
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.20.0.48
+nvidia-cublas==13.1.1.3
+pyarrow==25.0.0
+evaluate==0.4.6
+diffusers==0.39.0
+pydantic==2.13.4
+annotated-types==0.8.0
+protobuf==7.35.1
+sentry-sdk==2.66.1
+einops==0.8.2
+packaging==26.2
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-curand-cu12==10.3.5.147
+nvidia-cusparselt-cu12==0.6.2
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-cuda-runtime-cu12==12.4.127
+torchvision==0.21.0+cu124
+nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cusolver-cu12==11.6.1.9
+nvidia-cublas-cu12==12.4.5.8
+nvidia-cudnn-cu12==9.1.0.70
+huggingface_hub==1.26.0
+datasets==5.0.1
+safetensors==0.8.0
+accelerate==1.14.0
+pydantic_core==2.46.4
+ninja==1.13.0
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.7.1
+jaraco.text==4.0.0
+jaraco.context==6.1.0
+jaraco.functools==4.4.0
+more-itertools==10.8.0
+packaging==26.0
+platformdirs==4.4.0
+tomli==2.4.0
+wheel==0.46.3
+zipp==3.23.0

zain/Activation/wandb/run-20260814_212814-vtp6pf3m/files/wandb-metadata.json

@@ -0,0 +1,101 @@
+{
+  "os":  "Linux-5.15.0-126-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.11.15",
+  "startedAt":  "2026-08-14T21:28:14.182237Z",
+  "args":  [
+    "--config",
+    "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/configs/baseline.yaml",
+    "--variants",
+    "glu-waleed10",
+    "glu-silu-waleed10",
+    "mlp-waleed10",
+    "mlp-silu-waleed10",
+    "glu-waleed",
+    "glu-situglu_low",
+    "glu-waleedglu_low"
+  ],
+  "program":  "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py",
+  "codePath":  "sweep.py",
+  "codePathLocal":  "sweep.py",
+  "git":  {
+    "remote":  "https://github.com/w-ahmad1a10/Activation.git",
+    "commit":  "3fb47c6943d6927eafc39bd399a3d3a520bae74a"
+  },
+  "email":  "deepnevro@gmail.com",
+  "root":  "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation",
+  "host":  "deeplens-k3s-node1",
+  "executable":  "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/bin/python",
+  "cpu_count":  112,
+  "cpu_count_logical":  224,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1560765693952",
+      "used":  "706489905152"
+    }
+  },
+  "memory":  {
+    "total":  "2164089937920"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-39c684a5-fde6-83d7-1663-0859795881ae"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-68012e5a-38b6-b643-0ca6-62fb66720bf3"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-132944c4-b689-2b5f-89a4-d730401677ab"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-2df386cc-6d26-d0e2-7a2d-a057b0d95864"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bfa16575-1d94-1aa2-4537-2c93433f42ef"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bc6c3e3c-9b90-09ca-c034-774961847c54"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-00a441e1-7c95-e7d6-4c35-43d6b291aea9"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-1c4d29a2-4647-6fce-d8fc-0c5ecfbbd6ea"
+    }
+  ],
+  "cudaVersion":  "12.4",
+  "writerId":  "yu21vcw1fqgtt12osjsu76tw34s21eqd"
+}

zain/Activation/wandb/run-20260814_212814-vtp6pf3m/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":1},"_runtime":1}

zain/Activation/wandb/run-20260814_212814-vtp6pf3m/logs/debug-core.log

@@ -0,0 +1,32 @@
+{"time":"2026-08-14T21:28:13.875214015Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp61t5kns7/port-1193850.txt","pid":1193850,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-08-14T21:28:13.875684603Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1193850}
+{"time":"2026-08-14T21:28:13.875676582Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1193850-1195287-1939645545/socket","Net":"unix"}}
+{"time":"2026-08-14T21:28:14.052767875Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-08-14T21:28:14.184824068Z","level":"INFO","msg":"handleInformInit: received","streamId":"vtp6pf3m","id":"1(@)"}
+{"time":"2026-08-14T21:28:14.445813383Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"vtp6pf3m","id":"1(@)"}
+{"time":"2026-08-14T21:28:16.535398722Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"ppjgf4bx6kkl"}
+{"time":"2026-08-14T21:28:17.132108791Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"ppjgf4bx6kkl"}
+{"time":"2026-08-14T21:28:17.133244824Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"vtp6pf3m","id":"1(@)"}
+{"time":"2026-08-14T21:28:17.13384059Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"vtp6pf3m","id":"1(@)"}
+{"time":"2026-08-14T21:28:18.176700551Z","level":"INFO","msg":"handleInformInit: received","streamId":"fj94kr9p","id":"1(@)"}
+{"time":"2026-08-14T21:28:18.417597198Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"fj94kr9p","id":"1(@)"}
+{"time":"2026-08-14T21:28:19.674866991Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"e4d4etq11mlq"}
+{"time":"2026-08-14T21:28:20.222556488Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"e4d4etq11mlq"}
+{"time":"2026-08-14T21:28:20.223783577Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"fj94kr9p","id":"1(@)"}
+{"time":"2026-08-14T21:28:20.224344691Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"fj94kr9p","id":"1(@)"}
+{"time":"2026-08-14T21:28:20.755487912Z","level":"INFO","msg":"handleInformInit: received","streamId":"ucwxn3gs","id":"1(@)"}
+{"time":"2026-08-14T21:28:20.996759448Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"ucwxn3gs","id":"1(@)"}
+{"time":"2026-08-14T21:28:26.491790165Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"4jw8kj1lwl6y"}
+{"time":"2026-08-14T21:29:01.918410879Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"4jw8kj1lwl6y"}
+{"time":"2026-08-14T21:29:02.578939706Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"4jw8kj1lwl6y"}
+{"time":"2026-08-14T21:29:02.580571999Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"ucwxn3gs","id":"1(@)"}
+{"time":"2026-08-14T21:29:02.581104066Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"ucwxn3gs","id":"1(@)"}
+{"time":"2026-08-14T21:29:02.642980411Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-08-14T21:29:02.643036973Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-08-14T21:29:02.643048401Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-08-14T21:29:02.643056857Z","level":"INFO","msg":"server: is shutting down"}
+{"time":"2026-08-14T21:29:02.643101792Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-08-14T21:29:02.643077696Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-08-14T21:29:02.643124976Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-08-14T21:29:02.643304099Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1193850-1195287-1939645545/socket","Net":"unix"}}
+{"time":"2026-08-14T21:29:02.643377148Z","level":"INFO","msg":"server: all connections closed"}

zain/Activation/wandb/run-20260814_212814-vtp6pf3m/logs/debug-internal.log

@@ -0,0 +1,17 @@
+{"time":"2026-08-14T21:28:14.184976796Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-08-14T21:28:14.185271443Z","level":"INFO","msg":"stream: starting","core version":"0.28.1"}
+{"time":"2026-08-14T21:28:14.445644603Z","level":"INFO","msg":"stream: created new stream","id":"vtp6pf3m"}
+{"time":"2026-08-14T21:28:14.445718126Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-08-14T21:28:14.445807645Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-08-14T21:28:14.445824533Z","level":"INFO","msg":"writer: started","stream_id":"vtp6pf3m"}
+{"time":"2026-08-14T21:28:14.44583799Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-08-14T21:28:15.020933363Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-08-14T21:28:15.163288394Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:28:16.980913802Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-08-14T21:28:16.981147036Z","level":"INFO","msg":"filestream: sending request","total_files":2,"console_offset":0,"console_lines":51,"uploaded_len":5,"complete":true,"exit_code":0}
+{"time":"2026-08-14T21:28:17.128640703Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:28:17.130769132Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-08-14T21:28:17.133288135Z","level":"INFO","msg":"stream: finishing up"}
+{"time":"2026-08-14T21:28:17.133316341Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-08-14T21:28:17.133386114Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-08-14T21:28:17.133391408Z","level":"INFO","msg":"stream: all finished"}

zain/Activation/wandb/run-20260814_212814-vtp6pf3m/logs/debug.log

@@ -0,0 +1,27 @@
+2026-08-14 21:28:14,183 INFO    MainThread:1193850 [wandb_setup.py:_flush():81] Current SDK version is 0.28.1
+2026-08-14 21:28:14,183 INFO    MainThread:1193850 [wandb_setup.py:_flush():81] Configure stats pid to 1193850
+2026-08-14 21:28:14,183 INFO    MainThread:1193850 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-14 21:28:14,183 INFO    MainThread:1193850 [wandb_init.py:setup_run_log_directory():729] Logging user logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260814_212814-vtp6pf3m/logs/debug.log
+2026-08-14 21:28:14,183 INFO    MainThread:1193850 [wandb_init.py:setup_run_log_directory():730] Logging internal logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260814_212814-vtp6pf3m/logs/debug-internal.log
+2026-08-14 21:28:14,183 INFO    MainThread:1193850 [wandb_init.py:init():772] calling init triggers
+2026-08-14 21:28:14,183 INFO    MainThread:1193850 [wandb_init.py:init():777] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2026-08-14 21:28:14,183 INFO    MainThread:1193850 [wandb_init.py:init():820] starting backend
+2026-08-14 21:28:14,183 INFO    MainThread:1193850 [wandb_init.py:init():835] sending inform_init request
+2026-08-14 21:28:14,446 INFO    MainThread:1193850 [wandb_init.py:init():840] backend started and connected
+2026-08-14 21:28:14,447 INFO    MainThread:1193850 [wandb_init.py:init():910] updated telemetry
+2026-08-14 21:28:14,454 INFO    MainThread:1193850 [wandb_init.py:init():933] communicating run to backend with 90.0 second timeout
+2026-08-14 21:28:14,939 INFO    MainThread:1193850 [wandb_init.py:init():978] starting run threads in backend
+2026-08-14 21:28:15,013 INFO    MainThread:1193850 [wandb_run.py:_console_start():2621] atexit reg
+2026-08-14 21:28:15,013 INFO    MainThread:1193850 [wandb_run.py:_redirect():2471] redirect: wrap_raw
+2026-08-14 21:28:15,013 INFO    MainThread:1193850 [wandb_run.py:_redirect():2540] Wrapping output streams.
+2026-08-14 21:28:15,013 INFO    MainThread:1193850 [wandb_run.py:_redirect():2563] Redirects installed.
+2026-08-14 21:28:15,016 INFO    MainThread:1193850 [wandb_init.py:init():1016] run started, returning control to user process
+2026-08-14 21:28:15,017 INFO    MainThread:1193850 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 21, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'glu', 'activation': 'waleed10', 'waleed_beta': 10.0, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/glu-waleed10-21L_run', 'per_device_train_batch_size': 512, 'num_train_epochs': 1, 'max_steps': 1000, 'learning_rate': 0.001, 'lr_scheduler_type': 'constant', 'lr_scheduler_kwargs': None, 'warmup_steps': 0, 'optim': 'adamw_torch', 'optim_args': None, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 1.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-glu-waleed10-21L-4.0M-20260814-212813', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 60000, 'eval_delay': 0, 'per_device_eval_batch_size': 512, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 100, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/A-glu-waleed10-21L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
+2026-08-14 21:28:15,019 INFO    MainThread:1193850 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 3970432 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x1462d29d2090>>
+2026-08-14 21:28:15,019 INFO    MainThread:1193850 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 3970432 None
+2026-08-14 21:28:16,533 INFO    MainThread:1193850 [wandb_run.py:_finish():2383] finishing run deepnevro-deepnevro/research1/vtp6pf3m
+2026-08-14 21:28:16,534 INFO    MainThread:1193850 [wandb_run.py:_atexit_cleanup():2588] got exitcode: 0
+2026-08-14 21:28:16,535 INFO    MainThread:1193850 [wandb_run.py:_restore():2570] restore
+2026-08-14 21:28:16,535 INFO    MainThread:1193850 [wandb_run.py:_restore():2576] restore done
+2026-08-14 21:28:17,132 INFO    MainThread:1193850 [wandb_run.py:_footer_sync_info():3993] logging synced files

zain/Activation/wandb/run-20260814_212818-fj94kr9p/files/config.yaml

@@ -0,0 +1,438 @@
+_name_or_path:
+    value: ""
+_wandb:
+    value:
+        cli_version: 0.28.1
+        e:
+            xio5js2delb7osbi95ndowruwry5o7mi:
+                args:
+                    - --config
+                    - /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/configs/baseline.yaml
+                    - --variants
+                    - glu-waleed10
+                    - glu-silu-waleed10
+                    - mlp-waleed10
+                    - mlp-silu-waleed10
+                    - glu-waleed
+                    - glu-situglu_low
+                    - glu-waleedglu_low
+                codePath: sweep.py
+                codePathLocal: sweep.py
+                cpu_count: 112
+                cpu_count_logical: 224
+                cudaVersion: "12.4"
+                disk:
+                    /:
+                        total: "1560765693952"
+                        used: "706489917440"
+                email: deepnevro@gmail.com
+                executable: /mnt/data/zainulabideen/zain-exp/notebooks/my_env/bin/python
+                git:
+                    commit: 3fb47c6943d6927eafc39bd399a3d3a520bae74a
+                    remote: https://github.com/w-ahmad1a10/Activation.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-39c684a5-fde6-83d7-1663-0859795881ae
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-68012e5a-38b6-b643-0ca6-62fb66720bf3
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-132944c4-b689-2b5f-89a4-d730401677ab
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-2df386cc-6d26-d0e2-7a2d-a057b0d95864
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bfa16575-1d94-1aa2-4537-2c93433f42ef
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bc6c3e3c-9b90-09ca-c034-774961847c54
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-00a441e1-7c95-e7d6-4c35-43d6b291aea9
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-1c4d29a2-4647-6fce-d8fc-0c5ecfbbd6ea
+                host: deeplens-k3s-node1
+                memory:
+                    total: "2164089937920"
+                os: Linux-5.15.0-126-generic-x86_64-with-glibc2.35
+                program: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py
+                python: CPython 3.11.15
+                root: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation
+                startedAt: "2026-08-14T21:28:18.170018Z"
+                writerId: xio5js2delb7osbi95ndowruwry5o7mi
+        m:
+            - "1": train/global_step
+              "6":
+                - 3
+              "7": []
+            - "2": '*'
+              "5": 1
+              "6":
+                - 1
+              "7": []
+        python_version: 3.11.15
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 7
+                - 13
+                - 19
+                - 66
+            "4": 3.11.15
+            "5": 0.28.1
+            "6": 5.16.0.dev0
+            "9":
+                "1": transformers_trainer
+            "12": 0.28.1
+            "13": linux-x86_64
+accelerator_config:
+    value:
+        dispatch_batches: null
+        even_batches: true
+        gradient_accumulation_kwargs: null
+        non_blocking: false
+        split_batches: false
+        use_seedable_sampler: true
+activation:
+    value: silu-waleed10
+adam_beta1:
+    value: 0.9
+adam_beta2:
+    value: 0.999
+adam_epsilon:
+    value: 1e-08
+architectures:
+    value: null
+attention_bias:
+    value: false
+attention_dropout:
+    value: 0
+auto_find_batch_size:
+    value: false
+average_tokens_across_devices:
+    value: true
+batch_eval_metrics:
+    value: false
+bf16:
+    value: true
+bf16_full_eval:
+    value: false
+bos_token_id:
+    value: 1
+chunk_size_feed_forward:
+    value: 0
+data_seed:
+    value: 42
+dataloader_drop_last:
+    value: false
+dataloader_in_order:
+    value: true
+dataloader_multiprocessing_context:
+    value: null
+dataloader_num_workers:
+    value: 0
+dataloader_persistent_workers:
+    value: false
+dataloader_pin_memory:
+    value: true
+dataloader_prefetch_factor:
+    value: null
+ddp_backend:
+    value: null
+ddp_broadcast_buffers:
+    value: null
+ddp_bucket_cap_mb:
+    value: null
+ddp_find_unused_parameters:
+    value: null
+ddp_static_graph:
+    value: null
+ddp_timeout:
+    value: 1800
+debug:
+    value: []
+deepspeed:
+    value: null
+disable_tqdm:
+    value: false
+do_eval:
+    value: true
+do_predict:
+    value: false
+do_train:
+    value: false
+dtype:
+    value: null
+enable_jit_checkpoint:
+    value: false
+eos_token_id:
+    value: 2
+eval_accumulation_steps:
+    value: null
+eval_delay:
+    value: 0
+eval_do_concat_batches:
+    value: true
+eval_on_start:
+    value: false
+eval_steps:
+    value: 60000
+eval_strategy:
+    value: steps
+eval_use_gather_object:
+    value: false
+fp16:
+    value: false
+fp16_full_eval:
+    value: false
+fsdp:
+    value: null
+fsdp_config:
+    value: null
+full_determinism:
+    value: false
+gradient_accumulation_steps:
+    value: 1
+gradient_checkpointing:
+    value: false
+gradient_checkpointing_kwargs:
+    value: null
+greater_is_better:
+    value: null
+head_dim:
+    value: 32
+hidden_act:
+    value: silu
+hidden_size:
+    value: 128
+hub_always_push:
+    value: false
+hub_model_id:
+    value: w-ahmad/A-glu-silu-waleed10-21L
+hub_private_repo:
+    value: null
+hub_revision:
+    value: null
+hub_strategy:
+    value: every_save
+hub_token:
+    value: <HUB_TOKEN>
+id2label:
+    value:
+        "0": LABEL_0
+        "1": LABEL_1
+ignore_data_skip:
+    value: false
+include_for_metrics:
+    value: []
+include_num_input_tokens_seen:
+    value: "no"
+initializer_range:
+    value: 0.02
+intermediate_size:
+    value: 256
+is_encoder_decoder:
+    value: false
+label_names:
+    value: null
+label_smoothing_factor:
+    value: 0
+label2id:
+    value:
+        LABEL_0: 0
+        LABEL_1: 1
+learning_rate:
+    value: 0.001
+length_column_name:
+    value: length
+liger_kernel_config:
+    value: null
+load_best_model_at_end:
+    value: false
+local_rank:
+    value: -1
+log_level:
+    value: passive
+log_level_replica:
+    value: warning
+log_on_each_node:
+    value: true
+logging_first_step:
+    value: false
+logging_nan_inf_filter:
+    value: true
+logging_steps:
+    value: 20
+logging_strategy:
+    value: steps
+lr_scheduler_kwargs:
+    value: null
+lr_scheduler_type:
+    value: constant
+max_grad_norm:
+    value: 1
+max_position_embeddings:
+    value: 512
+max_steps:
+    value: 1000
+metric_for_best_model:
+    value: null
+mlp_bias:
+    value: false
+mlp_type:
+    value: glu
+model/num_parameters:
+    value: 3970432
+model_type:
+    value: tiny_llama
+neftune_noise_alpha:
+    value: null
+num_attention_heads:
+    value: 4
+num_hidden_layers:
+    value: 21
+num_key_value_heads:
+    value: 4
+num_train_epochs:
+    value: 1
+optim:
+    value: adamw_torch
+optim_args:
+    value: null
+optim_target_modules:
+    value: null
+output_attentions:
+    value: false
+output_dir:
+    value: out/glu-silu-waleed10-21L_run
+output_hidden_states:
+    value: false
+pad_token_id:
+    value: 0
+parallelism_config:
+    value: null
+per_device_eval_batch_size:
+    value: 512
+per_device_train_batch_size:
+    value: 512
+prediction_loss_only:
+    value: false
+pretraining_tp:
+    value: 1
+problem_type:
+    value: null
+project:
+    value: huggingface
+push_to_hub:
+    value: false
+remove_unused_columns:
+    value: false
+report_to:
+    value:
+        - wandb
+restore_callback_states_from_checkpoint:
+    value: false
+resume_from_checkpoint:
+    value: null
+return_dict:
+    value: true
+rms_norm_eps:
+    value: 1e-06
+rope_parameters:
+    value:
+        rope_theta: 10000
+        rope_type: default
+run_name:
+    value: LM-glu-silu-waleed10-21L-4.0M-20260814-212817
+save_on_each_node:
+    value: false
+save_only_model:
+    value: false
+save_steps:
+    value: 100
+save_strategy:
+    value: steps
+save_total_limit:
+    value: null
+seed:
+    value: 42
+skip_memory_metrics:
+    value: true
+tf32:
+    value: null
+tie_word_embeddings:
+    value: true
+tokenizer_name:
+    value: w-ahmad/tiny-stories-tokenizer
+torch_compile:
+    value: false
+torch_compile_backend:
+    value: null
+torch_compile_mode:
+    value: null
+torch_empty_cache_steps:
+    value: null
+trackio_bucket_id:
+    value: null
+trackio_space_id:
+    value: null
+trackio_static_space_id:
+    value: null
+train_sampling_strategy:
+    value: random
+transformers_version:
+    value: 5.16.0.dev0
+use_cache:
+    value: false
+use_cpu:
+    value: false
+use_liger_kernel:
+    value: false
+vocab_size:
+    value: 4096
+waleed_beta:
+    value: 10
+warmup_steps:
+    value: 0
+weight_decay:
+    value: 0.01

zain/Activation/wandb/run-20260814_212818-fj94kr9p/files/output.log

@@ -0,0 +1,49 @@
+  0%|          | 0/1000 [00:00<?, ?it/s]Traceback (most recent call last):
+!!! VARIANT glu-silu-waleed10-21L FAILED with: CUDA out of memory. Tried to allocate 3.99 GiB. GPU 0 has a total capacity of 79.32 GiB of which 428.75 MiB is free. Process 936813 has 43.96 GiB memory in use. Including non-PyTorch memory, this process has 34.93 GiB memory in use. Of the allocated memory 34.26 GiB is allocated by PyTorch, and 13.41 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py", line 171, in main
+    trainer.train(resume_from_checkpoint=checkpoint_path)
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/lib/python3.11/site-packages/transformers/trainer.py", line 1449, in train
+    return inner_training_loop(
+           ^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/lib/python3.11/site-packages/transformers/trainer.py", line 1531, in _inner_training_loop
+    self._run_epoch(
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/lib/python3.11/site-packages/transformers/trainer.py", line 1759, in _run_epoch
+    tr_loss_step = self.training_step(model, inputs, num_items_in_batch)
+                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/lib/python3.11/site-packages/transformers/trainer.py", line 1931, in training_step
+    loss = self.compute_loss(model, inputs, num_items_in_batch=num_items_in_batch)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/lib/python3.11/site-packages/transformers/trainer.py", line 2005, in compute_loss
+    outputs = model(**inputs)
+              ^^^^^^^^^^^^^^^
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/lib/python3.11/site-packages/accelerate/utils/operations.py", line 943, in forward
+    return model_forward(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/lib/python3.11/site-packages/accelerate/utils/operations.py", line 931, in __call__
+    return convert_to_fp32(self.model_forward(*args, **kwargs))
+                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/lib/python3.11/site-packages/torch/amp/autocast_mode.py", line 44, in decorate_autocast
+    return func(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/exp.py", line 320, in forward
+    loss = loss_fct(shift_logits.view(-1, self.config.vocab_size), shift_labels.view(-1))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/lib/python3.11/site-packages/torch/nn/modules/loss.py", line 1295, in forward
+    return F.cross_entropy(
+           ^^^^^^^^^^^^^^^^
+  File "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/lib/python3.11/site-packages/torch/nn/functional.py", line 3494, in cross_entropy
+    return torch._C._nn.cross_entropy_loss(
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 3.99 GiB. GPU 0 has a total capacity of 79.32 GiB of which 428.75 MiB is free. Process 936813 has 43.96 GiB memory in use. Including non-PyTorch memory, this process has 34.93 GiB memory in use. Of the allocated memory 34.26 GiB is allocated by PyTorch, and 13.41 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)

zain/Activation/wandb/run-20260814_212818-fj94kr9p/files/requirements.txt

@@ -0,0 +1,149 @@
+asttokens==3.0.1
+comm==0.2.3
+debugpy==1.8.21
+decorator==5.3.1
+executing==2.2.1
+nest-asyncio==1.6.0
+parso==0.8.7
+platformdirs==4.11.0
+psutil==7.2.2
+ptyprocess==0.7.0
+pure_eval==0.2.3
+Pygments==2.20.0
+pyzmq==27.1.0
+setuptools==83.0.0
+six==1.17.0
+tornado==6.5.7
+traitlets==5.15.0
+fsspec==2026.4.0
+wcwidth==0.8.2
+ipython_pygments_lexers==1.1.1
+jedi==0.20.0
+jupyter_core==5.9.1
+matplotlib-inline==0.2.2
+pexpect==4.9.0
+prompt_toolkit==3.0.53
+python-dateutil==2.9.0.post0
+stack_data==0.6.3
+wheel==0.47.0
+jupyter_client==8.9.1
+pip==26.1.2
+ipython==9.15.0
+ipykernel==7.2.0
+threadpoolctl==3.6.0
+pyparsing==3.3.2
+typing_extensions==4.15.0
+Jinja2==3.1.6
+narwhals==2.24.0
+kiwisolver==1.5.0
+joblib==1.5.3
+fonttools==4.63.0
+cycler==0.12.1
+scipy==1.17.1
+pandas==3.0.5
+contourpy==1.3.3
+scikit-learn==1.9.0
+matplotlib==3.11.1
+urllib3==2.7.0
+tqdm==4.70.0
+idna==3.18
+charset-normalizer==3.4.9
+certifi==2026.7.22
+requests==2.34.2
+seaborn==0.13.2
+uv==0.12.0
+shellingham==1.5.4
+mpmath==1.3.0
+attrs==26.1.0
+hf-xet==1.5.2
+nvidia-nccl-cu12==2.21.5
+MarkupSafe==3.0.3
+regex==2026.7.19
+importlib_metadata==9.0.0
+httpcore==1.0.9
+annotated-doc==0.0.5
+multidict==6.7.1
+aiohttp==3.14.3
+aiosignal==1.4.0
+xxhash==3.8.1
+aiohappyeyeballs==2.7.1
+mdurl==0.1.2
+cuda-toolkit==13.0.3.0
+networkx==3.6.1
+PyYAML==6.0.3
+nvidia-cufile==1.15.1.6
+typer==0.27.0
+torchaudio==2.6.0+cu124
+rich==15.0.0
+nvidia-cufft-cu12==11.2.1.3
+h11==0.16.0
+dill==0.4.1
+cuda-pathfinder==1.6.0
+filelock==3.29.0
+nvidia-nvtx-cu12==12.4.127
+httpx==0.28.1
+anyio==4.14.2
+numpy==2.4.4
+yarl==1.24.5
+click==8.4.2
+triton==3.2.0
+frozenlist==1.8.0
+zipp==4.1.0
+propcache==0.5.2
+tokenizers==0.22.2
+markdown-it-py==4.2.0
+nvidia-cuda-runtime==13.0.96
+cuda-bindings==13.3.1
+nvidia-cuda-cupti==13.0.85
+torch==2.6.0+cu124
+multiprocess==0.70.19
+pillow==12.2.0
+transformers==5.16.0.dev0
+wandb==0.28.1
+nvidia-curand==10.4.0.35
+sympy==1.13.1
+nvidia-cusparse==12.6.3.3
+nvidia-cuda-nvrtc==13.0.88
+typing-inspection==0.4.2
+nvidia-cusolver==12.0.4.66
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.20.0.48
+nvidia-cublas==13.1.1.3
+pyarrow==25.0.0
+evaluate==0.4.6
+diffusers==0.39.0
+pydantic==2.13.4
+annotated-types==0.8.0
+protobuf==7.35.1
+sentry-sdk==2.66.1
+einops==0.8.2
+packaging==26.2
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-curand-cu12==10.3.5.147
+nvidia-cusparselt-cu12==0.6.2
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-cuda-runtime-cu12==12.4.127
+torchvision==0.21.0+cu124
+nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cusolver-cu12==11.6.1.9
+nvidia-cublas-cu12==12.4.5.8
+nvidia-cudnn-cu12==9.1.0.70
+huggingface_hub==1.26.0
+datasets==5.0.1
+safetensors==0.8.0
+accelerate==1.14.0
+pydantic_core==2.46.4
+ninja==1.13.0
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.7.1
+jaraco.text==4.0.0
+jaraco.context==6.1.0
+jaraco.functools==4.4.0
+more-itertools==10.8.0
+packaging==26.0
+platformdirs==4.4.0
+tomli==2.4.0
+wheel==0.46.3
+zipp==3.23.0

zain/Activation/wandb/run-20260814_212818-fj94kr9p/files/wandb-metadata.json

@@ -0,0 +1,101 @@
+{
+  "os":  "Linux-5.15.0-126-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.11.15",
+  "startedAt":  "2026-08-14T21:28:18.170018Z",
+  "args":  [
+    "--config",
+    "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/configs/baseline.yaml",
+    "--variants",
+    "glu-waleed10",
+    "glu-silu-waleed10",
+    "mlp-waleed10",
+    "mlp-silu-waleed10",
+    "glu-waleed",
+    "glu-situglu_low",
+    "glu-waleedglu_low"
+  ],
+  "program":  "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py",
+  "codePath":  "sweep.py",
+  "codePathLocal":  "sweep.py",
+  "git":  {
+    "remote":  "https://github.com/w-ahmad1a10/Activation.git",
+    "commit":  "3fb47c6943d6927eafc39bd399a3d3a520bae74a"
+  },
+  "email":  "deepnevro@gmail.com",
+  "root":  "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation",
+  "host":  "deeplens-k3s-node1",
+  "executable":  "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/bin/python",
+  "cpu_count":  112,
+  "cpu_count_logical":  224,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1560765693952",
+      "used":  "706489917440"
+    }
+  },
+  "memory":  {
+    "total":  "2164089937920"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-39c684a5-fde6-83d7-1663-0859795881ae"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-68012e5a-38b6-b643-0ca6-62fb66720bf3"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-132944c4-b689-2b5f-89a4-d730401677ab"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-2df386cc-6d26-d0e2-7a2d-a057b0d95864"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bfa16575-1d94-1aa2-4537-2c93433f42ef"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bc6c3e3c-9b90-09ca-c034-774961847c54"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-00a441e1-7c95-e7d6-4c35-43d6b291aea9"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-1c4d29a2-4647-6fce-d8fc-0c5ecfbbd6ea"
+    }
+  ],
+  "cudaVersion":  "12.4",
+  "writerId":  "xio5js2delb7osbi95ndowruwry5o7mi"
+}

zain/Activation/wandb/run-20260814_212818-fj94kr9p/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_runtime":1,"_wandb":{"runtime":1}}

zain/Activation/wandb/run-20260814_212818-fj94kr9p/logs/debug-core.log

@@ -0,0 +1,32 @@
+{"time":"2026-08-14T21:28:13.875214015Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp61t5kns7/port-1193850.txt","pid":1193850,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-08-14T21:28:13.875684603Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1193850}
+{"time":"2026-08-14T21:28:13.875676582Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1193850-1195287-1939645545/socket","Net":"unix"}}
+{"time":"2026-08-14T21:28:14.052767875Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-08-14T21:28:14.184824068Z","level":"INFO","msg":"handleInformInit: received","streamId":"vtp6pf3m","id":"1(@)"}
+{"time":"2026-08-14T21:28:14.445813383Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"vtp6pf3m","id":"1(@)"}
+{"time":"2026-08-14T21:28:16.535398722Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"ppjgf4bx6kkl"}
+{"time":"2026-08-14T21:28:17.132108791Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"ppjgf4bx6kkl"}
+{"time":"2026-08-14T21:28:17.133244824Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"vtp6pf3m","id":"1(@)"}
+{"time":"2026-08-14T21:28:17.13384059Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"vtp6pf3m","id":"1(@)"}
+{"time":"2026-08-14T21:28:18.176700551Z","level":"INFO","msg":"handleInformInit: received","streamId":"fj94kr9p","id":"1(@)"}
+{"time":"2026-08-14T21:28:18.417597198Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"fj94kr9p","id":"1(@)"}
+{"time":"2026-08-14T21:28:19.674866991Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"e4d4etq11mlq"}
+{"time":"2026-08-14T21:28:20.222556488Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"e4d4etq11mlq"}
+{"time":"2026-08-14T21:28:20.223783577Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"fj94kr9p","id":"1(@)"}
+{"time":"2026-08-14T21:28:20.224344691Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"fj94kr9p","id":"1(@)"}
+{"time":"2026-08-14T21:28:20.755487912Z","level":"INFO","msg":"handleInformInit: received","streamId":"ucwxn3gs","id":"1(@)"}
+{"time":"2026-08-14T21:28:20.996759448Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"ucwxn3gs","id":"1(@)"}
+{"time":"2026-08-14T21:28:26.491790165Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"4jw8kj1lwl6y"}
+{"time":"2026-08-14T21:29:01.918410879Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"4jw8kj1lwl6y"}
+{"time":"2026-08-14T21:29:02.578939706Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"4jw8kj1lwl6y"}
+{"time":"2026-08-14T21:29:02.580571999Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"ucwxn3gs","id":"1(@)"}
+{"time":"2026-08-14T21:29:02.581104066Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"ucwxn3gs","id":"1(@)"}
+{"time":"2026-08-14T21:29:02.642980411Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-08-14T21:29:02.643036973Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-08-14T21:29:02.643048401Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-08-14T21:29:02.643056857Z","level":"INFO","msg":"server: is shutting down"}
+{"time":"2026-08-14T21:29:02.643101792Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-08-14T21:29:02.643077696Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-08-14T21:29:02.643124976Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-08-14T21:29:02.643304099Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1193850-1195287-1939645545/socket","Net":"unix"}}
+{"time":"2026-08-14T21:29:02.643377148Z","level":"INFO","msg":"server: all connections closed"}

zain/Activation/wandb/run-20260814_212818-fj94kr9p/logs/debug-internal.log

@@ -0,0 +1,17 @@
+{"time":"2026-08-14T21:28:18.176855561Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-08-14T21:28:18.176991497Z","level":"INFO","msg":"stream: starting","core version":"0.28.1"}
+{"time":"2026-08-14T21:28:18.417418398Z","level":"INFO","msg":"stream: created new stream","id":"fj94kr9p"}
+{"time":"2026-08-14T21:28:18.417508597Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-08-14T21:28:18.417590145Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-08-14T21:28:18.417602631Z","level":"INFO","msg":"writer: started","stream_id":"fj94kr9p"}
+{"time":"2026-08-14T21:28:18.417629181Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-08-14T21:28:18.755471757Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-08-14T21:28:18.858554768Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:28:20.070727266Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-08-14T21:28:20.070978952Z","level":"INFO","msg":"filestream: sending request","total_files":2,"console_offset":0,"console_lines":49,"uploaded_len":5,"complete":true,"exit_code":0}
+{"time":"2026-08-14T21:28:20.219838371Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:28:20.221222209Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-08-14T21:28:20.223817082Z","level":"INFO","msg":"stream: finishing up"}
+{"time":"2026-08-14T21:28:20.223834217Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-08-14T21:28:20.22390683Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-08-14T21:28:20.223912113Z","level":"INFO","msg":"stream: all finished"}

zain/Activation/wandb/run-20260814_212818-fj94kr9p/logs/debug.log

@@ -0,0 +1,24 @@
+2026-08-14 21:28:18,171 INFO    MainThread:1193850 [wandb_init.py:setup_run_log_directory():729] Logging user logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260814_212818-fj94kr9p/logs/debug.log
+2026-08-14 21:28:18,175 INFO    MainThread:1193850 [wandb_init.py:setup_run_log_directory():730] Logging internal logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260814_212818-fj94kr9p/logs/debug-internal.log
+2026-08-14 21:28:18,175 INFO    MainThread:1193850 [wandb_init.py:init():772] calling init triggers
+2026-08-14 21:28:18,175 INFO    MainThread:1193850 [wandb_init.py:init():777] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2026-08-14 21:28:18,175 INFO    MainThread:1193850 [wandb_init.py:init():820] starting backend
+2026-08-14 21:28:18,175 INFO    MainThread:1193850 [wandb_init.py:init():835] sending inform_init request
+2026-08-14 21:28:18,418 INFO    MainThread:1193850 [wandb_init.py:init():840] backend started and connected
+2026-08-14 21:28:18,419 INFO    MainThread:1193850 [wandb_init.py:init():910] updated telemetry
+2026-08-14 21:28:18,425 INFO    MainThread:1193850 [wandb_init.py:init():933] communicating run to backend with 90.0 second timeout
+2026-08-14 21:28:18,670 INFO    MainThread:1193850 [wandb_init.py:init():978] starting run threads in backend
+2026-08-14 21:28:18,749 INFO    MainThread:1193850 [wandb_run.py:_console_start():2621] atexit reg
+2026-08-14 21:28:18,750 INFO    MainThread:1193850 [wandb_run.py:_redirect():2471] redirect: wrap_raw
+2026-08-14 21:28:18,750 INFO    MainThread:1193850 [wandb_run.py:_redirect():2540] Wrapping output streams.
+2026-08-14 21:28:18,750 INFO    MainThread:1193850 [wandb_run.py:_redirect():2563] Redirects installed.
+2026-08-14 21:28:18,751 INFO    MainThread:1193850 [wandb_init.py:init():1016] run started, returning control to user process
+2026-08-14 21:28:18,752 INFO    MainThread:1193850 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 21, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'glu', 'activation': 'silu-waleed10', 'waleed_beta': 10.0, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/glu-silu-waleed10-21L_run', 'per_device_train_batch_size': 512, 'num_train_epochs': 1, 'max_steps': 1000, 'learning_rate': 0.001, 'lr_scheduler_type': 'constant', 'lr_scheduler_kwargs': None, 'warmup_steps': 0, 'optim': 'adamw_torch', 'optim_args': None, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 1.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-glu-silu-waleed10-21L-4.0M-20260814-212817', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 60000, 'eval_delay': 0, 'per_device_eval_batch_size': 512, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 100, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/A-glu-silu-waleed10-21L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
+2026-08-14 21:28:18,754 INFO    MainThread:1193850 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 3970432 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x1462d29e8f90>>
+2026-08-14 21:28:18,754 INFO    MainThread:1193850 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 3970432 None
+2026-08-14 21:28:19,673 INFO    MainThread:1193850 [wandb_run.py:_finish():2383] finishing run deepnevro-deepnevro/research1/fj94kr9p
+2026-08-14 21:28:19,674 INFO    MainThread:1193850 [wandb_run.py:_atexit_cleanup():2588] got exitcode: 0
+2026-08-14 21:28:19,674 INFO    MainThread:1193850 [wandb_run.py:_restore():2570] restore
+2026-08-14 21:28:19,674 INFO    MainThread:1193850 [wandb_run.py:_restore():2576] restore done
+2026-08-14 21:28:20,223 INFO    MainThread:1193850 [wandb_run.py:_footer_sync_info():3993] logging synced files

zain/Activation/wandb/run-20260814_212820-ucwxn3gs/files/config.yaml

@@ -0,0 +1,438 @@
+_name_or_path:
+    value: ""
+_wandb:
+    value:
+        cli_version: 0.28.1
+        e:
+            2drnyjxeagicq7ia3qgncq8z361n6gif:
+                args:
+                    - --config
+                    - /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/configs/baseline.yaml
+                    - --variants
+                    - glu-waleed10
+                    - glu-silu-waleed10
+                    - mlp-waleed10
+                    - mlp-silu-waleed10
+                    - glu-waleed
+                    - glu-situglu_low
+                    - glu-waleedglu_low
+                codePath: sweep.py
+                codePathLocal: sweep.py
+                cpu_count: 112
+                cpu_count_logical: 224
+                cudaVersion: "12.4"
+                disk:
+                    /:
+                        total: "1560765693952"
+                        used: "706489917440"
+                email: deepnevro@gmail.com
+                executable: /mnt/data/zainulabideen/zain-exp/notebooks/my_env/bin/python
+                git:
+                    commit: 3fb47c6943d6927eafc39bd399a3d3a520bae74a
+                    remote: https://github.com/w-ahmad1a10/Activation.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-39c684a5-fde6-83d7-1663-0859795881ae
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-68012e5a-38b6-b643-0ca6-62fb66720bf3
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-132944c4-b689-2b5f-89a4-d730401677ab
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-2df386cc-6d26-d0e2-7a2d-a057b0d95864
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bfa16575-1d94-1aa2-4537-2c93433f42ef
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bc6c3e3c-9b90-09ca-c034-774961847c54
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-00a441e1-7c95-e7d6-4c35-43d6b291aea9
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-1c4d29a2-4647-6fce-d8fc-0c5ecfbbd6ea
+                host: deeplens-k3s-node1
+                memory:
+                    total: "2164089937920"
+                os: Linux-5.15.0-126-generic-x86_64-with-glibc2.35
+                program: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py
+                python: CPython 3.11.15
+                root: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation
+                startedAt: "2026-08-14T21:28:20.753237Z"
+                writerId: 2drnyjxeagicq7ia3qgncq8z361n6gif
+        m:
+            - "1": train/global_step
+              "6":
+                - 3
+              "7": []
+            - "2": '*'
+              "5": 1
+              "6":
+                - 1
+              "7": []
+        python_version: 3.11.15
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 7
+                - 13
+                - 19
+                - 66
+            "4": 3.11.15
+            "5": 0.28.1
+            "6": 5.16.0.dev0
+            "9":
+                "1": transformers_trainer
+            "12": 0.28.1
+            "13": linux-x86_64
+accelerator_config:
+    value:
+        dispatch_batches: null
+        even_batches: true
+        gradient_accumulation_kwargs: null
+        non_blocking: false
+        split_batches: false
+        use_seedable_sampler: true
+activation:
+    value: waleed10
+adam_beta1:
+    value: 0.9
+adam_beta2:
+    value: 0.999
+adam_epsilon:
+    value: 1e-08
+architectures:
+    value: null
+attention_bias:
+    value: false
+attention_dropout:
+    value: 0
+auto_find_batch_size:
+    value: false
+average_tokens_across_devices:
+    value: true
+batch_eval_metrics:
+    value: false
+bf16:
+    value: true
+bf16_full_eval:
+    value: false
+bos_token_id:
+    value: 1
+chunk_size_feed_forward:
+    value: 0
+data_seed:
+    value: 42
+dataloader_drop_last:
+    value: false
+dataloader_in_order:
+    value: true
+dataloader_multiprocessing_context:
+    value: null
+dataloader_num_workers:
+    value: 0
+dataloader_persistent_workers:
+    value: false
+dataloader_pin_memory:
+    value: true
+dataloader_prefetch_factor:
+    value: null
+ddp_backend:
+    value: null
+ddp_broadcast_buffers:
+    value: null
+ddp_bucket_cap_mb:
+    value: null
+ddp_find_unused_parameters:
+    value: null
+ddp_static_graph:
+    value: null
+ddp_timeout:
+    value: 1800
+debug:
+    value: []
+deepspeed:
+    value: null
+disable_tqdm:
+    value: false
+do_eval:
+    value: true
+do_predict:
+    value: false
+do_train:
+    value: false
+dtype:
+    value: null
+enable_jit_checkpoint:
+    value: false
+eos_token_id:
+    value: 2
+eval_accumulation_steps:
+    value: null
+eval_delay:
+    value: 0
+eval_do_concat_batches:
+    value: true
+eval_on_start:
+    value: false
+eval_steps:
+    value: 60000
+eval_strategy:
+    value: steps
+eval_use_gather_object:
+    value: false
+fp16:
+    value: false
+fp16_full_eval:
+    value: false
+fsdp:
+    value: null
+fsdp_config:
+    value: null
+full_determinism:
+    value: false
+gradient_accumulation_steps:
+    value: 1
+gradient_checkpointing:
+    value: false
+gradient_checkpointing_kwargs:
+    value: null
+greater_is_better:
+    value: null
+head_dim:
+    value: 32
+hidden_act:
+    value: silu
+hidden_size:
+    value: 128
+hub_always_push:
+    value: false
+hub_model_id:
+    value: w-ahmad/A-mlp-waleed10-21L
+hub_private_repo:
+    value: null
+hub_revision:
+    value: null
+hub_strategy:
+    value: every_save
+hub_token:
+    value: <HUB_TOKEN>
+id2label:
+    value:
+        "0": LABEL_0
+        "1": LABEL_1
+ignore_data_skip:
+    value: false
+include_for_metrics:
+    value: []
+include_num_input_tokens_seen:
+    value: "no"
+initializer_range:
+    value: 0.02
+intermediate_size:
+    value: 256
+is_encoder_decoder:
+    value: false
+label_names:
+    value: null
+label_smoothing_factor:
+    value: 0
+label2id:
+    value:
+        LABEL_0: 0
+        LABEL_1: 1
+learning_rate:
+    value: 0.001
+length_column_name:
+    value: length
+liger_kernel_config:
+    value: null
+load_best_model_at_end:
+    value: false
+local_rank:
+    value: -1
+log_level:
+    value: passive
+log_level_replica:
+    value: warning
+log_on_each_node:
+    value: true
+logging_first_step:
+    value: false
+logging_nan_inf_filter:
+    value: true
+logging_steps:
+    value: 20
+logging_strategy:
+    value: steps
+lr_scheduler_kwargs:
+    value: null
+lr_scheduler_type:
+    value: constant
+max_grad_norm:
+    value: 1
+max_position_embeddings:
+    value: 512
+max_steps:
+    value: 1000
+metric_for_best_model:
+    value: null
+mlp_bias:
+    value: false
+mlp_type:
+    value: mlp
+model/num_parameters:
+    value: 3970432
+model_type:
+    value: tiny_llama
+neftune_noise_alpha:
+    value: null
+num_attention_heads:
+    value: 4
+num_hidden_layers:
+    value: 21
+num_key_value_heads:
+    value: 4
+num_train_epochs:
+    value: 1
+optim:
+    value: adamw_torch
+optim_args:
+    value: null
+optim_target_modules:
+    value: null
+output_attentions:
+    value: false
+output_dir:
+    value: out/mlp-waleed10-21L_run
+output_hidden_states:
+    value: false
+pad_token_id:
+    value: 0
+parallelism_config:
+    value: null
+per_device_eval_batch_size:
+    value: 512
+per_device_train_batch_size:
+    value: 512
+prediction_loss_only:
+    value: false
+pretraining_tp:
+    value: 1
+problem_type:
+    value: null
+project:
+    value: huggingface
+push_to_hub:
+    value: false
+remove_unused_columns:
+    value: false
+report_to:
+    value:
+        - wandb
+restore_callback_states_from_checkpoint:
+    value: false
+resume_from_checkpoint:
+    value: null
+return_dict:
+    value: true
+rms_norm_eps:
+    value: 1e-06
+rope_parameters:
+    value:
+        rope_theta: 10000
+        rope_type: default
+run_name:
+    value: LM-mlp-waleed10-21L-4.0M-20260814-212820
+save_on_each_node:
+    value: false
+save_only_model:
+    value: false
+save_steps:
+    value: 100
+save_strategy:
+    value: steps
+save_total_limit:
+    value: null
+seed:
+    value: 42
+skip_memory_metrics:
+    value: true
+tf32:
+    value: null
+tie_word_embeddings:
+    value: true
+tokenizer_name:
+    value: w-ahmad/tiny-stories-tokenizer
+torch_compile:
+    value: false
+torch_compile_backend:
+    value: null
+torch_compile_mode:
+    value: null
+torch_empty_cache_steps:
+    value: null
+trackio_bucket_id:
+    value: null
+trackio_space_id:
+    value: null
+trackio_static_space_id:
+    value: null
+train_sampling_strategy:
+    value: random
+transformers_version:
+    value: 5.16.0.dev0
+use_cache:
+    value: false
+use_cpu:
+    value: false
+use_liger_kernel:
+    value: false
+vocab_size:
+    value: 4096
+waleed_beta:
+    value: 10
+warmup_steps:
+    value: 0
+weight_decay:
+    value: 0.01

zain/Activation/wandb/run-20260814_212820-ucwxn3gs/files/output.log

@@ -0,0 +1,3 @@
+  5%|▍         | 46/1000 [00:39<15:11,  1.05it/s]
+{'loss': '7.273', 'grad_norm': '1.055', 'learning_rate': '0.001', 'epoch': '0.01078', 'train/total_time_seconds': '6.615', 'train/time_per_step_avg': '0.3307', 'train/epoch_time_elapsed': '14.85', 'train/estimated_remaining_minutes': '5.402'}
+{'loss': '5.986', 'grad_norm': '0.8242', 'learning_rate': '0.001', 'epoch': '0.02156', 'train/total_time_seconds': '18.31', 'train/time_per_step_avg': '0.4577', 'train/epoch_time_elapsed': '34.23', 'train/estimated_remaining_minutes': '7.323'}

zain/Activation/wandb/run-20260814_212820-ucwxn3gs/files/requirements.txt

@@ -0,0 +1,149 @@
+asttokens==3.0.1
+comm==0.2.3
+debugpy==1.8.21
+decorator==5.3.1
+executing==2.2.1
+nest-asyncio==1.6.0
+parso==0.8.7
+platformdirs==4.11.0
+psutil==7.2.2
+ptyprocess==0.7.0
+pure_eval==0.2.3
+Pygments==2.20.0
+pyzmq==27.1.0
+setuptools==83.0.0
+six==1.17.0
+tornado==6.5.7
+traitlets==5.15.0
+fsspec==2026.4.0
+wcwidth==0.8.2
+ipython_pygments_lexers==1.1.1
+jedi==0.20.0
+jupyter_core==5.9.1
+matplotlib-inline==0.2.2
+pexpect==4.9.0
+prompt_toolkit==3.0.53
+python-dateutil==2.9.0.post0
+stack_data==0.6.3
+wheel==0.47.0
+jupyter_client==8.9.1
+pip==26.1.2
+ipython==9.15.0
+ipykernel==7.2.0
+threadpoolctl==3.6.0
+pyparsing==3.3.2
+typing_extensions==4.15.0
+Jinja2==3.1.6
+narwhals==2.24.0
+kiwisolver==1.5.0
+joblib==1.5.3
+fonttools==4.63.0
+cycler==0.12.1
+scipy==1.17.1
+pandas==3.0.5
+contourpy==1.3.3
+scikit-learn==1.9.0
+matplotlib==3.11.1
+urllib3==2.7.0
+tqdm==4.70.0
+idna==3.18
+charset-normalizer==3.4.9
+certifi==2026.7.22
+requests==2.34.2
+seaborn==0.13.2
+uv==0.12.0
+shellingham==1.5.4
+mpmath==1.3.0
+attrs==26.1.0
+hf-xet==1.5.2
+nvidia-nccl-cu12==2.21.5
+MarkupSafe==3.0.3
+regex==2026.7.19
+importlib_metadata==9.0.0
+httpcore==1.0.9
+annotated-doc==0.0.5
+multidict==6.7.1
+aiohttp==3.14.3
+aiosignal==1.4.0
+xxhash==3.8.1
+aiohappyeyeballs==2.7.1
+mdurl==0.1.2
+cuda-toolkit==13.0.3.0
+networkx==3.6.1
+PyYAML==6.0.3
+nvidia-cufile==1.15.1.6
+typer==0.27.0
+torchaudio==2.6.0+cu124
+rich==15.0.0
+nvidia-cufft-cu12==11.2.1.3
+h11==0.16.0
+dill==0.4.1
+cuda-pathfinder==1.6.0
+filelock==3.29.0
+nvidia-nvtx-cu12==12.4.127
+httpx==0.28.1
+anyio==4.14.2
+numpy==2.4.4
+yarl==1.24.5
+click==8.4.2
+triton==3.2.0
+frozenlist==1.8.0
+zipp==4.1.0
+propcache==0.5.2
+tokenizers==0.22.2
+markdown-it-py==4.2.0
+nvidia-cuda-runtime==13.0.96
+cuda-bindings==13.3.1
+nvidia-cuda-cupti==13.0.85
+torch==2.6.0+cu124
+multiprocess==0.70.19
+pillow==12.2.0
+transformers==5.16.0.dev0
+wandb==0.28.1
+nvidia-curand==10.4.0.35
+sympy==1.13.1
+nvidia-cusparse==12.6.3.3
+nvidia-cuda-nvrtc==13.0.88
+typing-inspection==0.4.2
+nvidia-cusolver==12.0.4.66
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.20.0.48
+nvidia-cublas==13.1.1.3
+pyarrow==25.0.0
+evaluate==0.4.6
+diffusers==0.39.0
+pydantic==2.13.4
+annotated-types==0.8.0
+protobuf==7.35.1
+sentry-sdk==2.66.1
+einops==0.8.2
+packaging==26.2
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-curand-cu12==10.3.5.147
+nvidia-cusparselt-cu12==0.6.2
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-cuda-runtime-cu12==12.4.127
+torchvision==0.21.0+cu124
+nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cusolver-cu12==11.6.1.9
+nvidia-cublas-cu12==12.4.5.8
+nvidia-cudnn-cu12==9.1.0.70
+huggingface_hub==1.26.0
+datasets==5.0.1
+safetensors==0.8.0
+accelerate==1.14.0
+pydantic_core==2.46.4
+ninja==1.13.0
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.7.1
+jaraco.text==4.0.0
+jaraco.context==6.1.0
+jaraco.functools==4.4.0
+more-itertools==10.8.0
+packaging==26.0
+platformdirs==4.4.0
+tomli==2.4.0
+wheel==0.46.3
+zipp==3.23.0

zain/Activation/wandb/run-20260814_212820-ucwxn3gs/files/wandb-metadata.json

@@ -0,0 +1,101 @@
+{
+  "os":  "Linux-5.15.0-126-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.11.15",
+  "startedAt":  "2026-08-14T21:28:20.753237Z",
+  "args":  [
+    "--config",
+    "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/configs/baseline.yaml",
+    "--variants",
+    "glu-waleed10",
+    "glu-silu-waleed10",
+    "mlp-waleed10",
+    "mlp-silu-waleed10",
+    "glu-waleed",
+    "glu-situglu_low",
+    "glu-waleedglu_low"
+  ],
+  "program":  "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py",
+  "codePath":  "sweep.py",
+  "codePathLocal":  "sweep.py",
+  "git":  {
+    "remote":  "https://github.com/w-ahmad1a10/Activation.git",
+    "commit":  "3fb47c6943d6927eafc39bd399a3d3a520bae74a"
+  },
+  "email":  "deepnevro@gmail.com",
+  "root":  "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation",
+  "host":  "deeplens-k3s-node1",
+  "executable":  "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/bin/python",
+  "cpu_count":  112,
+  "cpu_count_logical":  224,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1560765693952",
+      "used":  "706489917440"
+    }
+  },
+  "memory":  {
+    "total":  "2164089937920"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-39c684a5-fde6-83d7-1663-0859795881ae"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-68012e5a-38b6-b643-0ca6-62fb66720bf3"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-132944c4-b689-2b5f-89a4-d730401677ab"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-2df386cc-6d26-d0e2-7a2d-a057b0d95864"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bfa16575-1d94-1aa2-4537-2c93433f42ef"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bc6c3e3c-9b90-09ca-c034-774961847c54"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-00a441e1-7c95-e7d6-4c35-43d6b291aea9"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-1c4d29a2-4647-6fce-d8fc-0c5ecfbbd6ea"
+    }
+  ],
+  "cudaVersion":  "12.4",
+  "writerId":  "2drnyjxeagicq7ia3qgncq8z361n6gif"
+}