diff --git "a/cocoad/coco0/mambaad_coco0_log_train.txt" "b/cocoad/coco0/mambaad_coco0_log_train.txt" new file mode 100644--- /dev/null +++ "b/cocoad/coco0/mambaad_coco0_log_train.txt" @@ -0,0 +1,7955 @@ +2024-04-02 01:00:20,559 - ==> Logging on master GPU: 0 +2024-04-02 01:00:20,559 - ==> Running Trainer: MAMBAADTrainer +2024-04-02 01:00:20,559 - ==> Using GPU: [0] for Training +2024-04-02 01:00:20,559 - ==> Building model +2024-04-02 01:00:20,957 - Loaded from checkpoint 'model/pretrain/resnet34-43635321.pth' +2024-04-02 01:00:40,110 - +------------------------------------ MAMBAAD ------------------------------------ +| module | #parameters or shape | #flops | +|:------------------------------------------------------------|:-----------------------|:-------------| +| model | 25.667M | 8.3G | +| net_t | 8.17M | 3.955G | +| net_t.conv1 | 9.408K | 0.154G | +| net_t.conv1.weight | (64, 3, 7, 7) | | +| net_t.bn1 | 0.128K | 2.097M | +| net_t.bn1.weight | (64,) | | +| net_t.bn1.bias | (64,) | | +| net_t.layer1 | 0.222M | 0.909G | +| net_t.layer1.0 | 73.984K | 0.303G | +| net_t.layer1.0.conv1 | 36.864K | 0.151G | +| net_t.layer1.0.conv1.weight | (64, 64, 3, 3) | | +| net_t.layer1.0.bn1 | 0.128K | 0.524M | +| net_t.layer1.0.bn1.weight | (64,) | | +| net_t.layer1.0.bn1.bias | (64,) | | +| net_t.layer1.0.conv2 | 36.864K | 0.151G | +| net_t.layer1.0.conv2.weight | (64, 64, 3, 3) | | +| net_t.layer1.0.bn2 | 0.128K | 0.524M | +| net_t.layer1.0.bn2.weight | (64,) | | +| net_t.layer1.0.bn2.bias | (64,) | | +| net_t.layer1.1 | 73.984K | 0.303G | +| net_t.layer1.1.conv1 | 36.864K | 0.151G | +| net_t.layer1.1.conv1.weight | (64, 64, 3, 3) | | +| net_t.layer1.1.bn1 | 0.128K | 0.524M | +| net_t.layer1.1.bn1.weight | (64,) | | +| net_t.layer1.1.bn1.bias | (64,) | | +| net_t.layer1.1.conv2 | 36.864K | 0.151G | +| net_t.layer1.1.conv2.weight | (64, 64, 3, 3) | | +| net_t.layer1.1.bn2 | 0.128K | 0.524M | +| net_t.layer1.1.bn2.weight | (64,) | | +| net_t.layer1.1.bn2.bias | (64,) | | +| net_t.layer1.2 | 73.984K | 0.303G | +| net_t.layer1.2.conv1 | 36.864K | 0.151G | +| net_t.layer1.2.conv1.weight | (64, 64, 3, 3) | | +| net_t.layer1.2.bn1 | 0.128K | 0.524M | +| net_t.layer1.2.bn1.weight | (64,) | | +| net_t.layer1.2.bn1.bias | (64,) | | +| net_t.layer1.2.conv2 | 36.864K | 0.151G | +| net_t.layer1.2.conv2.weight | (64, 64, 3, 3) | | +| net_t.layer1.2.bn2 | 0.128K | 0.524M | +| net_t.layer1.2.bn2.weight | (64,) | | +| net_t.layer1.2.bn2.bias | (64,) | | +| net_t.layer2 | 1.116M | 1.143G | +| net_t.layer2.0 | 0.23M | 0.236G | +| net_t.layer2.0.conv1 | 73.728K | 75.497M | +| net_t.layer2.0.conv1.weight | (128, 64, 3, 3) | | +| net_t.layer2.0.bn1 | 0.256K | 0.262M | +| net_t.layer2.0.bn1.weight | (128,) | | +| net_t.layer2.0.bn1.bias | (128,) | | +| net_t.layer2.0.conv2 | 0.147M | 0.151G | +| net_t.layer2.0.conv2.weight | (128, 128, 3, 3) | | +| net_t.layer2.0.bn2 | 0.256K | 0.262M | +| net_t.layer2.0.bn2.weight | (128,) | | +| net_t.layer2.0.bn2.bias | (128,) | | +| net_t.layer2.0.downsample | 8.448K | 8.651M | +| net_t.layer2.0.downsample.0 | 8.192K | 8.389M | +| net_t.layer2.0.downsample.1 | 0.256K | 0.262M | +| net_t.layer2.1 | 0.295M | 0.303G | +| net_t.layer2.1.conv1 | 0.147M | 0.151G | +| net_t.layer2.1.conv1.weight | (128, 128, 3, 3) | | +| net_t.layer2.1.bn1 | 0.256K | 0.262M | +| net_t.layer2.1.bn1.weight | (128,) | | +| net_t.layer2.1.bn1.bias | (128,) | | +| net_t.layer2.1.conv2 | 0.147M | 0.151G | +| net_t.layer2.1.conv2.weight | (128, 128, 3, 3) | | +| net_t.layer2.1.bn2 | 0.256K | 0.262M | +| net_t.layer2.1.bn2.weight | (128,) | | +| net_t.layer2.1.bn2.bias | (128,) | | +| net_t.layer2.2 | 0.295M | 0.303G | +| net_t.layer2.2.conv1 | 0.147M | 0.151G | +| net_t.layer2.2.conv1.weight | (128, 128, 3, 3) | | +| net_t.layer2.2.bn1 | 0.256K | 0.262M | +| net_t.layer2.2.bn1.weight | (128,) | | +| net_t.layer2.2.bn1.bias | (128,) | | +| net_t.layer2.2.conv2 | 0.147M | 0.151G | +| net_t.layer2.2.conv2.weight | (128, 128, 3, 3) | | +| net_t.layer2.2.bn2 | 0.256K | 0.262M | +| net_t.layer2.2.bn2.weight | (128,) | | +| net_t.layer2.2.bn2.bias | (128,) | | +| net_t.layer2.3 | 0.295M | 0.303G | +| net_t.layer2.3.conv1 | 0.147M | 0.151G | +| net_t.layer2.3.conv1.weight | (128, 128, 3, 3) | | +| net_t.layer2.3.bn1 | 0.256K | 0.262M | +| net_t.layer2.3.bn1.weight | (128,) | | +| net_t.layer2.3.bn1.bias | (128,) | | +| net_t.layer2.3.conv2 | 0.147M | 0.151G | +| net_t.layer2.3.conv2.weight | (128, 128, 3, 3) | | +| net_t.layer2.3.bn2 | 0.256K | 0.262M | +| net_t.layer2.3.bn2.weight | (128,) | | +| net_t.layer2.3.bn2.bias | (128,) | | +| net_t.layer3 | 6.822M | 1.747G | +| net_t.layer3.0 | 0.919M | 0.235G | +| net_t.layer3.0.conv1 | 0.295M | 75.497M | +| net_t.layer3.0.conv1.weight | (256, 128, 3, 3) | | +| net_t.layer3.0.bn1 | 0.512K | 0.131M | +| net_t.layer3.0.bn1.weight | (256,) | | +| net_t.layer3.0.bn1.bias | (256,) | | +| net_t.layer3.0.conv2 | 0.59M | 0.151G | +| net_t.layer3.0.conv2.weight | (256, 256, 3, 3) | | +| net_t.layer3.0.bn2 | 0.512K | 0.131M | +| net_t.layer3.0.bn2.weight | (256,) | | +| net_t.layer3.0.bn2.bias | (256,) | | +| net_t.layer3.0.downsample | 33.28K | 8.52M | +| net_t.layer3.0.downsample.0 | 32.768K | 8.389M | +| net_t.layer3.0.downsample.1 | 0.512K | 0.131M | +| net_t.layer3.1 | 1.181M | 0.302G | +| net_t.layer3.1.conv1 | 0.59M | 0.151G | +| net_t.layer3.1.conv1.weight | (256, 256, 3, 3) | | +| net_t.layer3.1.bn1 | 0.512K | 0.131M | +| net_t.layer3.1.bn1.weight | (256,) | | +| net_t.layer3.1.bn1.bias | (256,) | | +| net_t.layer3.1.conv2 | 0.59M | 0.151G | +| net_t.layer3.1.conv2.weight | (256, 256, 3, 3) | | +| net_t.layer3.1.bn2 | 0.512K | 0.131M | +| net_t.layer3.1.bn2.weight | (256,) | | +| net_t.layer3.1.bn2.bias | (256,) | | +| net_t.layer3.2 | 1.181M | 0.302G | +| net_t.layer3.2.conv1 | 0.59M | 0.151G | +| net_t.layer3.2.conv1.weight | (256, 256, 3, 3) | | +| net_t.layer3.2.bn1 | 0.512K | 0.131M | +| net_t.layer3.2.bn1.weight | (256,) | | +| net_t.layer3.2.bn1.bias | (256,) | | +| net_t.layer3.2.conv2 | 0.59M | 0.151G | +| net_t.layer3.2.conv2.weight | (256, 256, 3, 3) | | +| net_t.layer3.2.bn2 | 0.512K | 0.131M | +| net_t.layer3.2.bn2.weight | (256,) | | +| net_t.layer3.2.bn2.bias | (256,) | | +| net_t.layer3.3 | 1.181M | 0.302G | +| net_t.layer3.3.conv1 | 0.59M | 0.151G | +| net_t.layer3.3.conv1.weight | (256, 256, 3, 3) | | +| net_t.layer3.3.bn1 | 0.512K | 0.131M | +| net_t.layer3.3.bn1.weight | (256,) | | +| net_t.layer3.3.bn1.bias | (256,) | | +| net_t.layer3.3.conv2 | 0.59M | 0.151G | +| net_t.layer3.3.conv2.weight | (256, 256, 3, 3) | | +| net_t.layer3.3.bn2 | 0.512K | 0.131M | +| net_t.layer3.3.bn2.weight | (256,) | | +| net_t.layer3.3.bn2.bias | (256,) | | +| net_t.layer3.4 | 1.181M | 0.302G | +| net_t.layer3.4.conv1 | 0.59M | 0.151G | +| net_t.layer3.4.conv1.weight | (256, 256, 3, 3) | | +| net_t.layer3.4.bn1 | 0.512K | 0.131M | +| net_t.layer3.4.bn1.weight | (256,) | | +| net_t.layer3.4.bn1.bias | (256,) | | +| net_t.layer3.4.conv2 | 0.59M | 0.151G | +| net_t.layer3.4.conv2.weight | (256, 256, 3, 3) | | +| net_t.layer3.4.bn2 | 0.512K | 0.131M | +| net_t.layer3.4.bn2.weight | (256,) | | +| net_t.layer3.4.bn2.bias | (256,) | | +| net_t.layer3.5 | 1.181M | 0.302G | +| net_t.layer3.5.conv1 | 0.59M | 0.151G | +| net_t.layer3.5.conv1.weight | (256, 256, 3, 3) | | +| net_t.layer3.5.bn1 | 0.512K | 0.131M | +| net_t.layer3.5.bn1.weight | (256,) | | +| net_t.layer3.5.bn1.bias | (256,) | | +| net_t.layer3.5.conv2 | 0.59M | 0.151G | +| net_t.layer3.5.conv2.weight | (256, 256, 3, 3) | | +| net_t.layer3.5.bn2 | 0.512K | 0.131M | +| net_t.layer3.5.bn2.weight | (256,) | | +| net_t.layer3.5.bn2.bias | (256,) | | +| mff_oce | 1.458M | 0.269G | +| mff_oce.bn_layer | 0.94M | 66.47M | +| mff_oce.bn_layer.0 | 0.379M | 30.622M | +| mff_oce.bn_layer.0.conv1 | 32.768K | 8.389M | +| mff_oce.bn_layer.0.conv1.weight | (128, 256, 1, 1) | | +| mff_oce.bn_layer.0.bn1 | 0.256K | 65.536K | +| mff_oce.bn_layer.0.bn1.weight | (128,) | | +| mff_oce.bn_layer.0.bn1.bias | (128,) | | +| mff_oce.bn_layer.0.conv2 | 0.147M | 9.437M | +| mff_oce.bn_layer.0.conv2.weight | (128, 128, 3, 3) | | +| mff_oce.bn_layer.0.bn2 | 0.256K | 16.384K | +| mff_oce.bn_layer.0.bn2.weight | (128,) | | +| mff_oce.bn_layer.0.bn2.bias | (128,) | | +| mff_oce.bn_layer.0.conv3 | 65.536K | 4.194M | +| mff_oce.bn_layer.0.conv3.weight | (512, 128, 1, 1) | | +| mff_oce.bn_layer.0.bn3 | 1.024K | 65.536K | +| mff_oce.bn_layer.0.bn3.weight | (512,) | | +| mff_oce.bn_layer.0.bn3.bias | (512,) | | +| mff_oce.bn_layer.0.downsample | 0.132M | 8.454M | +| mff_oce.bn_layer.0.downsample.0 | 0.131M | 8.389M | +| mff_oce.bn_layer.0.downsample.1 | 1.024K | 65.536K | +| mff_oce.bn_layer.1 | 0.28M | 17.924M | +| mff_oce.bn_layer.1.conv1 | 65.536K | 4.194M | +| mff_oce.bn_layer.1.conv1.weight | (128, 512, 1, 1) | | +| mff_oce.bn_layer.1.bn1 | 0.256K | 16.384K | +| mff_oce.bn_layer.1.bn1.weight | (128,) | | +| mff_oce.bn_layer.1.bn1.bias | (128,) | | +| mff_oce.bn_layer.1.conv2 | 0.147M | 9.437M | +| mff_oce.bn_layer.1.conv2.weight | (128, 128, 3, 3) | | +| mff_oce.bn_layer.1.bn2 | 0.256K | 16.384K | +| mff_oce.bn_layer.1.bn2.weight | (128,) | | +| mff_oce.bn_layer.1.bn2.bias | (128,) | | +| mff_oce.bn_layer.1.conv3 | 65.536K | 4.194M | +| mff_oce.bn_layer.1.conv3.weight | (512, 128, 1, 1) | | +| mff_oce.bn_layer.1.bn3 | 1.024K | 65.536K | +| mff_oce.bn_layer.1.bn3.weight | (512,) | | +| mff_oce.bn_layer.1.bn3.bias | (512,) | | +| mff_oce.bn_layer.2 | 0.28M | 17.924M | +| mff_oce.bn_layer.2.conv1 | 65.536K | 4.194M | +| mff_oce.bn_layer.2.conv1.weight | (128, 512, 1, 1) | | +| mff_oce.bn_layer.2.bn1 | 0.256K | 16.384K | +| mff_oce.bn_layer.2.bn1.weight | (128,) | | +| mff_oce.bn_layer.2.bn1.bias | (128,) | | +| mff_oce.bn_layer.2.conv2 | 0.147M | 9.437M | +| mff_oce.bn_layer.2.conv2.weight | (128, 128, 3, 3) | | +| mff_oce.bn_layer.2.bn2 | 0.256K | 16.384K | +| mff_oce.bn_layer.2.bn2.weight | (128,) | | +| mff_oce.bn_layer.2.bn2.bias | (128,) | | +| mff_oce.bn_layer.2.conv3 | 65.536K | 4.194M | +| mff_oce.bn_layer.2.conv3.weight | (512, 128, 1, 1) | | +| mff_oce.bn_layer.2.bn3 | 1.024K | 65.536K | +| mff_oce.bn_layer.2.bn3.weight | (512,) | | +| mff_oce.bn_layer.2.bn3.bias | (512,) | | +| mff_oce.conv1 | 73.728K | 75.497M | +| mff_oce.conv1.weight | (128, 64, 3, 3) | | +| mff_oce.bn1 | 0.256K | 0.262M | +| mff_oce.bn1.weight | (128,) | | +| mff_oce.bn1.bias | (128,) | | +| mff_oce.conv2 | 0.295M | 75.497M | +| mff_oce.conv2.weight | (256, 128, 3, 3) | | +| mff_oce.bn2 | 0.512K | 0.131M | +| mff_oce.bn2.weight | (256,) | | +| mff_oce.bn2.bias | (256,) | | +| mff_oce.conv21 | 16.512K | 16.777M | +| mff_oce.conv21.weight | (128, 128, 1, 1) | | +| mff_oce.conv21.bias | (128,) | | +| mff_oce.bn21 | 0.256K | 0.262M | +| mff_oce.bn21.weight | (128,) | | +| mff_oce.bn21.bias | (128,) | | +| mff_oce.conv31 | 65.792K | 16.777M | +| mff_oce.conv31.weight | (256, 256, 1, 1) | | +| mff_oce.conv31.bias | (256,) | | +| mff_oce.bn31 | 0.512K | 0.131M | +| mff_oce.bn31.weight | (256,) | | +| mff_oce.bn31.bias | (256,) | | +| mff_oce.convf | 65.792K | 16.777M | +| mff_oce.convf.weight | (256, 256, 1, 1) | | +| mff_oce.convf.bias | (256,) | | +| mff_oce.bnf | 0.512K | 0.131M | +| mff_oce.bnf.weight | (256,) | | +| mff_oce.bnf.bias | (256,) | | +| net_s.layers_up | 16.038M | 4.076G | +| net_s.layers_up.0.blocks.0 | 9.441M | 0.577G | +| net_s.layers_up.0.blocks.0.smm_blocks | 7.561M | 0.456G | +| net_s.layers_up.0.blocks.0.smm_blocks.0 | 2.52M | 0.152G | +| net_s.layers_up.0.blocks.0.smm_blocks.0.ln_1 | 1.024K | 0.164M | +| net_s.layers_up.0.blocks.0.smm_blocks.0.self_attention | 2.519M | 0.152G | +| net_s.layers_up.0.blocks.0.smm_blocks.1 | 2.52M | 0.152G | +| net_s.layers_up.0.blocks.0.smm_blocks.1.ln_1 | 1.024K | 0.164M | +| net_s.layers_up.0.blocks.0.smm_blocks.1.self_attention | 2.519M | 0.152G | +| net_s.layers_up.0.blocks.0.smm_blocks.2 | 2.52M | 0.152G | +| net_s.layers_up.0.blocks.0.smm_blocks.2.ln_1 | 1.024K | 0.164M | +| net_s.layers_up.0.blocks.0.smm_blocks.2.self_attention | 2.519M | 0.152G | +| net_s.layers_up.0.blocks.0.conv1b3 | 0.263M | 16.908M | +| net_s.layers_up.0.blocks.0.conv1b3.0 | 0.263M | 16.777M | +| net_s.layers_up.0.blocks.0.conv1b3.0.weight | (512, 512, 1, 1) | | +| net_s.layers_up.0.blocks.0.conv1b3.0.bias | (512,) | | +| net_s.layers_up.0.blocks.0.conv1b3.1 | | 0.131M | +| net_s.layers_up.0.blocks.0.conv1a3 | 0.263M | 16.908M | +| net_s.layers_up.0.blocks.0.conv1a3.0 | 0.263M | 16.777M | +| net_s.layers_up.0.blocks.0.conv1a3.0.weight | (512, 512, 1, 1) | | +| net_s.layers_up.0.blocks.0.conv1a3.0.bias | (512,) | | +| net_s.layers_up.0.blocks.0.conv1a3.1 | | 0.131M | +| net_s.layers_up.0.blocks.0.conv1b5 | 0.263M | 16.908M | +| net_s.layers_up.0.blocks.0.conv1b5.0 | 0.263M | 16.777M | +| net_s.layers_up.0.blocks.0.conv1b5.0.weight | (512, 512, 1, 1) | | +| net_s.layers_up.0.blocks.0.conv1b5.0.bias | (512,) | | +| net_s.layers_up.0.blocks.0.conv1b5.1 | | 0.131M | +| net_s.layers_up.0.blocks.0.conv1a5 | 0.263M | 16.908M | +| net_s.layers_up.0.blocks.0.conv1a5.0 | 0.263M | 16.777M | +| net_s.layers_up.0.blocks.0.conv1a5.0.weight | (512, 512, 1, 1) | | +| net_s.layers_up.0.blocks.0.conv1a5.0.bias | (512,) | | +| net_s.layers_up.0.blocks.0.conv1a5.1 | | 0.131M | +| net_s.layers_up.0.blocks.0.conv33.0 | 4.608K | | +| net_s.layers_up.0.blocks.0.conv33.0.weight | (512, 1, 3, 3) | | +| net_s.layers_up.0.blocks.0.conv55 | 12.8K | 0.95M | +| net_s.layers_up.0.blocks.0.conv55.0 | 12.8K | 0.819M | +| net_s.layers_up.0.blocks.0.conv55.0.weight | (512, 1, 5, 5) | | +| net_s.layers_up.0.blocks.0.conv55.1 | | 0.131M | +| net_s.layers_up.0.blocks.0.conv77 | 25.088K | 1.737M | +| net_s.layers_up.0.blocks.0.conv77.0 | 25.088K | 1.606M | +| net_s.layers_up.0.blocks.0.conv77.0.weight | (512, 1, 7, 7) | | +| net_s.layers_up.0.blocks.0.conv77.1 | | 0.131M | +| net_s.layers_up.0.blocks.0.finalconv11 | 0.787M | 50.332M | +| net_s.layers_up.0.blocks.0.finalconv11.weight | (512, 1536, 1, 1) | | +| net_s.layers_up.0.blocks.0.finalconv11.bias | (512,) | | +| net_s.layers_up.1 | 4.432M | 0.961G | +| net_s.layers_up.1.blocks | 3.908M | 0.928G | +| net_s.layers_up.1.blocks.0 | 1.954M | 0.464G | +| net_s.layers_up.1.blocks.0.smm_blocks | 1.473M | 0.34G | +| net_s.layers_up.1.blocks.0.conv1b3 | 65.792K | 17.039M | +| net_s.layers_up.1.blocks.0.conv1a3 | 65.792K | 17.039M | +| net_s.layers_up.1.blocks.0.conv1b5 | 65.792K | 17.039M | +| net_s.layers_up.1.blocks.0.conv1a5 | 65.792K | 17.039M | +| net_s.layers_up.1.blocks.0.conv33.0 | 2.304K | | +| net_s.layers_up.1.blocks.0.conv55 | 6.4K | 1.901M | +| net_s.layers_up.1.blocks.0.conv77 | 12.544K | 3.473M | +| net_s.layers_up.1.blocks.0.finalconv11 | 0.197M | 50.332M | +| net_s.layers_up.1.blocks.1 | 1.954M | 0.464G | +| net_s.layers_up.1.blocks.1.smm_blocks | 1.473M | 0.34G | +| net_s.layers_up.1.blocks.1.conv1b3 | 65.792K | 17.039M | +| net_s.layers_up.1.blocks.1.conv1a3 | 65.792K | 17.039M | +| net_s.layers_up.1.blocks.1.conv1b5 | 65.792K | 17.039M | +| net_s.layers_up.1.blocks.1.conv1a5 | 65.792K | 17.039M | +| net_s.layers_up.1.blocks.1.conv33.0 | 2.304K | | +| net_s.layers_up.1.blocks.1.conv55 | 6.4K | 1.901M | +| net_s.layers_up.1.blocks.1.conv77 | 12.544K | 3.473M | +| net_s.layers_up.1.blocks.1.finalconv11 | 0.197M | 50.332M | +| net_s.layers_up.1.upsample | 0.525M | 33.882M | +| net_s.layers_up.1.upsample.expand | 0.524M | 33.554M | +| net_s.layers_up.1.upsample.expand.weight | (1024, 512) | | +| net_s.layers_up.1.upsample.norm | 0.512K | 0.328M | +| net_s.layers_up.1.upsample.norm.weight | (256,) | | +| net_s.layers_up.1.upsample.norm.bias | (256,) | | +| net_s.layers_up.2 | 1.816M | 1.529G | +| net_s.layers_up.2.blocks | 1.685M | 1.495G | +| net_s.layers_up.2.blocks.0 | 0.842M | 0.747G | +| net_s.layers_up.2.blocks.0.smm_blocks | 0.717M | 0.617G | +| net_s.layers_up.2.blocks.0.conv1b3 | 16.512K | 17.302M | +| net_s.layers_up.2.blocks.0.conv1a3 | 16.512K | 17.302M | +| net_s.layers_up.2.blocks.0.conv1b5 | 16.512K | 17.302M | +| net_s.layers_up.2.blocks.0.conv1a5 | 16.512K | 17.302M | +| net_s.layers_up.2.blocks.0.conv33.0 | 1.152K | | +| net_s.layers_up.2.blocks.0.conv55 | 3.2K | 3.801M | +| net_s.layers_up.2.blocks.0.conv77 | 6.272K | 6.947M | +| net_s.layers_up.2.blocks.0.finalconv11 | 49.28K | 50.332M | +| net_s.layers_up.2.blocks.1 | 0.842M | 0.747G | +| net_s.layers_up.2.blocks.1.smm_blocks | 0.717M | 0.617G | +| net_s.layers_up.2.blocks.1.conv1b3 | 16.512K | 17.302M | +| net_s.layers_up.2.blocks.1.conv1a3 | 16.512K | 17.302M | +| net_s.layers_up.2.blocks.1.conv1b5 | 16.512K | 17.302M | +| net_s.layers_up.2.blocks.1.conv1a5 | 16.512K | 17.302M | +| net_s.layers_up.2.blocks.1.conv33.0 | 1.152K | | +| net_s.layers_up.2.blocks.1.conv55 | 3.2K | 3.801M | +| net_s.layers_up.2.blocks.1.conv77 | 6.272K | 6.947M | +| net_s.layers_up.2.blocks.1.finalconv11 | 49.28K | 50.332M | +| net_s.layers_up.2.upsample | 0.131M | 34.21M | +| net_s.layers_up.2.upsample.expand | 0.131M | 33.554M | +| net_s.layers_up.2.upsample.expand.weight | (512, 256) | | +| net_s.layers_up.2.upsample.norm | 0.256K | 0.655M | +| net_s.layers_up.2.upsample.norm.weight | (128,) | | +| net_s.layers_up.2.upsample.norm.bias | (128,) | | +| net_s.layers_up.3 | 0.349M | 1.009G | +| net_s.layers_up.3.blocks.0 | 0.316M | 0.974G | +| net_s.layers_up.3.blocks.0.smm_blocks | 0.281M | 0.831G | +| net_s.layers_up.3.blocks.0.smm_blocks.0 | 93.824K | 0.277G | +| net_s.layers_up.3.blocks.0.smm_blocks.1 | 93.824K | 0.277G | +| net_s.layers_up.3.blocks.0.smm_blocks.2 | 93.824K | 0.277G | +| net_s.layers_up.3.blocks.0.conv1b3 | 4.16K | 17.826M | +| net_s.layers_up.3.blocks.0.conv1b3.0 | 4.16K | 16.777M | +| net_s.layers_up.3.blocks.0.conv1b3.1 | | 1.049M | +| net_s.layers_up.3.blocks.0.conv1a3 | 4.16K | 17.826M | +| net_s.layers_up.3.blocks.0.conv1a3.0 | 4.16K | 16.777M | +| net_s.layers_up.3.blocks.0.conv1a3.1 | | 1.049M | +| net_s.layers_up.3.blocks.0.conv1b5 | 4.16K | 17.826M | +| net_s.layers_up.3.blocks.0.conv1b5.0 | 4.16K | 16.777M | +| net_s.layers_up.3.blocks.0.conv1b5.1 | | 1.049M | +| net_s.layers_up.3.blocks.0.conv1a5 | 4.16K | 17.826M | +| net_s.layers_up.3.blocks.0.conv1a5.0 | 4.16K | 16.777M | +| net_s.layers_up.3.blocks.0.conv1a5.1 | | 1.049M | +| net_s.layers_up.3.blocks.0.conv33.0 | 0.576K | | +| net_s.layers_up.3.blocks.0.conv33.0.weight | (64, 1, 3, 3) | | +| net_s.layers_up.3.blocks.0.conv55 | 1.6K | 7.602M | +| net_s.layers_up.3.blocks.0.conv55.0 | 1.6K | 6.554M | +| net_s.layers_up.3.blocks.0.conv55.1 | | 1.049M | +| net_s.layers_up.3.blocks.0.conv77 | 3.136K | 13.894M | +| net_s.layers_up.3.blocks.0.conv77.0 | 3.136K | 12.845M | +| net_s.layers_up.3.blocks.0.conv77.1 | | 1.049M | +| net_s.layers_up.3.blocks.0.finalconv11 | 12.352K | 50.332M | +| net_s.layers_up.3.blocks.0.finalconv11.weight | (64, 192, 1, 1) | | +| net_s.layers_up.3.blocks.0.finalconv11.bias | (64,) | | +| net_s.layers_up.3.upsample | 32.896K | 34.865M | +| net_s.layers_up.3.upsample.expand | 32.768K | 33.554M | +| net_s.layers_up.3.upsample.expand.weight | (256, 128) | | +| net_s.layers_up.3.upsample.norm | 0.128K | 1.311M | +| net_s.layers_up.3.upsample.norm.weight | (64,) | | +| net_s.layers_up.3.upsample.norm.bias | (64,) | | +--------------------------------------------------------------------------------- +2024-04-02 01:00:40,111 - ==> Creating optimizer +2024-04-02 01:00:40,191 - ==> Loading dataset: DefaultAD +2024-04-02 01:00:40,372 - ==> ********** cfg ********** +epoch_full : 1000 +fvcore_b : 1 +fvcore_c : 3 +metrics : ['mAUROC_sp_max', 'mAP_sp_max', 'mF1_max_sp_max', 'mAUROC_px', 'mAP_px', 'mF1_max_px', 'mAUPRO_px', 'mF1_px_0.2_0.8_0.1', 'mAcc_px_0.2_0.8_0.1', 'mIoU_px_0.2_0.8_0.1', 'mIoU_max_px'] +evaluator.kwargs : {'metrics': ['mAUROC_sp_max', 'mAP_sp_max', 'mF1_max_sp_max', 'mAUROC_px', 'mAP_px', 'mF1_max_px', 'mAUPRO_px', 'mF1_px_0.2_0.8_0.1', 'mAcc_px_0.2_0.8_0.1', 'mIoU_px_0.2_0.8_0.1', 'mIoU_max_px'], 'pooling_ks': None, 'max_step_aupro': 100} +optim.lr : 0.005 +optim.kwargs : {'name': 'adamw', 'betas': (0.9, 0.999), 'eps': 1e-08, 'weight_decay': 0.0001, 'amsgrad': False} +trainer.name : MAMBAADTrainer +trainer.checkpoint : runs +trainer.logdir_sub : +trainer.resume_dir : +trainer.cuda_deterministic : False +trainer.epoch_full : 1000 +trainer.scheduler_kwargs : {'name': 'step', 'lr_noise': None, 'noise_pct': 0.67, 'noise_std': 1.0, 'noise_seed': 42, 'lr_min': 5e-05, 'warmup_lr': 5e-06, 'warmup_iters': -1, 'cooldown_iters': 0, 'warmup_epochs': 0, 'cooldown_epochs': 0, 'use_iters': True, 'patience_iters': 0, 'patience_epochs': 0, 'decay_iters': 0, 'decay_epochs': 800, 'cycle_decay': 0.1, 'decay_rate': 0.1} +trainer.mixup_kwargs : {'mixup_alpha': 0.8, 'cutmix_alpha': 1.0, 'cutmix_minmax': None, 'prob': 0.0, 'switch_prob': 0.5, 'mode': 'batch', 'correct_lam': True, 'label_smoothing': 0.1} +trainer.test_start_epoch : 1000 +trainer.test_per_epoch : 200 +trainer.find_unused_parameters : False +trainer.sync_BN : apex +trainer.dist_BN : +trainer.scaler : none +trainer.data.batch_size : 16 +trainer.data.batch_size_per_gpu : 16 +trainer.data.batch_size_test : 16 +trainer.data.batch_size_per_gpu_test : 16 +trainer.data.num_workers_per_gpu : 4 +trainer.data.drop_last : True +trainer.data.pin_memory : True +trainer.data.persistent_workers : False +trainer.data.num_workers : 4 +trainer.iter : 0 +trainer.epoch : 0 +trainer.iter_full : 1902000 +trainer.metric_recorder : {'mAUROC_sp_max_coco': [], 'mAP_sp_max_coco': [], 'mF1_max_sp_max_coco': [], 'mAUROC_px_coco': [], 'mAP_px_coco': [], 'mF1_max_px_coco': [], 'mAUPRO_px_coco': [], 'mF1_px_0.2_0.8_0.1_coco': [], 'mAcc_px_0.2_0.8_0.1_coco': [], 'mIoU_px_0.2_0.8_0.1_coco': [], 'mIoU_max_px_coco': []} +loss.loss_terms : [{'type': 'L2Loss', 'name': 'pixel', 'lam': 5.0}] +loss.clip_grad : 5.0 +loss.create_graph : False +loss.retain_graph : False +adv : False +logging.log_terms_train : [{'name': 'batch_t', 'fmt': ':>5.3f', 'add_name': 'avg'}, {'name': 'data_t', 'fmt': ':>5.3f'}, {'name': 'optim_t', 'fmt': ':>5.3f'}, {'name': 'lr', 'fmt': ':>7.6f'}, {'name': 'cos', 'suffixes': [''], 'fmt': ':>5.3f', 'add_name': 'avg'}] +logging.log_terms_test : [{'name': 'batch_t', 'fmt': ':>5.3f', 'add_name': 'avg'}, {'name': 'cos', 'suffixes': [''], 'fmt': ':>5.3f', 'add_name': 'avg'}] +logging.train_reset_log_per : 100 +logging.train_log_per : 100 +logging.test_log_per : 50 +data.sampler : naive +data.loader_type : pil +data.loader_type_target : pil_L +data.type : DefaultAD +data.root : data/coco +data.meta : meta_20_0.json +data.cls_names : ['coco'] +data.train_transforms : [{'type': 'Resize', 'size': (256, 256), 'interpolation': }, {'type': 'CenterCrop', 'size': (256, 256)}, {'type': 'ToTensor'}, {'type': 'Normalize', 'mean': (0.485, 0.456, 0.406), 'std': (0.229, 0.224, 0.225), 'inplace': True}] +data.test_transforms : [{'type': 'Resize', 'size': (256, 256), 'interpolation': }, {'type': 'CenterCrop', 'size': (256, 256)}, {'type': 'ToTensor'}, {'type': 'Normalize', 'mean': (0.485, 0.456, 0.406), 'std': (0.229, 0.224, 0.225), 'inplace': True}] +data.target_transforms : [{'type': 'Resize', 'size': (256, 256), 'interpolation': }, {'type': 'CenterCrop', 'size': (256, 256)}, {'type': 'ToTensor'}] +data.train_size : 1902 +data.test_size : 310 +data.train_length : 30438 +data.test_length : 4952 +model_t.name : timm_resnet34 +model_t.kwargs : {'pretrained': False, 'checkpoint_path': 'model/pretrain/resnet34-43635321.pth', 'strict': False, 'features_only': True, 'out_indices': [1, 2, 3]} +model_s : {'depths_decoder': [3, 4, 6, 3], 'scan_type': 'hilbert', 'num_direction': 8} +model.name : mambaadhcs57c1 +model.kwargs : {'pretrained': False, 'checkpoint_path': '', 'strict': True, 'model_t': Namespace(name='timm_resnet34', kwargs={'pretrained': False, 'checkpoint_path': 'model/pretrain/resnet34-43635321.pth', 'strict': False, 'features_only': True, 'out_indices': [1, 2, 3]}), 'model_s': {'depths_decoder': [3, 4, 6, 3], 'scan_type': 'hilbert', 'num_direction': 8}} +seed : 42 +size : 256 +warmup_epochs : 0 +test_start_epoch : 1000 +test_per_epoch : 200 +batch_train : 16 +batch_test_per : 16 +lr : 0.005 +weight_decay : 0.0001 +cfg_path : configs.mambaad.mambaad_coco0_nhcs57c1 +mode : train +sleep : -1 +memory : -1 +dist_url : env:// +logger_rank : 0 +opts : [] +command : python3 -m torch.distributed.launch --nproc_per_node=$nproc_per_node --nnodes=$nnodes --node_rank=$node_rank --master_addr=$master_addr --master_port=$master_port --use_env run.py -c configs.mambaad.mambaad_coco0_nhcs57c1 -m train --sleep -1 --memory -1 --dist_url env:// --logger_rank 0 +task_start_time : 3592051.819355288 +dist : False +world_size : 1 +rank : 0 +local_rank : 0 +ngpus_per_node : 1 +nnodes : 1 +master : True +logdir : runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020 +logger.filters : [] +logger.name : root +logger.level : 20 +logger.parent : None +logger.propagate : True +logger.disabled : False +logdir_train : runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020/show_train +logdir_test : runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020/show_test +2024-04-02 01:00:40,372 - ==> Starting training with 1 nodes x 1 GPUs +2024-04-02 01:02:53,933 - Train: 0.01% [100/1902000] [0.1/1000.0] [batch_t 0.605 (1.332)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 01:04:03,488 - Train: 0.01% [200/1902000] [0.1/1000.0] [batch_t 0.608 (0.695)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 01:05:15,307 - Train: 0.02% [300/1902000] [0.2/1000.0] [batch_t 0.768 (0.718)] [data_t 0.165] [optim_t 0.602] [lr 0.005000] +2024-04-02 01:06:16,789 - Train: 0.02% [400/1902000] [0.2/1000.0] [batch_t 0.604 (0.615)] [data_t 0.003] [optim_t 0.601] [lr 0.005000] +2024-04-02 01:09:54,500 - Train: 0.03% [500/1902000] [0.3/1000.0] [batch_t 0.606 (2.177)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 01:17:34,874 - Train: 0.03% [600/1902000] [0.3/1000.0] [batch_t 1.407 (4.604)] [data_t 0.002] [optim_t 1.405] [lr 0.005000] +2024-04-02 01:19:55,460 - Train: 0.04% [700/1902000] [0.4/1000.0] [batch_t 1.404 (1.406)] [data_t 0.002] [optim_t 1.401] [lr 0.005000] +2024-04-02 01:22:16,054 - Train: 0.04% [800/1902000] [0.4/1000.0] [batch_t 1.402 (1.406)] [data_t 0.002] [optim_t 1.399] [lr 0.005000] +2024-04-02 01:24:36,692 - Train: 0.05% [900/1902000] [0.5/1000.0] [batch_t 1.417 (1.406)] [data_t 0.002] [optim_t 1.415] [lr 0.005000] +2024-04-02 01:26:57,286 - Train: 0.05% [1000/1902000] [0.5/1000.0] [batch_t 1.412 (1.406)] [data_t 0.002] [optim_t 1.410] [lr 0.005000] +2024-04-02 01:29:17,973 - Train: 0.06% [1100/1902000] [0.6/1000.0] [batch_t 1.414 (1.407)] [data_t 0.002] [optim_t 1.411] [lr 0.005000] +2024-04-02 01:31:38,552 - Train: 0.06% [1200/1902000] [0.6/1000.0] [batch_t 1.407 (1.406)] [data_t 0.002] [optim_t 1.404] [lr 0.005000] +2024-04-02 01:33:59,232 - Train: 0.07% [1300/1902000] [0.7/1000.0] [batch_t 1.404 (1.407)] [data_t 0.002] [optim_t 1.401] [lr 0.005000] +2024-04-02 01:36:19,819 - Train: 0.07% [1400/1902000] [0.7/1000.0] [batch_t 1.408 (1.406)] [data_t 0.002] [optim_t 1.406] [lr 0.005000] +2024-04-02 01:38:40,458 - Train: 0.08% [1500/1902000] [0.8/1000.0] [batch_t 1.412 (1.406)] [data_t 0.002] [optim_t 1.410] [lr 0.005000] +2024-04-02 01:41:02,024 - Train: 0.08% [1600/1902000] [0.8/1000.0] [batch_t 1.409 (1.416)] [data_t 0.002] [optim_t 1.406] [lr 0.005000] +2024-04-02 01:43:22,703 - Train: 0.09% [1700/1902000] [0.9/1000.0] [batch_t 1.402 (1.407)] [data_t 0.002] [optim_t 1.400] [lr 0.005000] +2024-04-02 01:45:43,282 - Train: 0.09% [1800/1902000] [0.9/1000.0] [batch_t 1.408 (1.406)] [data_t 0.002] [optim_t 1.405] [lr 0.005000] +2024-04-02 01:48:03,951 - Train: 0.10% [1900/1902000] [1.0/1000.0] [batch_t 1.407 (1.407)] [data_t 0.002] [optim_t 1.405] [lr 0.005000] +2024-04-02 01:48:06,765 - ==> Total time: 0:47:46 Eta: 33 days, 3:23:37 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 01:50:26,745 - Train: 0.11% [2000/1902000] [1.1/1000.0] [batch_t 1.412 (1.410)] [data_t 0.002] [optim_t 1.409] [lr 0.005000] +2024-04-02 01:52:47,324 - Train: 0.11% [2100/1902000] [1.1/1000.0] [batch_t 1.405 (1.406)] [data_t 0.002] [optim_t 1.403] [lr 0.005000] +2024-04-02 01:55:07,788 - Train: 0.12% [2200/1902000] [1.2/1000.0] [batch_t 1.409 (1.405)] [data_t 0.002] [optim_t 1.407] [lr 0.005000] +2024-04-02 01:57:28,435 - Train: 0.12% [2300/1902000] [1.2/1000.0] [batch_t 1.412 (1.406)] [data_t 0.002] [optim_t 1.409] [lr 0.005000] +2024-04-02 01:59:49,072 - Train: 0.13% [2400/1902000] [1.3/1000.0] [batch_t 1.403 (1.406)] [data_t 0.002] [optim_t 1.401] [lr 0.005000] +2024-04-02 02:02:09,708 - Train: 0.13% [2500/1902000] [1.3/1000.0] [batch_t 1.412 (1.406)] [data_t 0.002] [optim_t 1.410] [lr 0.005000] +2024-04-02 02:04:30,273 - Train: 0.14% [2600/1902000] [1.4/1000.0] [batch_t 1.407 (1.406)] [data_t 0.002] [optim_t 1.404] [lr 0.005000] +2024-04-02 02:06:50,817 - Train: 0.14% [2700/1902000] [1.4/1000.0] [batch_t 1.414 (1.405)] [data_t 0.002] [optim_t 1.411] [lr 0.005000] +2024-04-02 02:08:14,192 - Train: 0.15% [2800/1902000] [1.5/1000.0] [batch_t 0.604 (0.834)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:09:22,150 - Train: 0.15% [2900/1902000] [1.5/1000.0] [batch_t 0.604 (0.679)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 02:10:23,986 - Train: 0.16% [3000/1902000] [1.6/1000.0] [batch_t 0.605 (0.618)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:11:24,416 - Train: 0.16% [3100/1902000] [1.6/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 02:12:27,530 - Train: 0.17% [3200/1902000] [1.7/1000.0] [batch_t 0.603 (0.631)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 02:13:32,038 - Train: 0.17% [3300/1902000] [1.7/1000.0] [batch_t 0.604 (0.645)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:14:44,604 - Train: 0.18% [3400/1902000] [1.8/1000.0] [batch_t 0.606 (0.726)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 02:15:48,015 - Train: 0.18% [3500/1902000] [1.8/1000.0] [batch_t 0.603 (0.634)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 02:16:51,264 - Train: 0.19% [3600/1902000] [1.9/1000.0] [batch_t 0.604 (0.632)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:17:53,203 - Train: 0.19% [3700/1902000] [1.9/1000.0] [batch_t 0.603 (0.619)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 02:19:06,084 - Train: 0.20% [3800/1902000] [2.0/1000.0] [batch_t 0.602 (0.729)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 02:19:08,504 - ==> Total time: 1:18:48 Eta: 27 days, 7:21:23 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 02:20:08,408 - Train: 0.21% [3900/1902000] [2.1/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:21:10,890 - Train: 0.21% [4000/1902000] [2.1/1000.0] [batch_t 0.605 (0.625)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:22:13,652 - Train: 0.22% [4100/1902000] [2.2/1000.0] [batch_t 0.606 (0.627)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 02:23:18,242 - Train: 0.22% [4200/1902000] [2.2/1000.0] [batch_t 0.603 (0.646)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 02:24:21,610 - Train: 0.23% [4300/1902000] [2.3/1000.0] [batch_t 0.604 (0.634)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 02:25:22,042 - Train: 0.23% [4400/1902000] [2.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 02:26:22,620 - Train: 0.24% [4500/1902000] [2.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 02:27:27,657 - Train: 0.24% [4600/1902000] [2.4/1000.0] [batch_t 0.604 (0.650)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:28:29,937 - Train: 0.25% [4700/1902000] [2.5/1000.0] [batch_t 0.605 (0.623)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:29:31,316 - Train: 0.25% [4800/1902000] [2.5/1000.0] [batch_t 0.602 (0.614)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 02:30:33,676 - Train: 0.26% [4900/1902000] [2.6/1000.0] [batch_t 0.604 (0.624)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 02:31:36,454 - Train: 0.26% [5000/1902000] [2.6/1000.0] [batch_t 0.603 (0.628)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 02:32:38,038 - Train: 0.27% [5100/1902000] [2.7/1000.0] [batch_t 0.604 (0.616)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:33:40,258 - Train: 0.27% [5200/1902000] [2.7/1000.0] [batch_t 0.606 (0.622)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 02:34:46,163 - Train: 0.28% [5300/1902000] [2.8/1000.0] [batch_t 0.602 (0.659)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 02:35:46,609 - Train: 0.28% [5400/1902000] [2.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 02:36:49,997 - Train: 0.29% [5500/1902000] [2.9/1000.0] [batch_t 0.603 (0.634)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 02:37:50,505 - Train: 0.29% [5600/1902000] [2.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:38:52,532 - Train: 0.30% [5700/1902000] [3.0/1000.0] [batch_t 0.605 (0.620)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:38:56,159 - ==> Total time: 1:38:35 Eta: 22 days, 18:06:16 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 02:39:54,825 - Train: 0.30% [5800/1902000] [3.0/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 02:40:55,801 - Train: 0.31% [5900/1902000] [3.1/1000.0] [batch_t 0.606 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 02:41:57,371 - Train: 0.32% [6000/1902000] [3.2/1000.0] [batch_t 0.605 (0.616)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:42:57,815 - Train: 0.32% [6100/1902000] [3.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 02:43:58,768 - Train: 0.33% [6200/1902000] [3.3/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 02:45:00,059 - Train: 0.33% [6300/1902000] [3.3/1000.0] [batch_t 0.607 (0.613)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 02:46:03,541 - Train: 0.34% [6400/1902000] [3.4/1000.0] [batch_t 0.606 (0.635)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 02:47:11,611 - Train: 0.34% [6500/1902000] [3.4/1000.0] [batch_t 0.605 (0.681)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:48:12,540 - Train: 0.35% [6600/1902000] [3.5/1000.0] [batch_t 0.604 (0.609)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 02:49:13,004 - Train: 0.35% [6700/1902000] [3.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 02:50:17,585 - Train: 0.36% [6800/1902000] [3.6/1000.0] [batch_t 0.604 (0.646)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:51:19,499 - Train: 0.36% [6900/1902000] [3.6/1000.0] [batch_t 0.596 (0.619)] [data_t 0.002] [optim_t 0.594] [lr 0.005000] +2024-04-02 02:52:19,974 - Train: 0.37% [7000/1902000] [3.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 02:53:22,916 - Train: 0.37% [7100/1902000] [3.7/1000.0] [batch_t 0.605 (0.629)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 02:54:25,500 - Train: 0.38% [7200/1902000] [3.8/1000.0] [batch_t 0.604 (0.626)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 02:55:25,977 - Train: 0.38% [7300/1902000] [3.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:56:27,994 - Train: 0.39% [7400/1902000] [3.9/1000.0] [batch_t 0.603 (0.620)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 02:57:30,147 - Train: 0.39% [7500/1902000] [3.9/1000.0] [batch_t 0.604 (0.621)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 02:58:51,238 - Train: 0.40% [7600/1902000] [4.0/1000.0] [batch_t 0.605 (0.811)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 02:58:56,076 - ==> Total time: 1:58:35 Eta: 20 days, 12:09:43 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 02:59:54,390 - Train: 0.40% [7700/1902000] [4.0/1000.0] [batch_t 0.604 (0.618)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 03:00:54,886 - Train: 0.41% [7800/1902000] [4.1/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-02 03:01:58,299 - Train: 0.42% [7900/1902000] [4.2/1000.0] [batch_t 0.604 (0.634)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 03:02:59,613 - Train: 0.42% [8000/1902000] [4.2/1000.0] [batch_t 0.604 (0.613)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 03:04:14,593 - Train: 0.43% [8100/1902000] [4.3/1000.0] [batch_t 0.605 (0.750)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 03:05:16,306 - Train: 0.43% [8200/1902000] [4.3/1000.0] [batch_t 0.605 (0.617)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 03:06:20,172 - Train: 0.44% [8300/1902000] [4.4/1000.0] [batch_t 0.606 (0.639)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 03:07:26,932 - Train: 0.44% [8400/1902000] [4.4/1000.0] [batch_t 0.602 (0.668)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 03:08:37,327 - Train: 0.45% [8500/1902000] [4.5/1000.0] [batch_t 0.602 (0.704)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 03:09:39,542 - Train: 0.45% [8600/1902000] [4.5/1000.0] [batch_t 0.604 (0.622)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 03:10:40,476 - Train: 0.46% [8700/1902000] [4.6/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 03:11:43,032 - Train: 0.46% [8800/1902000] [4.6/1000.0] [batch_t 0.607 (0.625)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 03:12:50,655 - Train: 0.47% [8900/1902000] [4.7/1000.0] [batch_t 0.604 (0.676)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 03:13:57,807 - Train: 0.47% [9000/1902000] [4.7/1000.0] [batch_t 0.604 (0.670)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 03:15:40,404 - Train: 0.48% [9100/1902000] [4.8/1000.0] [batch_t 2.800 (1.026)] [data_t 2.199] [optim_t 0.601] [lr 0.005000] +2024-04-02 03:16:46,653 - Train: 0.48% [9200/1902000] [4.8/1000.0] [batch_t 0.607 (0.662)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 03:18:39,186 - Train: 0.49% [9300/1902000] [4.9/1000.0] [batch_t 0.606 (1.125)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 03:19:43,732 - Train: 0.49% [9400/1902000] [4.9/1000.0] [batch_t 0.604 (0.645)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 03:20:46,439 - Train: 0.50% [9500/1902000] [5.0/1000.0] [batch_t 0.693 (0.627)] [data_t 0.002] [optim_t 0.691] [lr 0.005000] +2024-04-02 03:20:52,483 - ==> Total time: 2:20:32 Eta: 19 days, 10:06:08 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 03:22:58,002 - Train: 0.50% [9600/1902000] [5.0/1000.0] [batch_t 2.034 (1.374)] [data_t 1.431] [optim_t 0.603] [lr 0.005000] +2024-04-02 03:25:01,142 - Train: 0.51% [9700/1902000] [5.1/1000.0] [batch_t 0.607 (1.231)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 03:26:26,977 - Train: 0.52% [9800/1902000] [5.2/1000.0] [batch_t 0.604 (0.858)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 03:27:32,826 - Train: 0.52% [9900/1902000] [5.2/1000.0] [batch_t 0.605 (0.658)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 03:28:43,983 - Train: 0.53% [10000/1902000] [5.3/1000.0] [batch_t 0.602 (0.711)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 03:30:00,900 - Train: 0.53% [10100/1902000] [5.3/1000.0] [batch_t 0.608 (0.769)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 03:31:21,841 - Train: 0.54% [10200/1902000] [5.4/1000.0] [batch_t 0.602 (0.809)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 03:32:49,946 - Train: 0.54% [10300/1902000] [5.4/1000.0] [batch_t 0.604 (0.881)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 03:33:51,834 - Train: 0.55% [10400/1902000] [5.5/1000.0] [batch_t 0.606 (0.619)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 03:35:08,254 - Train: 0.55% [10500/1902000] [5.5/1000.0] [batch_t 0.824 (0.764)] [data_t 0.220] [optim_t 0.604] [lr 0.005000] +2024-04-02 03:37:43,029 - Train: 0.56% [10600/1902000] [5.6/1000.0] [batch_t 0.602 (1.548)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 03:38:46,567 - Train: 0.56% [10700/1902000] [5.6/1000.0] [batch_t 0.604 (0.635)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 03:39:49,281 - Train: 0.57% [10800/1902000] [5.7/1000.0] [batch_t 0.606 (0.627)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 03:41:14,609 - Train: 0.57% [10900/1902000] [5.7/1000.0] [batch_t 0.605 (0.853)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 03:42:44,653 - Train: 0.58% [11000/1902000] [5.8/1000.0] [batch_t 0.602 (0.900)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 03:43:55,611 - Train: 0.58% [11100/1902000] [5.8/1000.0] [batch_t 0.605 (0.709)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 03:45:02,073 - Train: 0.59% [11200/1902000] [5.9/1000.0] [batch_t 0.605 (0.665)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 03:46:06,082 - Train: 0.59% [11300/1902000] [5.9/1000.0] [batch_t 0.604 (0.640)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 03:47:29,847 - Train: 0.60% [11400/1902000] [6.0/1000.0] [batch_t 0.605 (0.838)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 03:47:37,100 - ==> Total time: 2:47:16 Eta: 19 days, 5:52:13 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 03:48:32,927 - Train: 0.60% [11500/1902000] [6.0/1000.0] [batch_t 0.607 (0.615)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 03:49:40,398 - Train: 0.61% [11600/1902000] [6.1/1000.0] [batch_t 0.605 (0.675)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 03:50:45,836 - Train: 0.62% [11700/1902000] [6.2/1000.0] [batch_t 0.605 (0.654)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 03:51:52,206 - Train: 0.62% [11800/1902000] [6.2/1000.0] [batch_t 0.605 (0.663)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 03:53:03,132 - Train: 0.63% [11900/1902000] [6.3/1000.0] [batch_t 0.606 (0.709)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 03:54:06,415 - Train: 0.63% [12000/1902000] [6.3/1000.0] [batch_t 0.606 (0.633)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 03:55:50,163 - Train: 0.64% [12100/1902000] [6.4/1000.0] [batch_t 0.603 (1.037)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 03:57:22,576 - Train: 0.64% [12200/1902000] [6.4/1000.0] [batch_t 0.604 (0.924)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 03:58:30,715 - Train: 0.65% [12300/1902000] [6.5/1000.0] [batch_t 0.604 (0.681)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 03:59:51,127 - Train: 0.65% [12400/1902000] [6.5/1000.0] [batch_t 0.763 (0.804)] [data_t 0.159] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:03:30,672 - Train: 0.66% [12500/1902000] [6.6/1000.0] [batch_t 3.022 (2.195)] [data_t 2.420] [optim_t 0.602] [lr 0.005000] +2024-04-02 04:05:02,165 - Train: 0.66% [12600/1902000] [6.6/1000.0] [batch_t 3.468 (0.915)] [data_t 2.867] [optim_t 0.601] [lr 0.005000] +2024-04-02 04:07:19,103 - Train: 0.67% [12700/1902000] [6.7/1000.0] [batch_t 0.917 (1.369)] [data_t 0.315] [optim_t 0.602] [lr 0.005000] +2024-04-02 04:09:14,039 - Train: 0.67% [12800/1902000] [6.7/1000.0] [batch_t 0.605 (1.149)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:10:20,324 - Train: 0.68% [12900/1902000] [6.8/1000.0] [batch_t 0.604 (0.663)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 04:11:53,055 - Train: 0.68% [13000/1902000] [6.8/1000.0] [batch_t 0.605 (0.927)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:13:32,888 - Train: 0.69% [13100/1902000] [6.9/1000.0] [batch_t 0.604 (0.998)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 04:14:49,114 - Train: 0.69% [13200/1902000] [6.9/1000.0] [batch_t 0.605 (0.762)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 04:15:55,915 - Train: 0.70% [13300/1902000] [7.0/1000.0] [batch_t 0.605 (0.668)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 04:16:04,381 - ==> Total time: 3:15:43 Eta: 19 days, 6:45:56 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 04:17:33,201 - Train: 0.70% [13400/1902000] [7.0/1000.0] [batch_t 0.605 (0.998)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 04:18:51,646 - Train: 0.71% [13500/1902000] [7.1/1000.0] [batch_t 0.605 (0.784)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:19:56,932 - Train: 0.72% [13600/1902000] [7.2/1000.0] [batch_t 0.603 (0.653)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 04:21:02,587 - Train: 0.72% [13700/1902000] [7.2/1000.0] [batch_t 0.605 (0.656)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 04:22:25,148 - Train: 0.73% [13800/1902000] [7.3/1000.0] [batch_t 0.602 (0.825)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 04:23:49,756 - Train: 0.73% [13900/1902000] [7.3/1000.0] [batch_t 0.606 (0.845)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:25:27,102 - Train: 0.74% [14000/1902000] [7.4/1000.0] [batch_t 0.605 (0.973)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 04:26:43,196 - Train: 0.74% [14100/1902000] [7.4/1000.0] [batch_t 0.606 (0.761)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:27:53,089 - Train: 0.75% [14200/1902000] [7.5/1000.0] [batch_t 0.602 (0.699)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 04:28:56,648 - Train: 0.75% [14300/1902000] [7.5/1000.0] [batch_t 0.605 (0.635)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 04:30:01,041 - Train: 0.76% [14400/1902000] [7.6/1000.0] [batch_t 0.607 (0.644)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 04:31:24,174 - Train: 0.76% [14500/1902000] [7.6/1000.0] [batch_t 0.605 (0.831)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 04:32:33,288 - Train: 0.77% [14600/1902000] [7.7/1000.0] [batch_t 0.604 (0.691)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 04:33:45,615 - Train: 0.77% [14700/1902000] [7.7/1000.0] [batch_t 0.605 (0.723)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 04:34:46,547 - Train: 0.78% [14800/1902000] [7.8/1000.0] [batch_t 0.607 (0.609)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 04:36:11,333 - Train: 0.78% [14900/1902000] [7.8/1000.0] [batch_t 0.606 (0.848)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:37:11,797 - Train: 0.79% [15000/1902000] [7.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 04:38:13,370 - Train: 0.79% [15100/1902000] [7.9/1000.0] [batch_t 0.604 (0.616)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 04:39:17,804 - Train: 0.80% [15200/1902000] [8.0/1000.0] [batch_t 1.599 (0.644)] [data_t 0.996] [optim_t 0.603] [lr 0.005000] +2024-04-02 04:39:28,834 - ==> Total time: 3:39:08 Eta: 18 days, 20:53:15 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 04:40:38,785 - Train: 0.80% [15300/1902000] [8.0/1000.0] [batch_t 0.606 (0.807)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:41:55,731 - Train: 0.81% [15400/1902000] [8.1/1000.0] [batch_t 0.611 (0.769)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-02 04:43:02,898 - Train: 0.81% [15500/1902000] [8.1/1000.0] [batch_t 0.605 (0.672)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 04:44:28,597 - Train: 0.82% [15600/1902000] [8.2/1000.0] [batch_t 1.168 (0.857)] [data_t 0.561] [optim_t 0.607] [lr 0.005000] +2024-04-02 04:45:34,687 - Train: 0.83% [15700/1902000] [8.3/1000.0] [batch_t 0.608 (0.661)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 04:46:38,611 - Train: 0.83% [15800/1902000] [8.3/1000.0] [batch_t 0.605 (0.639)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 04:47:41,118 - Train: 0.84% [15900/1902000] [8.4/1000.0] [batch_t 0.606 (0.625)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:48:41,560 - Train: 0.84% [16000/1902000] [8.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:49:46,372 - Train: 0.85% [16100/1902000] [8.5/1000.0] [batch_t 0.606 (0.648)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:50:50,413 - Train: 0.85% [16200/1902000] [8.5/1000.0] [batch_t 0.606 (0.640)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:52:02,166 - Train: 0.86% [16300/1902000] [8.6/1000.0] [batch_t 0.605 (0.717)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 04:53:12,151 - Train: 0.86% [16400/1902000] [8.6/1000.0] [batch_t 0.603 (0.700)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 04:54:15,643 - Train: 0.87% [16500/1902000] [8.7/1000.0] [batch_t 0.606 (0.635)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 04:55:20,254 - Train: 0.87% [16600/1902000] [8.7/1000.0] [batch_t 0.606 (0.646)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:56:27,427 - Train: 0.88% [16700/1902000] [8.8/1000.0] [batch_t 0.606 (0.671)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 04:57:28,516 - Train: 0.88% [16800/1902000] [8.8/1000.0] [batch_t 0.605 (0.611)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 04:58:31,681 - Train: 0.89% [16900/1902000] [8.9/1000.0] [batch_t 0.605 (0.632)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 04:59:33,842 - Train: 0.89% [17000/1902000] [8.9/1000.0] [batch_t 0.602 (0.622)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 05:00:37,471 - Train: 0.90% [17100/1902000] [9.0/1000.0] [batch_t 0.604 (0.636)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 05:00:48,351 - ==> Total time: 4:00:27 Eta: 18 days, 9:17:48 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 05:01:42,396 - Train: 0.90% [17200/1902000] [9.0/1000.0] [batch_t 0.605 (0.640)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:02:44,822 - Train: 0.91% [17300/1902000] [9.1/1000.0] [batch_t 0.604 (0.624)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:03:46,778 - Train: 0.91% [17400/1902000] [9.1/1000.0] [batch_t 0.605 (0.619)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:04:52,451 - Train: 0.92% [17500/1902000] [9.2/1000.0] [batch_t 0.604 (0.657)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 05:05:56,354 - Train: 0.93% [17600/1902000] [9.3/1000.0] [batch_t 0.605 (0.639)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:06:56,898 - Train: 0.93% [17700/1902000] [9.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:07:58,764 - Train: 0.94% [17800/1902000] [9.4/1000.0] [batch_t 0.604 (0.619)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 05:09:02,493 - Train: 0.94% [17900/1902000] [9.4/1000.0] [batch_t 0.605 (0.637)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:10:04,096 - Train: 0.95% [18000/1902000] [9.5/1000.0] [batch_t 0.606 (0.616)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:11:11,040 - Train: 0.95% [18100/1902000] [9.5/1000.0] [batch_t 0.605 (0.669)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:12:13,266 - Train: 0.96% [18200/1902000] [9.6/1000.0] [batch_t 0.608 (0.622)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 05:13:16,851 - Train: 0.96% [18300/1902000] [9.6/1000.0] [batch_t 0.605 (0.635)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:14:22,842 - Train: 0.97% [18400/1902000] [9.7/1000.0] [batch_t 0.605 (0.660)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:15:27,802 - Train: 0.97% [18500/1902000] [9.7/1000.0] [batch_t 0.604 (0.650)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 05:16:29,640 - Train: 0.98% [18600/1902000] [9.8/1000.0] [batch_t 0.606 (0.618)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:17:31,145 - Train: 0.98% [18700/1902000] [9.8/1000.0] [batch_t 0.609 (0.615)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-02 05:18:34,548 - Train: 0.99% [18800/1902000] [9.9/1000.0] [batch_t 0.606 (0.634)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:19:40,085 - Train: 0.99% [18900/1902000] [9.9/1000.0] [batch_t 0.605 (0.655)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:20:40,734 - Train: 1.00% [19000/1902000] [10.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:20:52,822 - ==> Total time: 4:20:32 Eta: 17 days, 21:53:21 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 05:21:44,153 - Train: 1.00% [19100/1902000] [10.0/1000.0] [batch_t 0.603 (0.624)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 05:22:48,108 - Train: 1.01% [19200/1902000] [10.1/1000.0] [batch_t 0.604 (0.639)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:23:48,538 - Train: 1.01% [19300/1902000] [10.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:24:50,756 - Train: 1.02% [19400/1902000] [10.2/1000.0] [batch_t 0.606 (0.622)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:25:51,687 - Train: 1.03% [19500/1902000] [10.3/1000.0] [batch_t 0.607 (0.609)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 05:26:54,742 - Train: 1.03% [19600/1902000] [10.3/1000.0] [batch_t 0.605 (0.630)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:27:56,338 - Train: 1.04% [19700/1902000] [10.4/1000.0] [batch_t 0.605 (0.616)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:29:02,841 - Train: 1.04% [19800/1902000] [10.4/1000.0] [batch_t 0.604 (0.665)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:30:05,121 - Train: 1.05% [19900/1902000] [10.5/1000.0] [batch_t 0.606 (0.623)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:31:08,644 - Train: 1.05% [20000/1902000] [10.5/1000.0] [batch_t 0.606 (0.635)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:32:17,252 - Train: 1.06% [20100/1902000] [10.6/1000.0] [batch_t 1.278 (0.686)] [data_t 0.673] [optim_t 0.605] [lr 0.005000] +2024-04-02 05:33:17,781 - Train: 1.06% [20200/1902000] [10.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:34:20,117 - Train: 1.07% [20300/1902000] [10.7/1000.0] [batch_t 0.605 (0.623)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:35:21,667 - Train: 1.07% [20400/1902000] [10.7/1000.0] [batch_t 0.605 (0.615)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:36:22,663 - Train: 1.08% [20500/1902000] [10.8/1000.0] [batch_t 0.606 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:37:24,764 - Train: 1.08% [20600/1902000] [10.8/1000.0] [batch_t 0.608 (0.621)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 05:38:26,296 - Train: 1.09% [20700/1902000] [10.9/1000.0] [batch_t 0.606 (0.615)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:39:29,874 - Train: 1.09% [20800/1902000] [10.9/1000.0] [batch_t 0.605 (0.636)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:40:34,745 - Train: 1.10% [20900/1902000] [11.0/1000.0] [batch_t 0.606 (0.649)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:40:48,050 - ==> Total time: 4:40:27 Eta: 17 days, 12:15:51 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 05:41:38,612 - Train: 1.10% [21000/1902000] [11.0/1000.0] [batch_t 0.604 (0.628)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 05:42:39,275 - Train: 1.11% [21100/1902000] [11.1/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 05:43:41,036 - Train: 1.11% [21200/1902000] [11.1/1000.0] [batch_t 0.607 (0.618)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 05:44:43,954 - Train: 1.12% [21300/1902000] [11.2/1000.0] [batch_t 0.604 (0.627)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 05:45:45,184 - Train: 1.13% [21400/1902000] [11.3/1000.0] [batch_t 0.605 (0.612)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:46:51,485 - Train: 1.13% [21500/1902000] [11.3/1000.0] [batch_t 0.605 (0.663)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:47:52,198 - Train: 1.14% [21600/1902000] [11.4/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:48:52,618 - Train: 1.14% [21700/1902000] [11.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:49:56,616 - Train: 1.15% [21800/1902000] [11.5/1000.0] [batch_t 0.605 (0.640)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:50:57,025 - Train: 1.15% [21900/1902000] [11.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 05:51:58,374 - Train: 1.16% [22000/1902000] [11.6/1000.0] [batch_t 0.604 (0.613)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 05:52:59,810 - Train: 1.16% [22100/1902000] [11.6/1000.0] [batch_t 0.605 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 05:54:35,375 - Train: 1.17% [22200/1902000] [11.7/1000.0] [batch_t 1.233 (0.956)] [data_t 0.631] [optim_t 0.602] [lr 0.005000] +2024-04-02 05:55:42,394 - Train: 1.17% [22300/1902000] [11.7/1000.0] [batch_t 0.610 (0.670)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-02 05:56:45,299 - Train: 1.18% [22400/1902000] [11.8/1000.0] [batch_t 0.605 (0.629)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 05:58:01,094 - Train: 1.18% [22500/1902000] [11.8/1000.0] [batch_t 0.608 (0.758)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 05:59:02,377 - Train: 1.19% [22600/1902000] [11.9/1000.0] [batch_t 0.604 (0.613)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:00:04,263 - Train: 1.19% [22700/1902000] [11.9/1000.0] [batch_t 0.603 (0.619)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 06:01:06,199 - Train: 1.20% [22800/1902000] [12.0/1000.0] [batch_t 0.607 (0.619)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 06:01:23,331 - ==> Total time: 5:01:02 Eta: 17 days, 5:06:14 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 06:02:15,210 - Train: 1.20% [22900/1902000] [12.0/1000.0] [batch_t 0.604 (0.654)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:03:17,084 - Train: 1.21% [23000/1902000] [12.1/1000.0] [batch_t 0.604 (0.619)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 06:04:22,824 - Train: 1.21% [23100/1902000] [12.1/1000.0] [batch_t 0.605 (0.657)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:05:23,246 - Train: 1.22% [23200/1902000] [12.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:06:26,814 - Train: 1.23% [23300/1902000] [12.3/1000.0] [batch_t 0.605 (0.636)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:07:30,654 - Train: 1.23% [23400/1902000] [12.3/1000.0] [batch_t 0.609 (0.638)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-02 06:08:33,536 - Train: 1.24% [23500/1902000] [12.4/1000.0] [batch_t 0.604 (0.629)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 06:09:34,456 - Train: 1.24% [23600/1902000] [12.4/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:10:35,755 - Train: 1.25% [23700/1902000] [12.5/1000.0] [batch_t 0.606 (0.613)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:11:38,149 - Train: 1.25% [23800/1902000] [12.5/1000.0] [batch_t 0.605 (0.624)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:12:41,656 - Train: 1.26% [23900/1902000] [12.6/1000.0] [batch_t 0.603 (0.635)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 06:13:43,677 - Train: 1.26% [24000/1902000] [12.6/1000.0] [batch_t 0.609 (0.620)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-02 06:14:44,348 - Train: 1.27% [24100/1902000] [12.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:15:47,610 - Train: 1.27% [24200/1902000] [12.7/1000.0] [batch_t 0.604 (0.633)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 06:16:48,112 - Train: 1.28% [24300/1902000] [12.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:17:53,560 - Train: 1.28% [24400/1902000] [12.8/1000.0] [batch_t 0.605 (0.654)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:18:54,321 - Train: 1.29% [24500/1902000] [12.9/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:20:08,923 - Train: 1.29% [24600/1902000] [12.9/1000.0] [batch_t 0.605 (0.746)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:21:10,821 - Train: 1.30% [24700/1902000] [13.0/1000.0] [batch_t 0.605 (0.619)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:21:26,555 - ==> Total time: 5:21:06 Eta: 16 days, 22:18:59 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 06:22:14,811 - Train: 1.30% [24800/1902000] [13.0/1000.0] [batch_t 0.605 (0.630)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:23:16,107 - Train: 1.31% [24900/1902000] [13.1/1000.0] [batch_t 0.605 (0.613)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:24:16,500 - Train: 1.31% [25000/1902000] [13.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:25:18,919 - Train: 1.32% [25100/1902000] [13.2/1000.0] [batch_t 0.606 (0.624)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:26:20,688 - Train: 1.32% [25200/1902000] [13.2/1000.0] [batch_t 0.605 (0.618)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:27:24,764 - Train: 1.33% [25300/1902000] [13.3/1000.0] [batch_t 0.606 (0.641)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:28:26,686 - Train: 1.34% [25400/1902000] [13.4/1000.0] [batch_t 0.609 (0.619)] [data_t 0.004] [optim_t 0.605] [lr 0.005000] +2024-04-02 06:29:29,282 - Train: 1.34% [25500/1902000] [13.4/1000.0] [batch_t 0.604 (0.626)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:30:31,910 - Train: 1.35% [25600/1902000] [13.5/1000.0] [batch_t 0.605 (0.626)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:31:34,482 - Train: 1.35% [25700/1902000] [13.5/1000.0] [batch_t 0.606 (0.626)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:32:35,645 - Train: 1.36% [25800/1902000] [13.6/1000.0] [batch_t 0.605 (0.612)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:33:36,740 - Train: 1.36% [25900/1902000] [13.6/1000.0] [batch_t 0.605 (0.611)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:34:38,077 - Train: 1.37% [26000/1902000] [13.7/1000.0] [batch_t 0.604 (0.613)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 06:35:40,865 - Train: 1.37% [26100/1902000] [13.7/1000.0] [batch_t 0.607 (0.628)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 06:36:43,846 - Train: 1.38% [26200/1902000] [13.8/1000.0] [batch_t 0.605 (0.630)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:37:46,237 - Train: 1.38% [26300/1902000] [13.8/1000.0] [batch_t 0.605 (0.624)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:38:49,631 - Train: 1.39% [26400/1902000] [13.9/1000.0] [batch_t 0.605 (0.634)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:39:52,306 - Train: 1.39% [26500/1902000] [13.9/1000.0] [batch_t 0.606 (0.627)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:40:53,510 - Train: 1.40% [26600/1902000] [14.0/1000.0] [batch_t 0.606 (0.612)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:41:12,009 - ==> Total time: 5:40:51 Eta: 16 days, 16:06:11 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 06:41:58,169 - Train: 1.40% [26700/1902000] [14.0/1000.0] [batch_t 0.604 (0.618)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:43:02,201 - Train: 1.41% [26800/1902000] [14.1/1000.0] [batch_t 0.605 (0.640)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:44:02,656 - Train: 1.41% [26900/1902000] [14.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 06:45:05,333 - Train: 1.42% [27000/1902000] [14.2/1000.0] [batch_t 0.604 (0.627)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 06:46:05,757 - Train: 1.42% [27100/1902000] [14.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:47:09,373 - Train: 1.43% [27200/1902000] [14.3/1000.0] [batch_t 1.959 (0.635)] [data_t 1.355] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:48:11,942 - Train: 1.44% [27300/1902000] [14.4/1000.0] [batch_t 0.605 (0.626)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:49:14,854 - Train: 1.44% [27400/1902000] [14.4/1000.0] [batch_t 0.604 (0.629)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 06:50:17,990 - Train: 1.45% [27500/1902000] [14.5/1000.0] [batch_t 0.604 (0.631)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:51:19,536 - Train: 1.45% [27600/1902000] [14.5/1000.0] [batch_t 0.605 (0.615)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:52:22,987 - Train: 1.46% [27700/1902000] [14.6/1000.0] [batch_t 0.606 (0.634)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:53:24,846 - Train: 1.46% [27800/1902000] [14.6/1000.0] [batch_t 0.604 (0.619)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 06:54:27,630 - Train: 1.47% [27900/1902000] [14.7/1000.0] [batch_t 0.604 (0.628)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:55:29,298 - Train: 1.47% [28000/1902000] [14.7/1000.0] [batch_t 0.605 (0.617)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:56:30,471 - Train: 1.48% [28100/1902000] [14.8/1000.0] [batch_t 0.604 (0.612)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 06:57:33,916 - Train: 1.48% [28200/1902000] [14.8/1000.0] [batch_t 0.605 (0.634)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 06:58:35,158 - Train: 1.49% [28300/1902000] [14.9/1000.0] [batch_t 0.605 (0.612)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 06:59:36,821 - Train: 1.49% [28400/1902000] [14.9/1000.0] [batch_t 0.604 (0.617)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 07:00:38,578 - Train: 1.50% [28500/1902000] [15.0/1000.0] [batch_t 0.605 (0.617)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 07:00:56,697 - ==> Total time: 6:00:36 Eta: 16 days, 10:39:38 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 07:01:42,954 - Train: 1.50% [28600/1902000] [15.0/1000.0] [batch_t 0.606 (0.642)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-02 07:02:49,060 - Train: 1.51% [28700/1902000] [15.1/1000.0] [batch_t 0.604 (0.661)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 07:03:49,837 - Train: 1.51% [28800/1902000] [15.1/1000.0] [batch_t 0.603 (0.608)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 07:04:53,383 - Train: 1.52% [28900/1902000] [15.2/1000.0] [batch_t 0.608 (0.635)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-02 07:05:54,426 - Train: 1.52% [29000/1902000] [15.2/1000.0] [batch_t 0.603 (0.610)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 07:06:56,816 - Train: 1.53% [29100/1902000] [15.3/1000.0] [batch_t 0.604 (0.624)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 07:08:04,923 - Train: 1.54% [29200/1902000] [15.4/1000.0] [batch_t 0.605 (0.681)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:09:10,161 - Train: 1.54% [29300/1902000] [15.4/1000.0] [batch_t 0.606 (0.652)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 07:10:13,084 - Train: 1.55% [29400/1902000] [15.5/1000.0] [batch_t 0.603 (0.629)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 07:11:14,917 - Train: 1.55% [29500/1902000] [15.5/1000.0] [batch_t 0.604 (0.618)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 07:12:19,623 - Train: 1.56% [29600/1902000] [15.6/1000.0] [batch_t 0.604 (0.647)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 07:13:30,579 - Train: 1.56% [29700/1902000] [15.6/1000.0] [batch_t 0.605 (0.709)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 07:14:33,592 - Train: 1.57% [29800/1902000] [15.7/1000.0] [batch_t 0.605 (0.630)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:15:36,728 - Train: 1.57% [29900/1902000] [15.7/1000.0] [batch_t 0.604 (0.631)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 07:16:39,406 - Train: 1.58% [30000/1902000] [15.8/1000.0] [batch_t 0.605 (0.627)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:17:45,417 - Train: 1.58% [30100/1902000] [15.8/1000.0] [batch_t 0.606 (0.660)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:18:50,409 - Train: 1.59% [30200/1902000] [15.9/1000.0] [batch_t 0.605 (0.650)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 07:19:55,633 - Train: 1.59% [30300/1902000] [15.9/1000.0] [batch_t 0.605 (0.651)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 07:20:56,734 - Train: 1.60% [30400/1902000] [16.0/1000.0] [batch_t 0.606 (0.611)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:21:33,130 - ==> Total time: 6:21:12 Eta: 16 days, 6:44:27 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 07:22:43,533 - Train: 1.60% [30500/1902000] [16.0/1000.0] [batch_t 0.604 (1.004)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 07:23:47,159 - Train: 1.61% [30600/1902000] [16.1/1000.0] [batch_t 0.605 (0.636)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 07:24:51,977 - Train: 1.61% [30700/1902000] [16.1/1000.0] [batch_t 0.603 (0.648)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 07:25:52,527 - Train: 1.62% [30800/1902000] [16.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 07:26:55,031 - Train: 1.62% [30900/1902000] [16.2/1000.0] [batch_t 0.605 (0.625)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:27:58,898 - Train: 1.63% [31000/1902000] [16.3/1000.0] [batch_t 0.605 (0.639)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 07:28:59,348 - Train: 1.64% [31100/1902000] [16.4/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-02 07:30:01,727 - Train: 1.64% [31200/1902000] [16.4/1000.0] [batch_t 0.605 (0.624)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 07:31:04,350 - Train: 1.65% [31300/1902000] [16.5/1000.0] [batch_t 0.606 (0.626)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:32:04,806 - Train: 1.65% [31400/1902000] [16.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:33:05,464 - Train: 1.66% [31500/1902000] [16.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 07:34:10,084 - Train: 1.66% [31600/1902000] [16.6/1000.0] [batch_t 0.604 (0.646)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 07:35:10,789 - Train: 1.67% [31700/1902000] [16.7/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 07:36:13,164 - Train: 1.67% [31800/1902000] [16.7/1000.0] [batch_t 0.607 (0.624)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 07:37:14,408 - Train: 1.68% [31900/1902000] [16.8/1000.0] [batch_t 0.604 (0.612)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 07:38:24,372 - Train: 1.68% [32000/1902000] [16.8/1000.0] [batch_t 0.605 (0.700)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:39:29,563 - Train: 1.69% [32100/1902000] [16.9/1000.0] [batch_t 0.606 (0.652)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:40:30,211 - Train: 1.69% [32200/1902000] [16.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 07:41:30,683 - Train: 1.70% [32300/1902000] [17.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:41:51,250 - ==> Total time: 6:41:30 Eta: 16 days, 2:56:53 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 07:42:33,924 - Train: 1.70% [32400/1902000] [17.0/1000.0] [batch_t 0.605 (0.621)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 07:43:35,796 - Train: 1.71% [32500/1902000] [17.1/1000.0] [batch_t 0.606 (0.619)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:44:41,314 - Train: 1.71% [32600/1902000] [17.1/1000.0] [batch_t 0.606 (0.655)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:45:46,591 - Train: 1.72% [32700/1902000] [17.2/1000.0] [batch_t 0.605 (0.653)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 07:46:48,008 - Train: 1.72% [32800/1902000] [17.2/1000.0] [batch_t 0.605 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:47:48,841 - Train: 1.73% [32900/1902000] [17.3/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 07:48:50,611 - Train: 1.74% [33000/1902000] [17.4/1000.0] [batch_t 0.606 (0.618)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:49:51,047 - Train: 1.74% [33100/1902000] [17.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 07:50:52,516 - Train: 1.75% [33200/1902000] [17.5/1000.0] [batch_t 0.605 (0.615)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 07:51:53,475 - Train: 1.75% [33300/1902000] [17.5/1000.0] [batch_t 0.604 (0.610)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 07:52:54,828 - Train: 1.76% [33400/1902000] [17.6/1000.0] [batch_t 0.606 (0.613)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:53:57,443 - Train: 1.76% [33500/1902000] [17.6/1000.0] [batch_t 0.604 (0.626)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 07:55:02,018 - Train: 1.77% [33600/1902000] [17.7/1000.0] [batch_t 0.605 (0.646)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 07:56:04,047 - Train: 1.77% [33700/1902000] [17.7/1000.0] [batch_t 0.606 (0.620)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:57:08,265 - Train: 1.78% [33800/1902000] [17.8/1000.0] [batch_t 1.028 (0.642)] [data_t 0.426] [optim_t 0.602] [lr 0.005000] +2024-04-02 07:58:14,473 - Train: 1.78% [33900/1902000] [17.8/1000.0] [batch_t 0.606 (0.662)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 07:59:19,934 - Train: 1.79% [34000/1902000] [17.9/1000.0] [batch_t 0.604 (0.655)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:00:20,352 - Train: 1.79% [34100/1902000] [17.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:01:21,325 - Train: 1.80% [34200/1902000] [18.0/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:01:43,082 - ==> Total time: 7:01:22 Eta: 15 days, 23:08:26 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 08:02:26,213 - Train: 1.80% [34300/1902000] [18.0/1000.0] [batch_t 0.605 (0.645)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:03:31,142 - Train: 1.81% [34400/1902000] [18.1/1000.0] [batch_t 0.606 (0.649)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:04:32,712 - Train: 1.81% [34500/1902000] [18.1/1000.0] [batch_t 0.608 (0.616)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 08:05:34,839 - Train: 1.82% [34600/1902000] [18.2/1000.0] [batch_t 0.605 (0.621)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:06:35,984 - Train: 1.82% [34700/1902000] [18.2/1000.0] [batch_t 0.604 (0.611)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 08:07:36,545 - Train: 1.83% [34800/1902000] [18.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 08:08:40,813 - Train: 1.83% [34900/1902000] [18.3/1000.0] [batch_t 0.606 (0.643)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:09:41,283 - Train: 1.84% [35000/1902000] [18.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:10:41,713 - Train: 1.85% [35100/1902000] [18.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 08:11:42,638 - Train: 1.85% [35200/1902000] [18.5/1000.0] [batch_t 0.607 (0.609)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 08:12:47,108 - Train: 1.86% [35300/1902000] [18.6/1000.0] [batch_t 0.605 (0.645)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:13:49,863 - Train: 1.86% [35400/1902000] [18.6/1000.0] [batch_t 0.603 (0.627)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 08:14:53,276 - Train: 1.87% [35500/1902000] [18.7/1000.0] [batch_t 0.606 (0.634)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:15:54,269 - Train: 1.87% [35600/1902000] [18.7/1000.0] [batch_t 0.603 (0.610)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 08:16:58,767 - Train: 1.88% [35700/1902000] [18.8/1000.0] [batch_t 0.604 (0.645)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 08:18:00,487 - Train: 1.88% [35800/1902000] [18.8/1000.0] [batch_t 0.605 (0.617)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:19:03,772 - Train: 1.89% [35900/1902000] [18.9/1000.0] [batch_t 0.606 (0.633)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:20:04,373 - Train: 1.89% [36000/1902000] [18.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:21:04,820 - Train: 1.90% [36100/1902000] [19.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:21:30,048 - ==> Total time: 7:21:09 Eta: 15 days, 19:37:45 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 08:22:09,890 - Train: 1.90% [36200/1902000] [19.0/1000.0] [batch_t 0.604 (0.618)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 08:23:10,422 - Train: 1.91% [36300/1902000] [19.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:24:12,326 - Train: 1.91% [36400/1902000] [19.1/1000.0] [batch_t 0.607 (0.619)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:25:13,765 - Train: 1.92% [36500/1902000] [19.2/1000.0] [batch_t 0.607 (0.613)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:26:15,707 - Train: 1.92% [36600/1902000] [19.2/1000.0] [batch_t 0.607 (0.619)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-02 08:27:16,305 - Train: 1.93% [36700/1902000] [19.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 08:28:23,481 - Train: 1.93% [36800/1902000] [19.3/1000.0] [batch_t 0.606 (0.672)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:29:24,031 - Train: 1.94% [36900/1902000] [19.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:30:25,407 - Train: 1.95% [37000/1902000] [19.5/1000.0] [batch_t 0.606 (0.614)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:31:26,041 - Train: 1.95% [37100/1902000] [19.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:32:28,482 - Train: 1.96% [37200/1902000] [19.6/1000.0] [batch_t 0.607 (0.624)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-02 08:33:30,571 - Train: 1.96% [37300/1902000] [19.6/1000.0] [batch_t 0.609 (0.621)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-02 08:34:31,112 - Train: 1.97% [37400/1902000] [19.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:35:32,226 - Train: 1.97% [37500/1902000] [19.7/1000.0] [batch_t 0.606 (0.611)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:36:33,520 - Train: 1.98% [37600/1902000] [19.8/1000.0] [batch_t 0.606 (0.613)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:37:34,040 - Train: 1.98% [37700/1902000] [19.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 08:38:34,866 - Train: 1.99% [37800/1902000] [19.9/1000.0] [batch_t 0.606 (0.608)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:39:40,683 - Train: 1.99% [37900/1902000] [19.9/1000.0] [batch_t 0.605 (0.658)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:40:44,107 - Train: 2.00% [38000/1902000] [20.0/1000.0] [batch_t 0.607 (0.634)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:41:10,142 - ==> Total time: 7:40:49 Eta: 15 days, 16:20:33 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 08:41:49,434 - Train: 2.00% [38100/1902000] [20.0/1000.0] [batch_t 0.605 (0.629)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:42:51,147 - Train: 2.01% [38200/1902000] [20.1/1000.0] [batch_t 0.608 (0.617)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-02 08:43:51,716 - Train: 2.01% [38300/1902000] [20.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:44:53,377 - Train: 2.02% [38400/1902000] [20.2/1000.0] [batch_t 0.606 (0.617)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:45:53,905 - Train: 2.02% [38500/1902000] [20.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 08:46:54,471 - Train: 2.03% [38600/1902000] [20.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:47:54,982 - Train: 2.03% [38700/1902000] [20.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 08:48:56,343 - Train: 2.04% [38800/1902000] [20.4/1000.0] [batch_t 0.606 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:49:57,930 - Train: 2.05% [38900/1902000] [20.5/1000.0] [batch_t 0.604 (0.616)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 08:50:59,571 - Train: 2.05% [39000/1902000] [20.5/1000.0] [batch_t 0.604 (0.616)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-02 08:52:00,717 - Train: 2.06% [39100/1902000] [20.6/1000.0] [batch_t 0.607 (0.611)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:53:01,188 - Train: 2.06% [39200/1902000] [20.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:54:33,047 - Train: 2.07% [39300/1902000] [20.7/1000.0] [batch_t 0.607 (0.918)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 08:55:33,442 - Train: 2.07% [39400/1902000] [20.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:56:35,026 - Train: 2.08% [39500/1902000] [20.8/1000.0] [batch_t 0.605 (0.616)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:57:37,941 - Train: 2.08% [39600/1902000] [20.8/1000.0] [batch_t 0.606 (0.629)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 08:58:39,306 - Train: 2.09% [39700/1902000] [20.9/1000.0] [batch_t 0.605 (0.614)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 08:59:40,992 - Train: 2.09% [39800/1902000] [20.9/1000.0] [batch_t 0.605 (0.617)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:00:44,548 - Train: 2.10% [39900/1902000] [21.0/1000.0] [batch_t 0.605 (0.635)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:01:09,910 - ==> Total time: 8:00:49 Eta: 15 days, 13:35:32 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 09:01:47,139 - Train: 2.10% [40000/1902000] [21.0/1000.0] [batch_t 0.603 (0.616)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 09:02:50,096 - Train: 2.11% [40100/1902000] [21.1/1000.0] [batch_t 0.605 (0.629)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:03:54,397 - Train: 2.11% [40200/1902000] [21.1/1000.0] [batch_t 0.604 (0.643)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 09:05:13,062 - Train: 2.12% [40300/1902000] [21.2/1000.0] [batch_t 0.604 (0.787)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:06:13,706 - Train: 2.12% [40400/1902000] [21.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 09:07:15,956 - Train: 2.13% [40500/1902000] [21.3/1000.0] [batch_t 0.605 (0.622)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:08:16,356 - Train: 2.13% [40600/1902000] [21.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:09:18,674 - Train: 2.14% [40700/1902000] [21.4/1000.0] [batch_t 0.604 (0.623)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 09:10:21,367 - Train: 2.15% [40800/1902000] [21.5/1000.0] [batch_t 0.604 (0.627)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:11:24,708 - Train: 2.15% [40900/1902000] [21.5/1000.0] [batch_t 0.605 (0.633)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 09:12:26,555 - Train: 2.16% [41000/1902000] [21.6/1000.0] [batch_t 0.605 (0.618)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:13:28,800 - Train: 2.16% [41100/1902000] [21.6/1000.0] [batch_t 0.605 (0.622)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 09:14:30,736 - Train: 2.17% [41200/1902000] [21.7/1000.0] [batch_t 0.604 (0.619)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 09:15:32,245 - Train: 2.17% [41300/1902000] [21.7/1000.0] [batch_t 0.605 (0.615)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 09:16:32,627 - Train: 2.18% [41400/1902000] [21.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:17:33,017 - Train: 2.18% [41500/1902000] [21.8/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 09:18:33,770 - Train: 2.19% [41600/1902000] [21.9/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 09:19:34,171 - Train: 2.19% [41700/1902000] [21.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 09:21:02,244 - Train: 2.20% [41800/1902000] [22.0/1000.0] [batch_t 0.604 (0.881)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:21:28,858 - ==> Total time: 8:21:08 Eta: 15 days, 11:17:56 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 09:22:04,879 - Train: 2.20% [41900/1902000] [22.0/1000.0] [batch_t 0.606 (0.616)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 09:23:06,566 - Train: 2.21% [42000/1902000] [22.1/1000.0] [batch_t 0.603 (0.617)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 09:24:10,514 - Train: 2.21% [42100/1902000] [22.1/1000.0] [batch_t 0.604 (0.639)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 09:25:11,307 - Train: 2.22% [42200/1902000] [22.2/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:26:12,166 - Train: 2.22% [42300/1902000] [22.2/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 09:27:12,635 - Train: 2.23% [42400/1902000] [22.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 09:28:13,045 - Train: 2.23% [42500/1902000] [22.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:29:13,423 - Train: 2.24% [42600/1902000] [22.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 09:30:13,828 - Train: 2.25% [42700/1902000] [22.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 09:31:14,375 - Train: 2.25% [42800/1902000] [22.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:32:14,802 - Train: 2.26% [42900/1902000] [22.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:33:15,210 - Train: 2.26% [43000/1902000] [22.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:34:15,624 - Train: 2.27% [43100/1902000] [22.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:35:17,196 - Train: 2.27% [43200/1902000] [22.7/1000.0] [batch_t 0.605 (0.616)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:36:19,049 - Train: 2.28% [43300/1902000] [22.8/1000.0] [batch_t 0.608 (0.618)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 09:37:38,469 - Train: 2.28% [43400/1902000] [22.8/1000.0] [batch_t 0.604 (0.794)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:38:49,507 - Train: 2.29% [43500/1902000] [22.9/1000.0] [batch_t 0.605 (0.710)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:39:50,444 - Train: 2.29% [43600/1902000] [22.9/1000.0] [batch_t 0.604 (0.609)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 09:40:51,648 - Train: 2.30% [43700/1902000] [23.0/1000.0] [batch_t 0.604 (0.612)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:41:21,216 - ==> Total time: 8:41:00 Eta: 15 days, 8:51:41 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 09:41:55,624 - Train: 2.30% [43800/1902000] [23.0/1000.0] [batch_t 0.605 (0.612)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:42:59,155 - Train: 2.31% [43900/1902000] [23.1/1000.0] [batch_t 0.603 (0.635)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 09:43:59,741 - Train: 2.31% [44000/1902000] [23.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 09:45:00,364 - Train: 2.32% [44100/1902000] [23.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:46:03,017 - Train: 2.32% [44200/1902000] [23.2/1000.0] [batch_t 0.603 (0.626)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 09:47:03,435 - Train: 2.33% [44300/1902000] [23.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:48:04,979 - Train: 2.33% [44400/1902000] [23.3/1000.0] [batch_t 0.606 (0.615)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 09:49:05,547 - Train: 2.34% [44500/1902000] [23.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:50:06,196 - Train: 2.34% [44600/1902000] [23.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-02 09:51:10,302 - Train: 2.35% [44700/1902000] [23.5/1000.0] [batch_t 0.603 (0.641)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 09:52:12,514 - Train: 2.36% [44800/1902000] [23.6/1000.0] [batch_t 0.606 (0.622)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 09:53:12,988 - Train: 2.36% [44900/1902000] [23.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 09:54:23,258 - Train: 2.37% [45000/1902000] [23.7/1000.0] [batch_t 0.606 (0.703)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 09:55:43,939 - Train: 2.37% [45100/1902000] [23.7/1000.0] [batch_t 4.931 (0.807)] [data_t 4.330] [optim_t 0.600] [lr 0.005000] +2024-04-02 09:59:44,242 - Train: 2.38% [45200/1902000] [23.8/1000.0] [batch_t 1.069 (2.403)] [data_t 0.467] [optim_t 0.602] [lr 0.005000] +2024-04-02 10:06:41,502 - Train: 2.38% [45300/1902000] [23.8/1000.0] [batch_t 1.413 (4.173)] [data_t 0.002] [optim_t 1.411] [lr 0.005000] +2024-04-02 10:13:42,921 - Train: 2.39% [45400/1902000] [23.9/1000.0] [batch_t 6.237 (4.214)] [data_t 4.829] [optim_t 1.408] [lr 0.005000] +2024-04-02 10:23:45,947 - Train: 2.39% [45500/1902000] [23.9/1000.0] [batch_t 5.614 (6.030)] [data_t 4.206] [optim_t 1.408] [lr 0.005000] +2024-04-02 10:34:41,297 - Train: 2.40% [45600/1902000] [24.0/1000.0] [batch_t 1.430 (6.553)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-02 10:36:04,108 - ==> Total time: 9:35:43 Eta: 16 days, 6:12:54 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 10:39:31,750 - Train: 2.40% [45700/1902000] [24.0/1000.0] [batch_t 1.421 (3.918)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-02 10:41:54,879 - Train: 2.41% [45800/1902000] [24.1/1000.0] [batch_t 1.425 (1.431)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-02 10:44:17,447 - Train: 2.41% [45900/1902000] [24.1/1000.0] [batch_t 1.428 (1.426)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-02 10:46:39,818 - Train: 2.42% [46000/1902000] [24.2/1000.0] [batch_t 1.416 (1.424)] [data_t 0.003] [optim_t 1.413] [lr 0.005000] +2024-04-02 10:49:02,246 - Train: 2.42% [46100/1902000] [24.2/1000.0] [batch_t 1.429 (1.424)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-02 10:51:31,254 - Train: 2.43% [46200/1902000] [24.3/1000.0] [batch_t 1.423 (1.490)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-02 10:54:10,499 - Train: 2.43% [46300/1902000] [24.3/1000.0] [batch_t 1.434 (1.592)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-02 10:56:32,944 - Train: 2.44% [46400/1902000] [24.4/1000.0] [batch_t 1.426 (1.424)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-02 10:58:56,544 - Train: 2.44% [46500/1902000] [24.4/1000.0] [batch_t 1.424 (1.436)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-02 11:03:08,288 - Train: 2.45% [46600/1902000] [24.5/1000.0] [batch_t 4.303 (2.517)] [data_t 2.881] [optim_t 1.423] [lr 0.005000] +2024-04-02 11:05:36,991 - Train: 2.46% [46700/1902000] [24.6/1000.0] [batch_t 1.420 (1.487)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-02 11:08:37,333 - Train: 2.46% [46800/1902000] [24.6/1000.0] [batch_t 0.604 (1.803)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 11:09:43,780 - Train: 2.47% [46900/1902000] [24.7/1000.0] [batch_t 0.605 (0.664)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 11:11:18,173 - Train: 2.47% [47000/1902000] [24.7/1000.0] [batch_t 0.606 (0.944)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 11:12:21,333 - Train: 2.48% [47100/1902000] [24.8/1000.0] [batch_t 0.605 (0.631)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 11:13:21,981 - Train: 2.48% [47200/1902000] [24.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 11:18:53,261 - Train: 2.49% [47300/1902000] [24.9/1000.0] [batch_t 0.884 (3.313)] [data_t 0.279] [optim_t 0.605] [lr 0.005000] +2024-04-02 11:21:02,141 - Train: 2.49% [47400/1902000] [24.9/1000.0] [batch_t 0.608 (1.289)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 11:22:20,143 - Train: 2.50% [47500/1902000] [25.0/1000.0] [batch_t 0.608 (0.780)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 11:22:50,332 - ==> Total time: 10:22:29 Eta: 16 days, 20:37:24 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 11:23:22,799 - Train: 2.50% [47600/1902000] [25.0/1000.0] [batch_t 0.605 (0.621)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 11:24:26,135 - Train: 2.51% [47700/1902000] [25.1/1000.0] [batch_t 0.605 (0.633)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 11:25:27,477 - Train: 2.51% [47800/1902000] [25.1/1000.0] [batch_t 0.610 (0.613)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-02 11:26:28,237 - Train: 2.52% [47900/1902000] [25.2/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 11:27:31,561 - Train: 2.52% [48000/1902000] [25.2/1000.0] [batch_t 0.605 (0.633)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 11:29:40,618 - Train: 2.53% [48100/1902000] [25.3/1000.0] [batch_t 0.604 (1.290)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 11:31:15,706 - Train: 2.53% [48200/1902000] [25.3/1000.0] [batch_t 0.609 (0.951)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-02 11:33:49,950 - Train: 2.54% [48300/1902000] [25.4/1000.0] [batch_t 0.604 (1.542)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 11:34:51,960 - Train: 2.54% [48400/1902000] [25.4/1000.0] [batch_t 0.605 (0.620)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 11:35:53,415 - Train: 2.55% [48500/1902000] [25.5/1000.0] [batch_t 0.604 (0.614)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 11:36:59,472 - Train: 2.56% [48600/1902000] [25.6/1000.0] [batch_t 0.603 (0.660)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 11:38:01,401 - Train: 2.56% [48700/1902000] [25.6/1000.0] [batch_t 0.606 (0.619)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 11:39:04,625 - Train: 2.57% [48800/1902000] [25.7/1000.0] [batch_t 0.604 (0.632)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 11:40:05,129 - Train: 2.57% [48900/1902000] [25.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 11:41:05,524 - Train: 2.58% [49000/1902000] [25.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 11:42:13,526 - Train: 2.58% [49100/1902000] [25.8/1000.0] [batch_t 0.604 (0.680)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 11:43:20,057 - Train: 2.59% [49200/1902000] [25.9/1000.0] [batch_t 0.604 (0.665)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 11:44:20,938 - Train: 2.59% [49300/1902000] [25.9/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 11:45:26,754 - Train: 2.60% [49400/1902000] [26.0/1000.0] [batch_t 0.602 (0.658)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 11:45:58,188 - ==> Total time: 10:45:37 Eta: 16 days, 19:06:14 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 11:46:31,010 - Train: 2.60% [49500/1902000] [26.0/1000.0] [batch_t 0.605 (0.647)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 11:47:34,574 - Train: 2.61% [49600/1902000] [26.1/1000.0] [batch_t 0.605 (0.636)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 11:48:35,991 - Train: 2.61% [49700/1902000] [26.1/1000.0] [batch_t 0.606 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 11:49:36,388 - Train: 2.62% [49800/1902000] [26.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 11:50:36,914 - Train: 2.62% [49900/1902000] [26.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 11:51:39,843 - Train: 2.63% [50000/1902000] [26.3/1000.0] [batch_t 0.606 (0.629)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 11:52:40,552 - Train: 2.63% [50100/1902000] [26.3/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 11:53:41,387 - Train: 2.64% [50200/1902000] [26.4/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 11:54:41,808 - Train: 2.64% [50300/1902000] [26.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 11:57:19,550 - Train: 2.65% [50400/1902000] [26.5/1000.0] [batch_t 0.604 (1.577)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 11:58:23,346 - Train: 2.66% [50500/1902000] [26.6/1000.0] [batch_t 0.605 (0.638)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 11:59:25,073 - Train: 2.66% [50600/1902000] [26.6/1000.0] [batch_t 0.605 (0.617)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:00:25,505 - Train: 2.67% [50700/1902000] [26.7/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 12:01:25,942 - Train: 2.67% [50800/1902000] [26.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:02:26,973 - Train: 2.68% [50900/1902000] [26.8/1000.0] [batch_t 0.610 (0.610)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-02 12:03:35,046 - Train: 2.68% [51000/1902000] [26.8/1000.0] [batch_t 0.604 (0.681)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 12:04:38,415 - Train: 2.69% [51100/1902000] [26.9/1000.0] [batch_t 0.605 (0.634)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:05:38,817 - Train: 2.69% [51200/1902000] [26.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 12:06:39,225 - Train: 2.70% [51300/1902000] [27.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:07:25,284 - ==> Total time: 11:07:04 Eta: 16 days, 16:39:35 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 12:07:58,166 - Train: 2.70% [51400/1902000] [27.0/1000.0] [batch_t 0.606 (0.669)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 12:10:53,378 - Train: 2.71% [51500/1902000] [27.1/1000.0] [batch_t 1.032 (1.752)] [data_t 0.429] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:12:03,390 - Train: 2.71% [51600/1902000] [27.1/1000.0] [batch_t 0.606 (0.700)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 12:13:15,418 - Train: 2.72% [51700/1902000] [27.2/1000.0] [batch_t 0.600 (0.720)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-02 12:14:15,818 - Train: 2.72% [51800/1902000] [27.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 12:15:17,423 - Train: 2.73% [51900/1902000] [27.3/1000.0] [batch_t 0.605 (0.616)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:16:18,527 - Train: 2.73% [52000/1902000] [27.3/1000.0] [batch_t 0.605 (0.611)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:17:18,949 - Train: 2.74% [52100/1902000] [27.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:18:19,495 - Train: 2.74% [52200/1902000] [27.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 12:19:20,688 - Train: 2.75% [52300/1902000] [27.5/1000.0] [batch_t 0.605 (0.612)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:20:21,951 - Train: 2.75% [52400/1902000] [27.5/1000.0] [batch_t 0.605 (0.613)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:21:22,378 - Train: 2.76% [52500/1902000] [27.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:22:22,832 - Train: 2.77% [52600/1902000] [27.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 12:23:24,289 - Train: 2.77% [52700/1902000] [27.7/1000.0] [batch_t 0.606 (0.614)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 12:24:24,812 - Train: 2.78% [52800/1902000] [27.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:25:25,350 - Train: 2.78% [52900/1902000] [27.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 12:26:26,270 - Train: 2.79% [53000/1902000] [27.9/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:27:26,703 - Train: 2.79% [53100/1902000] [27.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:28:27,576 - Train: 2.80% [53200/1902000] [28.0/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 12:29:01,397 - ==> Total time: 11:28:40 Eta: 16 days, 14:27:06 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 12:29:32,578 - Train: 2.80% [53300/1902000] [28.0/1000.0] [batch_t 0.605 (0.633)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:30:33,020 - Train: 2.81% [53400/1902000] [28.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:31:34,101 - Train: 2.81% [53500/1902000] [28.1/1000.0] [batch_t 0.606 (0.611)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 12:32:34,550 - Train: 2.82% [53600/1902000] [28.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 12:33:36,989 - Train: 2.82% [53700/1902000] [28.2/1000.0] [batch_t 0.605 (0.624)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 12:34:39,472 - Train: 2.83% [53800/1902000] [28.3/1000.0] [batch_t 0.977 (0.625)] [data_t 0.376] [optim_t 0.601] [lr 0.005000] +2024-04-02 12:35:48,465 - Train: 2.83% [53900/1902000] [28.3/1000.0] [batch_t 0.605 (0.690)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:36:53,598 - Train: 2.84% [54000/1902000] [28.4/1000.0] [batch_t 0.610 (0.651)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-02 12:37:55,570 - Train: 2.84% [54100/1902000] [28.4/1000.0] [batch_t 0.604 (0.620)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 12:38:56,919 - Train: 2.85% [54200/1902000] [28.5/1000.0] [batch_t 0.606 (0.613)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 12:39:57,472 - Train: 2.85% [54300/1902000] [28.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 12:41:00,477 - Train: 2.86% [54400/1902000] [28.6/1000.0] [batch_t 0.604 (0.630)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 12:42:00,900 - Train: 2.87% [54500/1902000] [28.7/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 12:43:01,326 - Train: 2.87% [54600/1902000] [28.7/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 12:44:01,869 - Train: 2.88% [54700/1902000] [28.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:45:02,309 - Train: 2.88% [54800/1902000] [28.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:46:03,105 - Train: 2.89% [54900/1902000] [28.9/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 12:47:04,839 - Train: 2.89% [55000/1902000] [28.9/1000.0] [batch_t 0.605 (0.617)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:48:07,155 - Train: 2.90% [55100/1902000] [29.0/1000.0] [batch_t 0.605 (0.623)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:48:42,708 - ==> Total time: 11:48:22 Eta: 16 days, 11:18:11 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 12:49:09,954 - Train: 2.90% [55200/1902000] [29.0/1000.0] [batch_t 0.602 (0.614)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 12:50:12,390 - Train: 2.91% [55300/1902000] [29.1/1000.0] [batch_t 0.605 (0.624)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:51:13,570 - Train: 2.91% [55400/1902000] [29.1/1000.0] [batch_t 0.606 (0.612)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 12:52:33,041 - Train: 2.92% [55500/1902000] [29.2/1000.0] [batch_t 0.605 (0.795)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:53:40,380 - Train: 2.92% [55600/1902000] [29.2/1000.0] [batch_t 0.611 (0.673)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-02 12:54:40,836 - Train: 2.93% [55700/1902000] [29.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:55:42,870 - Train: 2.93% [55800/1902000] [29.3/1000.0] [batch_t 0.605 (0.620)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 12:56:52,920 - Train: 2.94% [55900/1902000] [29.4/1000.0] [batch_t 0.603 (0.700)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 12:58:02,477 - Train: 2.94% [56000/1902000] [29.4/1000.0] [batch_t 0.609 (0.695)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-02 12:59:32,051 - Train: 2.95% [56100/1902000] [29.5/1000.0] [batch_t 0.606 (0.896)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 13:05:37,367 - Train: 2.95% [56200/1902000] [29.5/1000.0] [batch_t 1.411 (3.653)] [data_t 0.002] [optim_t 1.409] [lr 0.005000] +2024-04-02 13:07:57,920 - Train: 2.96% [56300/1902000] [29.6/1000.0] [batch_t 1.407 (1.405)] [data_t 0.002] [optim_t 1.405] [lr 0.005000] +2024-04-02 13:10:18,556 - Train: 2.97% [56400/1902000] [29.7/1000.0] [batch_t 1.410 (1.406)] [data_t 0.002] [optim_t 1.408] [lr 0.005000] +2024-04-02 13:12:52,677 - Train: 2.97% [56500/1902000] [29.7/1000.0] [batch_t 1.412 (1.541)] [data_t 0.002] [optim_t 1.410] [lr 0.005000] +2024-04-02 13:15:36,740 - Train: 2.98% [56600/1902000] [29.8/1000.0] [batch_t 24.925 (1.641)] [data_t 23.514] [optim_t 1.411] [lr 0.005000] +2024-04-02 13:17:57,310 - Train: 2.98% [56700/1902000] [29.8/1000.0] [batch_t 1.410 (1.406)] [data_t 0.002] [optim_t 1.407] [lr 0.005000] +2024-04-02 13:20:17,926 - Train: 2.99% [56800/1902000] [29.9/1000.0] [batch_t 1.412 (1.406)] [data_t 0.002] [optim_t 1.410] [lr 0.005000] +2024-04-02 13:22:38,520 - Train: 2.99% [56900/1902000] [29.9/1000.0] [batch_t 1.409 (1.406)] [data_t 0.002] [optim_t 1.407] [lr 0.005000] +2024-04-02 13:24:58,987 - Train: 3.00% [57000/1902000] [30.0/1000.0] [batch_t 1.409 (1.405)] [data_t 0.002] [optim_t 1.407] [lr 0.005000] +2024-04-02 13:26:23,341 - ==> Total time: 12:26:02 Eta: 16 days, 18:02:12 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 13:27:23,752 - Train: 3.00% [57100/1902000] [30.0/1000.0] [batch_t 1.412 (1.462)] [data_t 0.002] [optim_t 1.410] [lr 0.005000] +2024-04-02 13:29:44,397 - Train: 3.01% [57200/1902000] [30.1/1000.0] [batch_t 1.408 (1.406)] [data_t 0.002] [optim_t 1.406] [lr 0.005000] +2024-04-02 13:32:04,911 - Train: 3.01% [57300/1902000] [30.1/1000.0] [batch_t 1.414 (1.405)] [data_t 0.002] [optim_t 1.412] [lr 0.005000] +2024-04-02 13:34:25,461 - Train: 3.02% [57400/1902000] [30.2/1000.0] [batch_t 1.400 (1.405)] [data_t 0.002] [optim_t 1.398] [lr 0.005000] +2024-04-02 13:36:46,035 - Train: 3.02% [57500/1902000] [30.2/1000.0] [batch_t 1.406 (1.406)] [data_t 0.002] [optim_t 1.404] [lr 0.005000] +2024-04-02 13:39:06,628 - Train: 3.03% [57600/1902000] [30.3/1000.0] [batch_t 1.414 (1.406)] [data_t 0.002] [optim_t 1.412] [lr 0.005000] +2024-04-02 13:41:27,260 - Train: 3.03% [57700/1902000] [30.3/1000.0] [batch_t 1.408 (1.406)] [data_t 0.002] [optim_t 1.406] [lr 0.005000] +2024-04-02 13:43:47,852 - Train: 3.04% [57800/1902000] [30.4/1000.0] [batch_t 1.405 (1.406)] [data_t 0.002] [optim_t 1.403] [lr 0.005000] +2024-04-02 13:46:08,339 - Train: 3.04% [57900/1902000] [30.4/1000.0] [batch_t 1.407 (1.405)] [data_t 0.002] [optim_t 1.405] [lr 0.005000] +2024-04-02 13:48:28,783 - Train: 3.05% [58000/1902000] [30.5/1000.0] [batch_t 1.398 (1.404)] [data_t 0.002] [optim_t 1.396] [lr 0.005000] +2024-04-02 13:50:49,248 - Train: 3.05% [58100/1902000] [30.5/1000.0] [batch_t 1.400 (1.405)] [data_t 0.002] [optim_t 1.398] [lr 0.005000] +2024-04-02 13:53:09,588 - Train: 3.06% [58200/1902000] [30.6/1000.0] [batch_t 1.403 (1.403)] [data_t 0.002] [optim_t 1.400] [lr 0.005000] +2024-04-02 13:55:29,998 - Train: 3.07% [58300/1902000] [30.7/1000.0] [batch_t 1.410 (1.404)] [data_t 0.002] [optim_t 1.408] [lr 0.005000] +2024-04-02 13:57:50,434 - Train: 3.07% [58400/1902000] [30.7/1000.0] [batch_t 1.412 (1.404)] [data_t 0.002] [optim_t 1.410] [lr 0.005000] +2024-04-02 14:00:10,899 - Train: 3.08% [58500/1902000] [30.8/1000.0] [batch_t 1.409 (1.405)] [data_t 0.002] [optim_t 1.407] [lr 0.005000] +2024-04-02 14:02:31,431 - Train: 3.08% [58600/1902000] [30.8/1000.0] [batch_t 1.409 (1.405)] [data_t 0.002] [optim_t 1.406] [lr 0.005000] +2024-04-02 14:04:51,850 - Train: 3.09% [58700/1902000] [30.9/1000.0] [batch_t 1.412 (1.404)] [data_t 0.002] [optim_t 1.410] [lr 0.005000] +2024-04-02 14:07:12,310 - Train: 3.09% [58800/1902000] [30.9/1000.0] [batch_t 1.412 (1.405)] [data_t 0.002] [optim_t 1.409] [lr 0.005000] +2024-04-02 14:08:26,928 - Train: 3.10% [58900/1902000] [31.0/1000.0] [batch_t 0.605 (0.746)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 14:09:04,385 - ==> Total time: 13:08:43 Eta: 17 days, 2:54:13 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 14:09:31,959 - Train: 3.10% [59000/1902000] [31.0/1000.0] [batch_t 0.605 (0.632)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 14:10:33,078 - Train: 3.11% [59100/1902000] [31.1/1000.0] [batch_t 0.604 (0.611)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 14:11:34,560 - Train: 3.11% [59200/1902000] [31.1/1000.0] [batch_t 0.604 (0.615)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 14:12:35,899 - Train: 3.12% [59300/1902000] [31.2/1000.0] [batch_t 0.605 (0.613)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 14:13:36,670 - Train: 3.12% [59400/1902000] [31.2/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 14:14:37,559 - Train: 3.13% [59500/1902000] [31.3/1000.0] [batch_t 0.603 (0.609)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 14:15:39,243 - Train: 3.13% [59600/1902000] [31.3/1000.0] [batch_t 0.605 (0.617)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 14:16:39,679 - Train: 3.14% [59700/1902000] [31.4/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 14:17:43,220 - Train: 3.14% [59800/1902000] [31.4/1000.0] [batch_t 0.606 (0.635)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 14:18:45,978 - Train: 3.15% [59900/1902000] [31.5/1000.0] [batch_t 0.605 (0.627)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 14:19:46,885 - Train: 3.15% [60000/1902000] [31.5/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 14:20:48,692 - Train: 3.16% [60100/1902000] [31.6/1000.0] [batch_t 0.605 (0.618)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 14:21:49,715 - Train: 3.17% [60200/1902000] [31.7/1000.0] [batch_t 0.604 (0.610)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 14:22:53,203 - Train: 3.17% [60300/1902000] [31.7/1000.0] [batch_t 0.603 (0.635)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 14:23:54,385 - Train: 3.18% [60400/1902000] [31.8/1000.0] [batch_t 0.603 (0.612)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 14:24:56,304 - Train: 3.18% [60500/1902000] [31.8/1000.0] [batch_t 0.603 (0.619)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 14:25:58,368 - Train: 3.19% [60600/1902000] [31.9/1000.0] [batch_t 0.603 (0.621)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 14:27:00,618 - Train: 3.19% [60700/1902000] [31.9/1000.0] [batch_t 0.605 (0.622)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 14:28:06,060 - Train: 3.20% [60800/1902000] [32.0/1000.0] [batch_t 0.605 (0.654)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 14:28:44,782 - ==> Total time: 13:28:24 Eta: 16 days, 23:34:15 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 14:29:14,224 - Train: 3.20% [60900/1902000] [32.0/1000.0] [batch_t 0.605 (0.765)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 14:30:18,126 - Train: 3.21% [61000/1902000] [32.1/1000.0] [batch_t 0.603 (0.639)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 14:31:25,301 - Train: 3.21% [61100/1902000] [32.1/1000.0] [batch_t 0.605 (0.672)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 14:32:35,962 - Train: 3.22% [61200/1902000] [32.2/1000.0] [batch_t 0.606 (0.707)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 14:33:37,449 - Train: 3.22% [61300/1902000] [32.2/1000.0] [batch_t 0.605 (0.615)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 14:34:39,831 - Train: 3.23% [61400/1902000] [32.3/1000.0] [batch_t 0.603 (0.624)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 14:35:42,902 - Train: 3.23% [61500/1902000] [32.3/1000.0] [batch_t 0.607 (0.631)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 14:36:44,242 - Train: 3.24% [61600/1902000] [32.4/1000.0] [batch_t 0.603 (0.613)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 14:37:50,824 - Train: 3.24% [61700/1902000] [32.4/1000.0] [batch_t 0.607 (0.666)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 14:39:06,711 - Train: 3.25% [61800/1902000] [32.5/1000.0] [batch_t 0.602 (0.759)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 14:40:10,787 - Train: 3.25% [61900/1902000] [32.5/1000.0] [batch_t 1.223 (0.641)] [data_t 0.623] [optim_t 0.600] [lr 0.005000] +2024-04-02 14:41:22,255 - Train: 3.26% [62000/1902000] [32.6/1000.0] [batch_t 0.605 (0.715)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 14:42:26,595 - Train: 3.26% [62100/1902000] [32.6/1000.0] [batch_t 0.605 (0.643)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 14:43:26,988 - Train: 3.27% [62200/1902000] [32.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 14:44:32,154 - Train: 3.28% [62300/1902000] [32.8/1000.0] [batch_t 0.604 (0.652)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 14:45:34,390 - Train: 3.28% [62400/1902000] [32.8/1000.0] [batch_t 0.606 (0.622)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 14:46:38,884 - Train: 3.29% [62500/1902000] [32.9/1000.0] [batch_t 0.604 (0.645)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 14:47:39,570 - Train: 3.29% [62600/1902000] [32.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 14:48:40,657 - Train: 3.30% [62700/1902000] [33.0/1000.0] [batch_t 0.605 (0.611)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 14:49:24,438 - ==> Total time: 13:49:03 Eta: 16 days, 20:54:08 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 14:49:47,223 - Train: 3.30% [62800/1902000] [33.0/1000.0] [batch_t 0.605 (0.617)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 14:50:49,126 - Train: 3.31% [62900/1902000] [33.1/1000.0] [batch_t 0.605 (0.619)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 14:51:51,162 - Train: 3.31% [63000/1902000] [33.1/1000.0] [batch_t 0.604 (0.620)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 14:53:00,162 - Train: 3.32% [63100/1902000] [33.2/1000.0] [batch_t 0.604 (0.690)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 14:54:03,616 - Train: 3.32% [63200/1902000] [33.2/1000.0] [batch_t 0.604 (0.634)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 14:55:21,801 - Train: 3.33% [63300/1902000] [33.3/1000.0] [batch_t 0.609 (0.782)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-02 14:56:51,115 - Train: 3.33% [63400/1902000] [33.3/1000.0] [batch_t 2.534 (0.893)] [data_t 1.933] [optim_t 0.602] [lr 0.005000] +2024-04-02 15:00:18,039 - Train: 3.34% [63500/1902000] [33.4/1000.0] [batch_t 0.603 (2.069)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 15:01:53,521 - Train: 3.34% [63600/1902000] [33.4/1000.0] [batch_t 0.605 (0.955)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 15:03:24,497 - Train: 3.35% [63700/1902000] [33.5/1000.0] [batch_t 5.514 (0.910)] [data_t 4.912] [optim_t 0.603] [lr 0.005000] +2024-04-02 15:06:14,417 - Train: 3.35% [63800/1902000] [33.5/1000.0] [batch_t 0.602 (1.699)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 15:07:39,058 - Train: 3.36% [63900/1902000] [33.6/1000.0] [batch_t 3.243 (0.846)] [data_t 2.639] [optim_t 0.604] [lr 0.005000] +2024-04-02 15:08:47,923 - Train: 3.36% [64000/1902000] [33.6/1000.0] [batch_t 0.604 (0.689)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 15:10:52,033 - Train: 3.37% [64100/1902000] [33.7/1000.0] [batch_t 0.603 (1.241)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 15:12:41,236 - Train: 3.38% [64200/1902000] [33.8/1000.0] [batch_t 0.605 (1.092)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 15:15:17,936 - Train: 3.38% [64300/1902000] [33.8/1000.0] [batch_t 0.603 (1.567)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 15:16:19,435 - Train: 3.39% [64400/1902000] [33.9/1000.0] [batch_t 0.614 (0.615)] [data_t 0.002] [optim_t 0.612] [lr 0.005000] +2024-04-02 15:17:44,824 - Train: 3.39% [64500/1902000] [33.9/1000.0] [batch_t 0.604 (0.854)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 15:18:45,503 - Train: 3.40% [64600/1902000] [34.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 15:19:31,195 - ==> Total time: 14:19:10 Eta: 16 days, 22:50:46 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 15:19:52,359 - Train: 3.40% [64700/1902000] [34.0/1000.0] [batch_t 0.610 (0.618)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-02 15:21:22,524 - Train: 3.41% [64800/1902000] [34.1/1000.0] [batch_t 0.605 (0.902)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 15:22:28,528 - Train: 3.41% [64900/1902000] [34.1/1000.0] [batch_t 0.605 (0.660)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 15:23:29,315 - Train: 3.42% [65000/1902000] [34.2/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 15:24:30,801 - Train: 3.42% [65100/1902000] [34.2/1000.0] [batch_t 0.606 (0.615)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 15:25:35,563 - Train: 3.43% [65200/1902000] [34.3/1000.0] [batch_t 0.605 (0.648)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 15:26:37,838 - Train: 3.43% [65300/1902000] [34.3/1000.0] [batch_t 0.606 (0.623)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 15:27:47,230 - Train: 3.44% [65400/1902000] [34.4/1000.0] [batch_t 0.604 (0.694)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 15:28:51,810 - Train: 3.44% [65500/1902000] [34.4/1000.0] [batch_t 0.606 (0.646)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 15:29:52,235 - Train: 3.45% [65600/1902000] [34.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 15:30:54,942 - Train: 3.45% [65700/1902000] [34.5/1000.0] [batch_t 0.605 (0.627)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 15:31:59,616 - Train: 3.46% [65800/1902000] [34.6/1000.0] [batch_t 0.604 (0.647)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 15:33:00,504 - Train: 3.46% [65900/1902000] [34.6/1000.0] [batch_t 0.613 (0.609)] [data_t 0.002] [optim_t 0.610] [lr 0.005000] +2024-04-02 15:34:03,467 - Train: 3.47% [66000/1902000] [34.7/1000.0] [batch_t 0.687 (0.630)] [data_t 0.002] [optim_t 0.685] [lr 0.005000] +2024-04-02 15:35:06,110 - Train: 3.48% [66100/1902000] [34.8/1000.0] [batch_t 0.606 (0.626)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 15:36:06,849 - Train: 3.48% [66200/1902000] [34.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 15:37:10,319 - Train: 3.49% [66300/1902000] [34.9/1000.0] [batch_t 0.606 (0.635)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 15:38:12,320 - Train: 3.49% [66400/1902000] [34.9/1000.0] [batch_t 0.605 (0.620)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 15:39:13,662 - Train: 3.50% [66500/1902000] [35.0/1000.0] [batch_t 0.604 (0.613)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 15:39:55,970 - ==> Total time: 14:39:35 Eta: 16 days, 20:11:35 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 15:40:17,854 - Train: 3.50% [66600/1902000] [35.0/1000.0] [batch_t 0.606 (0.679)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 15:41:22,067 - Train: 3.51% [66700/1902000] [35.1/1000.0] [batch_t 0.605 (0.642)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 15:42:22,605 - Train: 3.51% [66800/1902000] [35.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 15:43:23,406 - Train: 3.52% [66900/1902000] [35.2/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 15:44:24,070 - Train: 3.52% [67000/1902000] [35.2/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 15:45:26,722 - Train: 3.53% [67100/1902000] [35.3/1000.0] [batch_t 0.601 (0.626)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-02 15:46:28,346 - Train: 3.53% [67200/1902000] [35.3/1000.0] [batch_t 0.606 (0.616)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 15:47:29,223 - Train: 3.54% [67300/1902000] [35.4/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 15:48:30,428 - Train: 3.54% [67400/1902000] [35.4/1000.0] [batch_t 0.605 (0.612)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 15:49:31,996 - Train: 3.55% [67500/1902000] [35.5/1000.0] [batch_t 0.605 (0.616)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 15:50:32,393 - Train: 3.55% [67600/1902000] [35.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 15:51:34,609 - Train: 3.56% [67700/1902000] [35.6/1000.0] [batch_t 0.604 (0.622)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 15:52:37,531 - Train: 3.56% [67800/1902000] [35.6/1000.0] [batch_t 0.603 (0.629)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 15:53:41,321 - Train: 3.57% [67900/1902000] [35.7/1000.0] [batch_t 0.603 (0.638)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 15:54:41,724 - Train: 3.58% [68000/1902000] [35.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 15:55:42,352 - Train: 3.58% [68100/1902000] [35.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 15:56:44,792 - Train: 3.59% [68200/1902000] [35.9/1000.0] [batch_t 0.604 (0.624)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 15:57:46,610 - Train: 3.59% [68300/1902000] [35.9/1000.0] [batch_t 0.605 (0.618)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 15:58:48,239 - Train: 3.60% [68400/1902000] [36.0/1000.0] [batch_t 0.605 (0.616)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 15:59:34,037 - ==> Total time: 14:59:13 Eta: 16 days, 17:19:16 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 15:59:52,982 - Train: 3.60% [68500/1902000] [36.0/1000.0] [batch_t 0.604 (0.618)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 16:00:53,479 - Train: 3.61% [68600/1902000] [36.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-02 16:01:53,896 - Train: 3.61% [68700/1902000] [36.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 16:02:54,718 - Train: 3.62% [68800/1902000] [36.2/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 16:03:56,513 - Train: 3.62% [68900/1902000] [36.2/1000.0] [batch_t 0.605 (0.618)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 16:04:57,623 - Train: 3.63% [69000/1902000] [36.3/1000.0] [batch_t 0.609 (0.611)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-02 16:05:58,713 - Train: 3.63% [69100/1902000] [36.3/1000.0] [batch_t 0.604 (0.611)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 16:07:05,914 - Train: 3.64% [69200/1902000] [36.4/1000.0] [batch_t 0.604 (0.672)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 16:08:11,480 - Train: 3.64% [69300/1902000] [36.4/1000.0] [batch_t 0.605 (0.656)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 16:09:13,091 - Train: 3.65% [69400/1902000] [36.5/1000.0] [batch_t 0.605 (0.616)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 16:10:19,470 - Train: 3.65% [69500/1902000] [36.5/1000.0] [batch_t 0.605 (0.664)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 16:11:29,386 - Train: 3.66% [69600/1902000] [36.6/1000.0] [batch_t 0.608 (0.699)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-02 16:12:31,496 - Train: 3.66% [69700/1902000] [36.6/1000.0] [batch_t 0.606 (0.621)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 16:13:42,907 - Train: 3.67% [69800/1902000] [36.7/1000.0] [batch_t 0.604 (0.714)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 16:14:46,052 - Train: 3.68% [69900/1902000] [36.8/1000.0] [batch_t 0.604 (0.631)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 16:15:55,120 - Train: 3.68% [70000/1902000] [36.8/1000.0] [batch_t 0.605 (0.691)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 16:17:34,272 - Train: 3.69% [70100/1902000] [36.9/1000.0] [batch_t 0.604 (0.991)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 16:18:41,714 - Train: 3.69% [70200/1902000] [36.9/1000.0] [batch_t 0.604 (0.674)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 16:19:59,914 - Train: 3.70% [70300/1902000] [37.0/1000.0] [batch_t 0.602 (0.782)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 16:20:52,019 - ==> Total time: 15:20:31 Eta: 16 days, 15:18:32 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 16:21:13,823 - Train: 3.70% [70400/1902000] [37.0/1000.0] [batch_t 0.604 (0.775)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 16:23:01,771 - Train: 3.71% [70500/1902000] [37.1/1000.0] [batch_t 0.605 (1.079)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 16:27:23,163 - Train: 3.71% [70600/1902000] [37.1/1000.0] [batch_t 1.409 (2.614)] [data_t 0.002] [optim_t 1.406] [lr 0.005000] +2024-04-02 16:30:07,104 - Train: 3.72% [70700/1902000] [37.2/1000.0] [batch_t 1.413 (1.639)] [data_t 0.002] [optim_t 1.411] [lr 0.005000] +2024-04-02 16:32:51,194 - Train: 3.72% [70800/1902000] [37.2/1000.0] [batch_t 1.412 (1.641)] [data_t 0.002] [optim_t 1.410] [lr 0.005000] +2024-04-02 16:36:28,244 - Train: 3.73% [70900/1902000] [37.3/1000.0] [batch_t 1.407 (2.170)] [data_t 0.002] [optim_t 1.404] [lr 0.005000] +2024-04-02 16:43:54,152 - Train: 3.73% [71000/1902000] [37.3/1000.0] [batch_t 1.412 (4.459)] [data_t 0.002] [optim_t 1.410] [lr 0.005000] +2024-04-02 16:46:20,792 - Train: 3.74% [71100/1902000] [37.4/1000.0] [batch_t 1.402 (1.466)] [data_t 0.002] [optim_t 1.400] [lr 0.005000] +2024-04-02 16:48:41,264 - Train: 3.74% [71200/1902000] [37.4/1000.0] [batch_t 1.406 (1.405)] [data_t 0.002] [optim_t 1.404] [lr 0.005000] +2024-04-02 16:51:01,772 - Train: 3.75% [71300/1902000] [37.5/1000.0] [batch_t 1.404 (1.405)] [data_t 0.002] [optim_t 1.401] [lr 0.005000] +2024-04-02 16:53:22,206 - Train: 3.75% [71400/1902000] [37.5/1000.0] [batch_t 1.400 (1.404)] [data_t 0.002] [optim_t 1.398] [lr 0.005000] +2024-04-02 16:56:46,248 - Train: 3.76% [71500/1902000] [37.6/1000.0] [batch_t 1.456 (2.040)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-02 16:59:17,399 - Train: 3.76% [71600/1902000] [37.6/1000.0] [batch_t 1.450 (1.511)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-02 17:01:42,431 - Train: 3.77% [71700/1902000] [37.7/1000.0] [batch_t 1.456 (1.450)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-02 17:04:12,872 - Train: 3.77% [71800/1902000] [37.7/1000.0] [batch_t 1.450 (1.504)] [data_t 0.003] [optim_t 1.446] [lr 0.005000] +2024-04-02 17:06:37,893 - Train: 3.78% [71900/1902000] [37.8/1000.0] [batch_t 1.450 (1.450)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-02 17:08:39,376 - Train: 3.79% [72000/1902000] [37.9/1000.0] [batch_t 3.461 (1.215)] [data_t 2.859] [optim_t 0.602] [lr 0.005000] +2024-04-02 17:11:52,282 - Train: 3.79% [72100/1902000] [37.9/1000.0] [batch_t 0.607 (1.929)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-02 17:13:33,200 - Train: 3.80% [72200/1902000] [38.0/1000.0] [batch_t 0.605 (1.009)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 17:14:22,168 - ==> Total time: 16:14:01 Eta: 17 days, 2:58:17 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 17:14:43,897 - Train: 3.80% [72300/1902000] [38.0/1000.0] [batch_t 0.607 (0.836)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-02 17:15:44,987 - Train: 3.81% [72400/1902000] [38.1/1000.0] [batch_t 0.604 (0.611)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-02 17:16:49,668 - Train: 3.81% [72500/1902000] [38.1/1000.0] [batch_t 0.606 (0.647)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-02 17:17:54,846 - Train: 3.82% [72600/1902000] [38.2/1000.0] [batch_t 0.607 (0.652)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 17:19:04,620 - Train: 3.82% [72700/1902000] [38.2/1000.0] [batch_t 0.604 (0.698)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 17:20:10,949 - Train: 3.83% [72800/1902000] [38.3/1000.0] [batch_t 0.606 (0.663)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-02 17:21:13,808 - Train: 3.83% [72900/1902000] [38.3/1000.0] [batch_t 0.621 (0.629)] [data_t 0.017] [optim_t 0.604] [lr 0.005000] +2024-04-02 17:22:18,291 - Train: 3.84% [73000/1902000] [38.4/1000.0] [batch_t 0.609 (0.645)] [data_t 0.003] [optim_t 0.607] [lr 0.005000] +2024-04-02 17:23:54,766 - Train: 3.84% [73100/1902000] [38.4/1000.0] [batch_t 1.375 (0.965)] [data_t 0.771] [optim_t 0.604] [lr 0.005000] +2024-04-02 17:26:01,178 - Train: 3.85% [73200/1902000] [38.5/1000.0] [batch_t 2.250 (1.264)] [data_t 1.646] [optim_t 0.604] [lr 0.005000] +2024-04-02 17:28:52,446 - Train: 3.85% [73300/1902000] [38.5/1000.0] [batch_t 1.840 (1.713)] [data_t 1.238] [optim_t 0.602] [lr 0.005000] +2024-04-02 17:30:23,363 - Train: 3.86% [73400/1902000] [38.6/1000.0] [batch_t 0.606 (0.909)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 17:32:14,277 - Train: 3.86% [73500/1902000] [38.6/1000.0] [batch_t 5.800 (1.109)] [data_t 5.198] [optim_t 0.603] [lr 0.005000] +2024-04-02 17:33:32,587 - Train: 3.87% [73600/1902000] [38.7/1000.0] [batch_t 0.605 (0.783)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 17:34:33,319 - Train: 3.87% [73700/1902000] [38.7/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 17:35:34,415 - Train: 3.88% [73800/1902000] [38.8/1000.0] [batch_t 0.606 (0.611)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 17:36:36,472 - Train: 3.89% [73900/1902000] [38.9/1000.0] [batch_t 0.604 (0.620)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 17:37:39,678 - Train: 3.89% [74000/1902000] [38.9/1000.0] [batch_t 0.606 (0.632)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 17:38:54,510 - Train: 3.90% [74100/1902000] [39.0/1000.0] [batch_t 0.607 (0.748)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 17:39:44,119 - ==> Total time: 16:39:23 Eta: 17 days, 2:26:05 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 17:40:00,338 - Train: 3.90% [74200/1902000] [39.0/1000.0] [batch_t 0.605 (0.655)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 17:41:01,654 - Train: 3.91% [74300/1902000] [39.1/1000.0] [batch_t 0.606 (0.613)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 17:42:03,664 - Train: 3.91% [74400/1902000] [39.1/1000.0] [batch_t 0.606 (0.620)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 17:43:10,241 - Train: 3.92% [74500/1902000] [39.2/1000.0] [batch_t 0.605 (0.666)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 17:44:30,693 - Train: 3.92% [74600/1902000] [39.2/1000.0] [batch_t 0.605 (0.804)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-02 17:45:33,258 - Train: 3.93% [74700/1902000] [39.3/1000.0] [batch_t 0.607 (0.626)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 17:46:37,256 - Train: 3.93% [74800/1902000] [39.3/1000.0] [batch_t 0.608 (0.640)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-02 17:47:37,874 - Train: 3.94% [74900/1902000] [39.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 17:48:38,413 - Train: 3.94% [75000/1902000] [39.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 17:49:50,096 - Train: 3.95% [75100/1902000] [39.5/1000.0] [batch_t 0.604 (0.717)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 17:50:50,592 - Train: 3.95% [75200/1902000] [39.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 17:51:52,966 - Train: 3.96% [75300/1902000] [39.6/1000.0] [batch_t 0.606 (0.624)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-02 17:52:56,424 - Train: 3.96% [75400/1902000] [39.6/1000.0] [batch_t 0.605 (0.634)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 17:53:59,091 - Train: 3.97% [75500/1902000] [39.7/1000.0] [batch_t 0.607 (0.627)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 17:55:01,702 - Train: 3.97% [75600/1902000] [39.7/1000.0] [batch_t 0.605 (0.626)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 17:56:03,233 - Train: 3.98% [75700/1902000] [39.8/1000.0] [batch_t 0.606 (0.615)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 17:57:20,758 - Train: 3.99% [75800/1902000] [39.9/1000.0] [batch_t 1.384 (0.775)] [data_t 0.780] [optim_t 0.604] [lr 0.005000] +2024-04-02 17:58:35,284 - Train: 3.99% [75900/1902000] [39.9/1000.0] [batch_t 0.607 (0.745)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 17:59:37,475 - Train: 4.00% [76000/1902000] [40.0/1000.0] [batch_t 0.606 (0.622)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-02 18:00:29,766 - ==> Total time: 17:00:09 Eta: 17 days, 0:03:42 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 18:00:44,471 - Train: 4.00% [76100/1902000] [40.0/1000.0] [batch_t 0.605 (0.650)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 18:01:57,246 - Train: 4.01% [76200/1902000] [40.1/1000.0] [batch_t 2.427 (0.728)] [data_t 1.824] [optim_t 0.603] [lr 0.005000] +2024-04-02 18:03:10,423 - Train: 4.01% [76300/1902000] [40.1/1000.0] [batch_t 0.606 (0.732)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:04:13,388 - Train: 4.02% [76400/1902000] [40.2/1000.0] [batch_t 0.604 (0.630)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-02 18:05:18,245 - Train: 4.02% [76500/1902000] [40.2/1000.0] [batch_t 0.604 (0.648)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 18:06:20,797 - Train: 4.03% [76600/1902000] [40.3/1000.0] [batch_t 0.605 (0.625)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:07:23,590 - Train: 4.03% [76700/1902000] [40.3/1000.0] [batch_t 0.603 (0.628)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 18:08:36,123 - Train: 4.04% [76800/1902000] [40.4/1000.0] [batch_t 0.606 (0.725)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:09:40,685 - Train: 4.04% [76900/1902000] [40.4/1000.0] [batch_t 0.605 (0.646)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 18:10:41,782 - Train: 4.05% [77000/1902000] [40.5/1000.0] [batch_t 0.606 (0.611)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:11:43,889 - Train: 4.05% [77100/1902000] [40.5/1000.0] [batch_t 0.606 (0.621)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:12:50,927 - Train: 4.06% [77200/1902000] [40.6/1000.0] [batch_t 0.604 (0.670)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 18:13:51,955 - Train: 4.06% [77300/1902000] [40.6/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 18:14:54,934 - Train: 4.07% [77400/1902000] [40.7/1000.0] [batch_t 0.605 (0.630)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 18:16:04,366 - Train: 4.07% [77500/1902000] [40.7/1000.0] [batch_t 0.606 (0.694)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 18:17:17,353 - Train: 4.08% [77600/1902000] [40.8/1000.0] [batch_t 0.605 (0.730)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 18:18:21,153 - Train: 4.09% [77700/1902000] [40.9/1000.0] [batch_t 0.605 (0.638)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 18:19:30,006 - Train: 4.09% [77800/1902000] [40.9/1000.0] [batch_t 6.792 (0.688)] [data_t 6.190] [optim_t 0.602] [lr 0.005000] +2024-04-02 18:20:30,456 - Train: 4.10% [77900/1902000] [41.0/1000.0] [batch_t 0.610 (0.604)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-02 18:21:20,397 - ==> Total time: 17:20:59 Eta: 16 days, 21:49:12 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 18:21:33,011 - Train: 4.10% [78000/1902000] [41.0/1000.0] [batch_t 0.606 (0.624)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:22:33,604 - Train: 4.11% [78100/1902000] [41.1/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 18:23:34,064 - Train: 4.11% [78200/1902000] [41.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:24:34,605 - Train: 4.12% [78300/1902000] [41.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 18:25:35,080 - Train: 4.12% [78400/1902000] [41.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 18:26:35,565 - Train: 4.13% [78500/1902000] [41.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 18:27:36,096 - Train: 4.13% [78600/1902000] [41.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 18:28:36,614 - Train: 4.14% [78700/1902000] [41.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 18:29:37,103 - Train: 4.14% [78800/1902000] [41.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 18:30:37,613 - Train: 4.15% [78900/1902000] [41.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:31:38,111 - Train: 4.15% [79000/1902000] [41.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 18:32:38,590 - Train: 4.16% [79100/1902000] [41.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 18:33:39,152 - Train: 4.16% [79200/1902000] [41.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-02 18:34:40,833 - Train: 4.17% [79300/1902000] [41.7/1000.0] [batch_t 0.607 (0.617)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:35:42,803 - Train: 4.17% [79400/1902000] [41.7/1000.0] [batch_t 0.605 (0.620)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 18:36:44,684 - Train: 4.18% [79500/1902000] [41.8/1000.0] [batch_t 0.602 (0.619)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-02 18:37:45,166 - Train: 4.19% [79600/1902000] [41.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:39:02,292 - Train: 4.19% [79700/1902000] [41.9/1000.0] [batch_t 0.606 (0.771)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:40:02,834 - Train: 4.20% [79800/1902000] [42.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:40:55,243 - ==> Total time: 17:40:34 Eta: 16 days, 19:11:18 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 18:41:06,808 - Train: 4.20% [79900/1902000] [42.0/1000.0] [batch_t 0.605 (0.630)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 18:42:07,265 - Train: 4.21% [80000/1902000] [42.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 18:43:18,379 - Train: 4.21% [80100/1902000] [42.1/1000.0] [batch_t 0.605 (0.711)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-02 18:44:18,866 - Train: 4.22% [80200/1902000] [42.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:45:20,386 - Train: 4.22% [80300/1902000] [42.2/1000.0] [batch_t 0.605 (0.615)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 18:46:21,369 - Train: 4.23% [80400/1902000] [42.3/1000.0] [batch_t 0.603 (0.610)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-02 18:47:24,501 - Train: 4.23% [80500/1902000] [42.3/1000.0] [batch_t 0.606 (0.631)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:48:25,916 - Train: 4.24% [80600/1902000] [42.4/1000.0] [batch_t 0.606 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:49:26,501 - Train: 4.24% [80700/1902000] [42.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:50:28,038 - Train: 4.25% [80800/1902000] [42.5/1000.0] [batch_t 0.607 (0.615)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-02 18:51:28,850 - Train: 4.25% [80900/1902000] [42.5/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-02 18:52:56,151 - Train: 4.26% [81000/1902000] [42.6/1000.0] [batch_t 1.255 (0.873)] [data_t 0.652] [optim_t 0.603] [lr 0.005000] +2024-04-02 18:54:28,409 - Train: 4.26% [81100/1902000] [42.6/1000.0] [batch_t 1.367 (0.922)] [data_t 0.763] [optim_t 0.604] [lr 0.005000] +2024-04-02 18:55:36,551 - Train: 4.27% [81200/1902000] [42.7/1000.0] [batch_t 0.606 (0.681)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-02 19:08:01,997 - Train: 4.27% [81300/1902000] [42.7/1000.0] [batch_t 4.170 (7.454)] [data_t 2.732] [optim_t 1.438] [lr 0.005000] +2024-04-02 19:18:01,385 - Train: 4.28% [81400/1902000] [42.8/1000.0] [batch_t 1.422 (5.994)] [data_t 0.002] [optim_t 1.420] [lr 0.005000] +2024-04-02 19:20:27,190 - Train: 4.28% [81500/1902000] [42.8/1000.0] [batch_t 1.428 (1.458)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-02 19:25:15,472 - Train: 4.29% [81600/1902000] [42.9/1000.0] [batch_t 1.422 (2.883)] [data_t 0.002] [optim_t 1.420] [lr 0.005000] +2024-04-02 19:29:01,401 - Train: 4.30% [81700/1902000] [43.0/1000.0] [batch_t 3.108 (2.259)] [data_t 1.694] [optim_t 1.414] [lr 0.005000] +2024-04-02 19:32:24,430 - ==> Total time: 18:32:03 Eta: 17 days, 4:29:55 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 19:33:13,429 - Train: 4.30% [81800/1902000] [43.0/1000.0] [batch_t 1.427 (3.328)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-02 19:35:55,788 - Train: 4.31% [81900/1902000] [43.1/1000.0] [batch_t 1.446 (1.624)] [data_t 0.003] [optim_t 1.444] [lr 0.005000] +2024-04-02 19:38:19,098 - Train: 4.31% [82000/1902000] [43.1/1000.0] [batch_t 1.436 (1.433)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-02 19:40:41,759 - Train: 4.32% [82100/1902000] [43.2/1000.0] [batch_t 1.424 (1.426)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-02 19:43:04,388 - Train: 4.32% [82200/1902000] [43.2/1000.0] [batch_t 1.433 (1.426)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-02 19:45:26,948 - Train: 4.33% [82300/1902000] [43.3/1000.0] [batch_t 1.422 (1.426)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-02 19:47:49,519 - Train: 4.33% [82400/1902000] [43.3/1000.0] [batch_t 1.409 (1.426)] [data_t 0.002] [optim_t 1.406] [lr 0.005000] +2024-04-02 19:50:12,137 - Train: 4.34% [82500/1902000] [43.4/1000.0] [batch_t 1.423 (1.426)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-02 19:52:41,172 - Train: 4.34% [82600/1902000] [43.4/1000.0] [batch_t 1.422 (1.490)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-02 19:55:07,972 - Train: 4.35% [82700/1902000] [43.5/1000.0] [batch_t 1.431 (1.468)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-02 19:57:30,582 - Train: 4.35% [82800/1902000] [43.5/1000.0] [batch_t 1.439 (1.426)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-02 19:59:53,091 - Train: 4.36% [82900/1902000] [43.6/1000.0] [batch_t 1.434 (1.425)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-02 20:02:58,503 - Train: 4.36% [83000/1902000] [43.6/1000.0] [batch_t 1.429 (1.854)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-02 20:06:16,351 - Train: 4.37% [83100/1902000] [43.7/1000.0] [batch_t 1.426 (1.978)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-02 20:08:41,164 - Train: 4.37% [83200/1902000] [43.7/1000.0] [batch_t 1.432 (1.448)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-02 20:11:51,048 - Train: 4.38% [83300/1902000] [43.8/1000.0] [batch_t 1.438 (1.899)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-02 20:14:24,417 - Train: 4.38% [83400/1902000] [43.8/1000.0] [batch_t 1.426 (1.534)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-02 20:17:48,044 - Train: 4.39% [83500/1902000] [43.9/1000.0] [batch_t 1.430 (2.036)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-02 20:20:10,641 - Train: 4.40% [83600/1902000] [44.0/1000.0] [batch_t 1.438 (1.426)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-02 20:22:16,109 - ==> Total time: 19:21:55 Eta: 17 days, 12:45:30 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 20:22:36,376 - Train: 4.40% [83700/1902000] [44.0/1000.0] [batch_t 1.422 (1.500)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-02 20:28:56,616 - Train: 4.41% [83800/1902000] [44.1/1000.0] [batch_t 2.816 (3.802)] [data_t 1.393] [optim_t 1.424] [lr 0.005000] +2024-04-02 20:31:19,684 - Train: 4.41% [83900/1902000] [44.1/1000.0] [batch_t 1.422 (1.431)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-02 20:33:42,329 - Train: 4.42% [84000/1902000] [44.2/1000.0] [batch_t 1.418 (1.426)] [data_t 0.003] [optim_t 1.415] [lr 0.005000] +2024-04-02 20:36:04,886 - Train: 4.42% [84100/1902000] [44.2/1000.0] [batch_t 1.429 (1.425)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-02 20:38:27,486 - Train: 4.43% [84200/1902000] [44.3/1000.0] [batch_t 1.427 (1.426)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-02 20:40:50,178 - Train: 4.43% [84300/1902000] [44.3/1000.0] [batch_t 1.431 (1.427)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-02 20:43:32,597 - Train: 4.44% [84400/1902000] [44.4/1000.0] [batch_t 1.429 (1.624)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-02 20:45:55,120 - Train: 4.44% [84500/1902000] [44.4/1000.0] [batch_t 1.412 (1.425)] [data_t 0.003] [optim_t 1.409] [lr 0.005000] +2024-04-02 20:48:49,707 - Train: 4.45% [84600/1902000] [44.5/1000.0] [batch_t 1.421 (1.746)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-02 20:51:15,458 - Train: 4.45% [84700/1902000] [44.5/1000.0] [batch_t 1.423 (1.457)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-02 20:53:41,304 - Train: 4.46% [84800/1902000] [44.6/1000.0] [batch_t 1.421 (1.458)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-02 20:56:03,720 - Train: 4.46% [84900/1902000] [44.6/1000.0] [batch_t 1.424 (1.424)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-02 20:58:32,117 - Train: 4.47% [85000/1902000] [44.7/1000.0] [batch_t 1.436 (1.484)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-02 21:00:54,941 - Train: 4.47% [85100/1902000] [44.7/1000.0] [batch_t 1.425 (1.428)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-02 21:04:26,563 - Train: 4.48% [85200/1902000] [44.8/1000.0] [batch_t 1.429 (2.116)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-02 21:07:08,225 - Train: 4.48% [85300/1902000] [44.8/1000.0] [batch_t 1.436 (1.617)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-02 21:09:33,667 - Train: 4.49% [85400/1902000] [44.9/1000.0] [batch_t 1.422 (1.454)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-02 21:12:10,962 - Train: 4.50% [85500/1902000] [45.0/1000.0] [batch_t 1.434 (1.573)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-02 21:18:02,809 - ==> Total time: 20:17:42 Eta: 17 days, 22:42:24 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 21:18:30,036 - Train: 4.50% [85600/1902000] [45.0/1000.0] [batch_t 1.422 (2.445)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-02 21:20:52,496 - Train: 4.51% [85700/1902000] [45.1/1000.0] [batch_t 1.433 (1.425)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-02 21:23:54,959 - Train: 4.51% [85800/1902000] [45.1/1000.0] [batch_t 1.426 (1.825)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-02 21:26:30,018 - Train: 4.52% [85900/1902000] [45.2/1000.0] [batch_t 1.435 (1.551)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-02 21:28:55,120 - Train: 4.52% [86000/1902000] [45.2/1000.0] [batch_t 1.435 (1.451)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-02 21:31:17,798 - Train: 4.53% [86100/1902000] [45.3/1000.0] [batch_t 1.431 (1.427)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-02 21:33:48,303 - Train: 4.53% [86200/1902000] [45.3/1000.0] [batch_t 1.426 (1.505)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-02 21:42:16,570 - Train: 4.54% [86300/1902000] [45.4/1000.0] [batch_t 1.432 (5.083)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-02 21:45:18,320 - Train: 4.54% [86400/1902000] [45.4/1000.0] [batch_t 5.677 (1.817)] [data_t 4.249] [optim_t 1.428] [lr 0.005000] +2024-04-02 21:47:50,805 - Train: 4.55% [86500/1902000] [45.5/1000.0] [batch_t 1.411 (1.525)] [data_t 0.002] [optim_t 1.409] [lr 0.005000] +2024-04-02 21:50:17,269 - Train: 4.55% [86600/1902000] [45.5/1000.0] [batch_t 1.499 (1.465)] [data_t 0.078] [optim_t 1.421] [lr 0.005000] +2024-04-02 21:54:57,313 - Train: 4.56% [86700/1902000] [45.6/1000.0] [batch_t 1.421 (2.800)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-02 21:57:19,820 - Train: 4.56% [86800/1902000] [45.6/1000.0] [batch_t 1.430 (1.425)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-02 21:59:42,608 - Train: 4.57% [86900/1902000] [45.7/1000.0] [batch_t 1.434 (1.428)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-02 22:02:05,242 - Train: 4.57% [87000/1902000] [45.7/1000.0] [batch_t 1.427 (1.426)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-02 22:04:32,969 - Train: 4.58% [87100/1902000] [45.8/1000.0] [batch_t 1.447 (1.477)] [data_t 0.002] [optim_t 1.444] [lr 0.005000] +2024-04-02 22:07:14,371 - Train: 4.58% [87200/1902000] [45.8/1000.0] [batch_t 1.431 (1.614)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-02 22:09:37,164 - Train: 4.59% [87300/1902000] [45.9/1000.0] [batch_t 1.441 (1.428)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-02 22:11:59,534 - Train: 4.60% [87400/1902000] [46.0/1000.0] [batch_t 1.423 (1.424)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-02 22:14:10,840 - ==> Total time: 21:13:50 Eta: 18 days, 8:18:19 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 22:14:24,516 - Train: 4.60% [87500/1902000] [46.0/1000.0] [batch_t 1.434 (1.478)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-02 22:16:47,073 - Train: 4.61% [87600/1902000] [46.1/1000.0] [batch_t 1.438 (1.425)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-02 22:19:09,594 - Train: 4.61% [87700/1902000] [46.1/1000.0] [batch_t 1.447 (1.425)] [data_t 0.003] [optim_t 1.443] [lr 0.005000] +2024-04-02 22:21:32,315 - Train: 4.62% [87800/1902000] [46.2/1000.0] [batch_t 1.416 (1.427)] [data_t 0.002] [optim_t 1.414] [lr 0.005000] +2024-04-02 22:23:54,916 - Train: 4.62% [87900/1902000] [46.2/1000.0] [batch_t 1.421 (1.426)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-02 22:26:28,259 - Train: 4.63% [88000/1902000] [46.3/1000.0] [batch_t 1.425 (1.494)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-02 22:28:50,662 - Train: 4.63% [88100/1902000] [46.3/1000.0] [batch_t 1.421 (1.424)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-02 22:31:16,952 - Train: 4.64% [88200/1902000] [46.4/1000.0] [batch_t 1.440 (1.463)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-02 22:33:39,532 - Train: 4.64% [88300/1902000] [46.4/1000.0] [batch_t 1.436 (1.426)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-02 22:36:05,030 - Train: 4.65% [88400/1902000] [46.5/1000.0] [batch_t 1.431 (1.455)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-02 22:38:27,774 - Train: 4.65% [88500/1902000] [46.5/1000.0] [batch_t 1.416 (1.427)] [data_t 0.002] [optim_t 1.413] [lr 0.005000] +2024-04-02 22:40:50,444 - Train: 4.66% [88600/1902000] [46.6/1000.0] [batch_t 1.430 (1.427)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-02 22:43:12,971 - Train: 4.66% [88700/1902000] [46.6/1000.0] [batch_t 1.439 (1.425)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-02 22:45:35,474 - Train: 4.67% [88800/1902000] [46.7/1000.0] [batch_t 1.426 (1.425)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-02 22:47:58,143 - Train: 4.67% [88900/1902000] [46.7/1000.0] [batch_t 1.435 (1.427)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-02 22:50:24,042 - Train: 4.68% [89000/1902000] [46.8/1000.0] [batch_t 1.436 (1.459)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-02 22:52:46,440 - Train: 4.68% [89100/1902000] [46.8/1000.0] [batch_t 1.429 (1.424)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-02 22:55:09,015 - Train: 4.69% [89200/1902000] [46.9/1000.0] [batch_t 1.421 (1.426)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-02 22:57:31,537 - Train: 4.70% [89300/1902000] [47.0/1000.0] [batch_t 1.426 (1.425)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-02 22:59:45,513 - ==> Total time: 21:59:25 Eta: 18 days, 13:53:17 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 22:59:56,184 - Train: 4.70% [89400/1902000] [47.0/1000.0] [batch_t 1.431 (1.464)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-02 23:02:18,817 - Train: 4.71% [89500/1902000] [47.1/1000.0] [batch_t 1.419 (1.426)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-02 23:04:41,228 - Train: 4.71% [89600/1902000] [47.1/1000.0] [batch_t 1.423 (1.424)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-02 23:07:03,831 - Train: 4.72% [89700/1902000] [47.2/1000.0] [batch_t 1.431 (1.426)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-02 23:09:26,189 - Train: 4.72% [89800/1902000] [47.2/1000.0] [batch_t 1.423 (1.423)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-02 23:11:59,954 - Train: 4.73% [89900/1902000] [47.3/1000.0] [batch_t 1.432 (1.538)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-02 23:14:32,310 - Train: 4.73% [90000/1902000] [47.3/1000.0] [batch_t 1.419 (1.523)] [data_t 0.003] [optim_t 1.416] [lr 0.005000] +2024-04-02 23:17:03,524 - Train: 4.74% [90100/1902000] [47.4/1000.0] [batch_t 1.420 (1.512)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-02 23:19:26,250 - Train: 4.74% [90200/1902000] [47.4/1000.0] [batch_t 1.422 (1.427)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-02 23:21:48,816 - Train: 4.75% [90300/1902000] [47.5/1000.0] [batch_t 1.421 (1.426)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-02 23:24:19,546 - Train: 4.75% [90400/1902000] [47.5/1000.0] [batch_t 5.746 (1.507)] [data_t 4.327] [optim_t 1.419] [lr 0.005000] +2024-04-02 23:26:54,182 - Train: 4.76% [90500/1902000] [47.6/1000.0] [batch_t 1.422 (1.546)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-02 23:29:16,696 - Train: 4.76% [90600/1902000] [47.6/1000.0] [batch_t 1.432 (1.425)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-02 23:31:39,041 - Train: 4.77% [90700/1902000] [47.7/1000.0] [batch_t 1.430 (1.423)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-02 23:34:01,810 - Train: 4.77% [90800/1902000] [47.7/1000.0] [batch_t 1.431 (1.428)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-02 23:36:24,386 - Train: 4.78% [90900/1902000] [47.8/1000.0] [batch_t 1.429 (1.426)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-02 23:38:46,654 - Train: 4.78% [91000/1902000] [47.8/1000.0] [batch_t 1.417 (1.423)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-02 23:41:13,523 - Train: 4.79% [91100/1902000] [47.9/1000.0] [batch_t 1.422 (1.469)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-02 23:43:36,070 - Train: 4.79% [91200/1902000] [47.9/1000.0] [batch_t 1.426 (1.425)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-02 23:45:53,165 - ==> Total time: 22:45:32 Eta: 18 days, 19:23:18 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-02 23:46:00,992 - Train: 4.80% [91300/1902000] [48.0/1000.0] [batch_t 1.435 (1.534)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-02 23:48:23,521 - Train: 4.81% [91400/1902000] [48.1/1000.0] [batch_t 1.438 (1.425)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-02 23:50:46,062 - Train: 4.81% [91500/1902000] [48.1/1000.0] [batch_t 1.431 (1.425)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-02 23:53:08,713 - Train: 4.82% [91600/1902000] [48.2/1000.0] [batch_t 1.431 (1.426)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-02 23:55:31,014 - Train: 4.82% [91700/1902000] [48.2/1000.0] [batch_t 1.423 (1.423)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-02 23:57:53,616 - Train: 4.83% [91800/1902000] [48.3/1000.0] [batch_t 1.425 (1.426)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-03 00:00:16,077 - Train: 4.83% [91900/1902000] [48.3/1000.0] [batch_t 1.425 (1.425)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-03 00:02:38,472 - Train: 4.84% [92000/1902000] [48.4/1000.0] [batch_t 1.436 (1.424)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-03 00:05:01,082 - Train: 4.84% [92100/1902000] [48.4/1000.0] [batch_t 1.426 (1.426)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 00:07:23,570 - Train: 4.85% [92200/1902000] [48.5/1000.0] [batch_t 1.426 (1.425)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 00:09:46,191 - Train: 4.85% [92300/1902000] [48.5/1000.0] [batch_t 1.435 (1.426)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 00:12:08,861 - Train: 4.86% [92400/1902000] [48.6/1000.0] [batch_t 1.413 (1.427)] [data_t 0.003] [optim_t 1.410] [lr 0.005000] +2024-04-03 00:14:31,509 - Train: 4.86% [92500/1902000] [48.6/1000.0] [batch_t 1.437 (1.426)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-03 00:16:54,076 - Train: 4.87% [92600/1902000] [48.7/1000.0] [batch_t 1.427 (1.426)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-03 00:19:16,881 - Train: 4.87% [92700/1902000] [48.7/1000.0] [batch_t 1.431 (1.428)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 00:21:39,520 - Train: 4.88% [92800/1902000] [48.8/1000.0] [batch_t 1.437 (1.426)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-03 00:24:02,075 - Train: 4.88% [92900/1902000] [48.8/1000.0] [batch_t 1.424 (1.425)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 00:26:24,740 - Train: 4.89% [93000/1902000] [48.9/1000.0] [batch_t 1.438 (1.427)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-03 00:28:47,479 - Train: 4.89% [93100/1902000] [48.9/1000.0] [batch_t 1.423 (1.427)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 00:31:07,296 - ==> Total time: 23:30:46 Eta: 19 days, 0:20:39 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 00:31:12,094 - Train: 4.90% [93200/1902000] [49.0/1000.0] [batch_t 1.423 (1.549)] [data_t 0.002] [optim_t 1.420] [lr 0.005000] +2024-04-03 00:33:34,747 - Train: 4.91% [93300/1902000] [49.1/1000.0] [batch_t 1.427 (1.426)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 00:35:57,432 - Train: 4.91% [93400/1902000] [49.1/1000.0] [batch_t 1.428 (1.427)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-03 00:38:20,207 - Train: 4.92% [93500/1902000] [49.2/1000.0] [batch_t 1.418 (1.428)] [data_t 0.003] [optim_t 1.416] [lr 0.005000] +2024-04-03 00:40:42,944 - Train: 4.92% [93600/1902000] [49.2/1000.0] [batch_t 1.427 (1.427)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-03 00:43:05,559 - Train: 4.93% [93700/1902000] [49.3/1000.0] [batch_t 1.422 (1.426)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-03 00:45:28,293 - Train: 4.93% [93800/1902000] [49.3/1000.0] [batch_t 1.420 (1.427)] [data_t 0.002] [optim_t 1.417] [lr 0.005000] +2024-04-03 00:47:51,050 - Train: 4.94% [93900/1902000] [49.4/1000.0] [batch_t 1.428 (1.427)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 00:50:13,681 - Train: 4.94% [94000/1902000] [49.4/1000.0] [batch_t 1.424 (1.426)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-03 00:52:36,428 - Train: 4.95% [94100/1902000] [49.5/1000.0] [batch_t 1.446 (1.427)] [data_t 0.002] [optim_t 1.444] [lr 0.005000] +2024-04-03 00:54:59,113 - Train: 4.95% [94200/1902000] [49.5/1000.0] [batch_t 1.432 (1.427)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 00:57:21,809 - Train: 4.96% [94300/1902000] [49.6/1000.0] [batch_t 1.427 (1.427)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-03 00:59:44,555 - Train: 4.96% [94400/1902000] [49.6/1000.0] [batch_t 1.436 (1.427)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 01:02:07,211 - Train: 4.97% [94500/1902000] [49.7/1000.0] [batch_t 1.426 (1.426)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 01:04:29,741 - Train: 4.97% [94600/1902000] [49.7/1000.0] [batch_t 1.433 (1.425)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-03 01:06:52,467 - Train: 4.98% [94700/1902000] [49.8/1000.0] [batch_t 1.429 (1.427)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-03 01:09:15,137 - Train: 4.98% [94800/1902000] [49.8/1000.0] [batch_t 1.432 (1.427)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 01:11:37,766 - Train: 4.99% [94900/1902000] [49.9/1000.0] [batch_t 1.422 (1.426)] [data_t 0.002] [optim_t 1.420] [lr 0.005000] +2024-04-03 01:14:00,481 - Train: 4.99% [95000/1902000] [49.9/1000.0] [batch_t 1.432 (1.427)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-03 01:16:23,104 - Train: 5.00% [95100/1902000] [50.0/1000.0] [batch_t 1.441 (1.426)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-03 01:16:23,108 - ==> Total time: 1 day, 0:16:02 Eta: 19 days, 5:04:49 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 01:18:47,688 - Train: 5.01% [95200/1902000] [50.1/1000.0] [batch_t 1.432 (1.430)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 01:21:10,394 - Train: 5.01% [95300/1902000] [50.1/1000.0] [batch_t 1.432 (1.427)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-03 01:23:33,057 - Train: 5.02% [95400/1902000] [50.2/1000.0] [batch_t 1.441 (1.427)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-03 01:25:55,741 - Train: 5.02% [95500/1902000] [50.2/1000.0] [batch_t 1.437 (1.427)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-03 01:28:18,463 - Train: 5.03% [95600/1902000] [50.3/1000.0] [batch_t 1.425 (1.427)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-03 01:30:41,169 - Train: 5.03% [95700/1902000] [50.3/1000.0] [batch_t 1.431 (1.427)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 01:33:03,813 - Train: 5.04% [95800/1902000] [50.4/1000.0] [batch_t 1.436 (1.426)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 01:35:26,477 - Train: 5.04% [95900/1902000] [50.4/1000.0] [batch_t 1.420 (1.427)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-03 01:37:49,028 - Train: 5.05% [96000/1902000] [50.5/1000.0] [batch_t 1.418 (1.425)] [data_t 0.002] [optim_t 1.415] [lr 0.005000] +2024-04-03 01:40:11,688 - Train: 5.05% [96100/1902000] [50.5/1000.0] [batch_t 1.435 (1.426)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 01:42:34,244 - Train: 5.06% [96200/1902000] [50.6/1000.0] [batch_t 1.434 (1.425)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-03 01:44:56,790 - Train: 5.06% [96300/1902000] [50.6/1000.0] [batch_t 1.423 (1.425)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 01:47:19,423 - Train: 5.07% [96400/1902000] [50.7/1000.0] [batch_t 1.422 (1.426)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-03 01:49:42,034 - Train: 5.07% [96500/1902000] [50.7/1000.0] [batch_t 1.424 (1.426)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-03 01:52:04,612 - Train: 5.08% [96600/1902000] [50.8/1000.0] [batch_t 1.426 (1.426)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 01:54:27,452 - Train: 5.08% [96700/1902000] [50.8/1000.0] [batch_t 1.437 (1.428)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-03 01:56:50,128 - Train: 5.09% [96800/1902000] [50.9/1000.0] [batch_t 1.426 (1.427)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-03 01:59:12,753 - Train: 5.09% [96900/1902000] [50.9/1000.0] [batch_t 1.418 (1.426)] [data_t 0.002] [optim_t 1.415] [lr 0.005000] +2024-04-03 02:01:35,435 - Train: 5.10% [97000/1902000] [51.0/1000.0] [batch_t 1.429 (1.427)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-03 02:01:38,305 - ==> Total time: 1 day, 1:01:17 Eta: 19 days, 9:35:54 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 02:03:59,475 - Train: 5.11% [97100/1902000] [51.1/1000.0] [batch_t 1.423 (1.427)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-03 02:06:22,193 - Train: 5.11% [97200/1902000] [51.1/1000.0] [batch_t 1.427 (1.427)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-03 02:08:44,829 - Train: 5.12% [97300/1902000] [51.2/1000.0] [batch_t 1.426 (1.426)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 02:11:07,534 - Train: 5.12% [97400/1902000] [51.2/1000.0] [batch_t 1.417 (1.427)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-03 02:13:30,199 - Train: 5.13% [97500/1902000] [51.3/1000.0] [batch_t 1.426 (1.427)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 02:15:52,797 - Train: 5.13% [97600/1902000] [51.3/1000.0] [batch_t 1.422 (1.426)] [data_t 0.002] [optim_t 1.420] [lr 0.005000] +2024-04-03 02:18:15,416 - Train: 5.14% [97700/1902000] [51.4/1000.0] [batch_t 1.434 (1.426)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-03 02:20:38,072 - Train: 5.14% [97800/1902000] [51.4/1000.0] [batch_t 1.421 (1.426)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-03 02:23:00,770 - Train: 5.15% [97900/1902000] [51.5/1000.0] [batch_t 1.416 (1.427)] [data_t 0.003] [optim_t 1.413] [lr 0.005000] +2024-04-03 02:25:23,374 - Train: 5.15% [98000/1902000] [51.5/1000.0] [batch_t 1.435 (1.426)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-03 02:27:46,033 - Train: 5.16% [98100/1902000] [51.6/1000.0] [batch_t 1.437 (1.426)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-03 02:30:08,606 - Train: 5.16% [98200/1902000] [51.6/1000.0] [batch_t 1.451 (1.426)] [data_t 0.002] [optim_t 1.449] [lr 0.005000] +2024-04-03 02:32:31,252 - Train: 5.17% [98300/1902000] [51.7/1000.0] [batch_t 1.438 (1.426)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-03 02:34:53,953 - Train: 5.17% [98400/1902000] [51.7/1000.0] [batch_t 1.427 (1.427)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-03 02:37:16,537 - Train: 5.18% [98500/1902000] [51.8/1000.0] [batch_t 1.437 (1.426)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-03 02:39:39,036 - Train: 5.18% [98600/1902000] [51.8/1000.0] [batch_t 1.413 (1.425)] [data_t 0.003] [optim_t 1.411] [lr 0.005000] +2024-04-03 02:42:01,890 - Train: 5.19% [98700/1902000] [51.9/1000.0] [batch_t 1.440 (1.428)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-03 02:44:24,468 - Train: 5.19% [98800/1902000] [51.9/1000.0] [batch_t 1.427 (1.426)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-03 02:46:47,077 - Train: 5.20% [98900/1902000] [52.0/1000.0] [batch_t 1.430 (1.426)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-03 02:46:52,811 - ==> Total time: 1 day, 1:46:32 Eta: 19 days, 13:54:35 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 02:49:12,168 - Train: 5.21% [99000/1902000] [52.1/1000.0] [batch_t 1.422 (1.434)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-03 02:51:34,788 - Train: 5.21% [99100/1902000] [52.1/1000.0] [batch_t 1.431 (1.426)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-03 02:53:57,437 - Train: 5.22% [99200/1902000] [52.2/1000.0] [batch_t 1.430 (1.426)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 02:56:20,176 - Train: 5.22% [99300/1902000] [52.2/1000.0] [batch_t 1.431 (1.427)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 02:58:42,892 - Train: 5.23% [99400/1902000] [52.3/1000.0] [batch_t 1.415 (1.427)] [data_t 0.003] [optim_t 1.412] [lr 0.005000] +2024-04-03 03:01:05,602 - Train: 5.23% [99500/1902000] [52.3/1000.0] [batch_t 1.434 (1.427)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-03 03:03:28,140 - Train: 5.24% [99600/1902000] [52.4/1000.0] [batch_t 1.433 (1.425)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-03 03:05:50,738 - Train: 5.24% [99700/1902000] [52.4/1000.0] [batch_t 1.435 (1.426)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 03:08:13,435 - Train: 5.25% [99800/1902000] [52.5/1000.0] [batch_t 1.428 (1.427)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 03:10:36,141 - Train: 5.25% [99900/1902000] [52.5/1000.0] [batch_t 1.431 (1.427)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 03:12:58,910 - Train: 5.26% [100000/1902000] [52.6/1000.0] [batch_t 1.419 (1.428)] [data_t 0.003] [optim_t 1.416] [lr 0.005000] +2024-04-03 03:15:21,400 - Train: 5.26% [100100/1902000] [52.6/1000.0] [batch_t 1.427 (1.425)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-03 03:17:43,967 - Train: 5.27% [100200/1902000] [52.7/1000.0] [batch_t 1.430 (1.426)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 03:20:06,461 - Train: 5.27% [100300/1902000] [52.7/1000.0] [batch_t 1.426 (1.425)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-03 03:22:29,229 - Train: 5.28% [100400/1902000] [52.8/1000.0] [batch_t 1.432 (1.428)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 03:24:52,001 - Train: 5.28% [100500/1902000] [52.8/1000.0] [batch_t 1.433 (1.428)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-03 03:27:14,675 - Train: 5.29% [100600/1902000] [52.9/1000.0] [batch_t 1.432 (1.427)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 03:29:37,515 - Train: 5.29% [100700/1902000] [52.9/1000.0] [batch_t 1.430 (1.428)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-03 03:32:00,234 - Train: 5.30% [100800/1902000] [53.0/1000.0] [batch_t 1.437 (1.427)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-03 03:32:08,787 - ==> Total time: 1 day, 2:31:48 Eta: 19 days, 18:02:15 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 03:34:24,461 - Train: 5.30% [100900/1902000] [53.0/1000.0] [batch_t 1.419 (1.429)] [data_t 0.003] [optim_t 1.416] [lr 0.005000] +2024-04-03 03:36:47,023 - Train: 5.31% [101000/1902000] [53.1/1000.0] [batch_t 1.425 (1.426)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 03:39:09,560 - Train: 5.32% [101100/1902000] [53.2/1000.0] [batch_t 1.413 (1.425)] [data_t 0.002] [optim_t 1.411] [lr 0.005000] +2024-04-03 03:41:32,230 - Train: 5.32% [101200/1902000] [53.2/1000.0] [batch_t 1.440 (1.427)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-03 03:43:54,843 - Train: 5.33% [101300/1902000] [53.3/1000.0] [batch_t 1.435 (1.426)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-03 03:46:17,513 - Train: 5.33% [101400/1902000] [53.3/1000.0] [batch_t 1.432 (1.427)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 03:48:40,077 - Train: 5.34% [101500/1902000] [53.4/1000.0] [batch_t 1.423 (1.426)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-03 03:51:02,751 - Train: 5.34% [101600/1902000] [53.4/1000.0] [batch_t 1.430 (1.427)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 03:53:25,325 - Train: 5.35% [101700/1902000] [53.5/1000.0] [batch_t 1.422 (1.426)] [data_t 0.002] [optim_t 1.420] [lr 0.005000] +2024-04-03 03:55:48,023 - Train: 5.35% [101800/1902000] [53.5/1000.0] [batch_t 1.426 (1.427)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 03:58:10,680 - Train: 5.36% [101900/1902000] [53.6/1000.0] [batch_t 1.417 (1.426)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-03 04:00:33,273 - Train: 5.36% [102000/1902000] [53.6/1000.0] [batch_t 1.438 (1.426)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-03 04:02:56,079 - Train: 5.37% [102100/1902000] [53.7/1000.0] [batch_t 1.441 (1.428)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-03 04:05:18,618 - Train: 5.37% [102200/1902000] [53.7/1000.0] [batch_t 1.431 (1.425)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 04:07:41,301 - Train: 5.38% [102300/1902000] [53.8/1000.0] [batch_t 1.445 (1.427)] [data_t 0.003] [optim_t 1.443] [lr 0.005000] +2024-04-03 04:10:03,760 - Train: 5.38% [102400/1902000] [53.8/1000.0] [batch_t 1.424 (1.424)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 04:12:26,505 - Train: 5.39% [102500/1902000] [53.9/1000.0] [batch_t 1.453 (1.427)] [data_t 0.003] [optim_t 1.450] [lr 0.005000] +2024-04-03 04:14:49,031 - Train: 5.39% [102600/1902000] [53.9/1000.0] [batch_t 1.426 (1.425)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 04:17:11,536 - Train: 5.40% [102700/1902000] [54.0/1000.0] [batch_t 1.423 (1.425)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-03 04:17:22,955 - ==> Total time: 1 day, 3:17:02 Eta: 19 days, 21:58:32 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 04:19:36,771 - Train: 5.40% [102800/1902000] [54.0/1000.0] [batch_t 1.426 (1.436)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-03 04:21:59,369 - Train: 5.41% [102900/1902000] [54.1/1000.0] [batch_t 1.432 (1.426)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-03 04:24:22,124 - Train: 5.42% [103000/1902000] [54.2/1000.0] [batch_t 1.431 (1.427)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 04:26:44,559 - Train: 5.42% [103100/1902000] [54.2/1000.0] [batch_t 1.418 (1.424)] [data_t 0.003] [optim_t 1.415] [lr 0.005000] +2024-04-03 04:29:07,333 - Train: 5.43% [103200/1902000] [54.3/1000.0] [batch_t 1.445 (1.428)] [data_t 0.002] [optim_t 1.443] [lr 0.005000] +2024-04-03 04:31:29,984 - Train: 5.43% [103300/1902000] [54.3/1000.0] [batch_t 1.436 (1.426)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-03 04:33:52,565 - Train: 5.44% [103400/1902000] [54.4/1000.0] [batch_t 1.427 (1.426)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-03 04:36:15,366 - Train: 5.44% [103500/1902000] [54.4/1000.0] [batch_t 1.440 (1.428)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-03 04:38:37,985 - Train: 5.45% [103600/1902000] [54.5/1000.0] [batch_t 1.447 (1.426)] [data_t 0.003] [optim_t 1.444] [lr 0.005000] +2024-04-03 04:41:00,758 - Train: 5.45% [103700/1902000] [54.5/1000.0] [batch_t 1.431 (1.428)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 04:43:23,390 - Train: 5.46% [103800/1902000] [54.6/1000.0] [batch_t 1.424 (1.426)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 04:45:46,036 - Train: 5.46% [103900/1902000] [54.6/1000.0] [batch_t 1.418 (1.426)] [data_t 0.003] [optim_t 1.415] [lr 0.005000] +2024-04-03 04:48:08,653 - Train: 5.47% [104000/1902000] [54.7/1000.0] [batch_t 1.438 (1.426)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-03 04:50:31,373 - Train: 5.47% [104100/1902000] [54.7/1000.0] [batch_t 1.422 (1.427)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-03 04:52:54,069 - Train: 5.48% [104200/1902000] [54.8/1000.0] [batch_t 1.434 (1.427)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-03 04:55:16,768 - Train: 5.48% [104300/1902000] [54.8/1000.0] [batch_t 1.426 (1.427)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 04:57:39,398 - Train: 5.49% [104400/1902000] [54.9/1000.0] [batch_t 1.425 (1.426)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-03 05:00:02,091 - Train: 5.49% [104500/1902000] [54.9/1000.0] [batch_t 1.419 (1.427)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-03 05:02:24,934 - Train: 5.50% [104600/1902000] [55.0/1000.0] [batch_t 1.428 (1.428)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-03 05:02:39,217 - ==> Total time: 1 day, 4:02:18 Eta: 20 days, 1:45:11 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 05:04:49,672 - Train: 5.50% [104700/1902000] [55.0/1000.0] [batch_t 1.438 (1.431)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-03 05:07:12,404 - Train: 5.51% [104800/1902000] [55.1/1000.0] [batch_t 1.430 (1.427)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 05:09:35,105 - Train: 5.52% [104900/1902000] [55.2/1000.0] [batch_t 1.437 (1.427)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-03 05:11:57,752 - Train: 5.52% [105000/1902000] [55.2/1000.0] [batch_t 1.446 (1.426)] [data_t 0.003] [optim_t 1.443] [lr 0.005000] +2024-04-03 05:14:20,271 - Train: 5.53% [105100/1902000] [55.3/1000.0] [batch_t 1.417 (1.425)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-03 05:16:42,922 - Train: 5.53% [105200/1902000] [55.3/1000.0] [batch_t 1.426 (1.426)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 05:19:05,410 - Train: 5.54% [105300/1902000] [55.4/1000.0] [batch_t 1.436 (1.425)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-03 05:21:27,857 - Train: 5.54% [105400/1902000] [55.4/1000.0] [batch_t 1.426 (1.424)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 05:23:50,358 - Train: 5.55% [105500/1902000] [55.5/1000.0] [batch_t 1.426 (1.425)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-03 05:26:12,914 - Train: 5.55% [105600/1902000] [55.5/1000.0] [batch_t 1.440 (1.425)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-03 05:28:35,555 - Train: 5.56% [105700/1902000] [55.6/1000.0] [batch_t 1.432 (1.426)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 05:30:58,063 - Train: 5.56% [105800/1902000] [55.6/1000.0] [batch_t 1.422 (1.425)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-03 05:33:20,734 - Train: 5.57% [105900/1902000] [55.7/1000.0] [batch_t 1.438 (1.427)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-03 05:35:43,419 - Train: 5.57% [106000/1902000] [55.7/1000.0] [batch_t 1.426 (1.427)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-03 05:38:06,087 - Train: 5.58% [106100/1902000] [55.8/1000.0] [batch_t 1.436 (1.427)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 05:40:28,740 - Train: 5.58% [106200/1902000] [55.8/1000.0] [batch_t 1.427 (1.426)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-03 05:42:51,306 - Train: 5.59% [106300/1902000] [55.9/1000.0] [batch_t 1.434 (1.426)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-03 05:45:13,902 - Train: 5.59% [106400/1902000] [55.9/1000.0] [batch_t 1.434 (1.426)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-03 05:47:36,581 - Train: 5.60% [106500/1902000] [56.0/1000.0] [batch_t 1.422 (1.427)] [data_t 0.002] [optim_t 1.420] [lr 0.005000] +2024-04-03 05:47:53,714 - ==> Total time: 1 day, 4:47:33 Eta: 20 days, 5:21:37 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 05:50:01,394 - Train: 5.60% [106600/1902000] [56.0/1000.0] [batch_t 1.435 (1.430)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-03 05:52:24,019 - Train: 5.61% [106700/1902000] [56.1/1000.0] [batch_t 1.432 (1.426)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 05:54:46,833 - Train: 5.62% [106800/1902000] [56.2/1000.0] [batch_t 1.433 (1.428)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-03 05:57:09,682 - Train: 5.62% [106900/1902000] [56.2/1000.0] [batch_t 1.430 (1.428)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 05:59:32,239 - Train: 5.63% [107000/1902000] [56.3/1000.0] [batch_t 1.432 (1.425)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-03 06:01:54,777 - Train: 5.63% [107100/1902000] [56.3/1000.0] [batch_t 1.422 (1.425)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-03 06:04:17,389 - Train: 5.64% [107200/1902000] [56.4/1000.0] [batch_t 1.421 (1.426)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-03 06:06:39,992 - Train: 5.64% [107300/1902000] [56.4/1000.0] [batch_t 1.431 (1.426)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-03 06:09:02,660 - Train: 5.65% [107400/1902000] [56.5/1000.0] [batch_t 1.442 (1.427)] [data_t 0.003] [optim_t 1.439] [lr 0.005000] +2024-04-03 06:11:25,282 - Train: 5.65% [107500/1902000] [56.5/1000.0] [batch_t 1.441 (1.426)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-03 06:13:48,075 - Train: 5.66% [107600/1902000] [56.6/1000.0] [batch_t 1.422 (1.428)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-03 06:16:10,687 - Train: 5.66% [107700/1902000] [56.6/1000.0] [batch_t 1.432 (1.426)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 06:18:33,376 - Train: 5.67% [107800/1902000] [56.7/1000.0] [batch_t 1.412 (1.427)] [data_t 0.003] [optim_t 1.409] [lr 0.005000] +2024-04-03 06:20:56,034 - Train: 5.67% [107900/1902000] [56.7/1000.0] [batch_t 1.436 (1.426)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-03 06:23:18,653 - Train: 5.68% [108000/1902000] [56.8/1000.0] [batch_t 1.426 (1.426)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 06:25:41,289 - Train: 5.68% [108100/1902000] [56.8/1000.0] [batch_t 1.425 (1.426)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 06:28:03,947 - Train: 5.69% [108200/1902000] [56.9/1000.0] [batch_t 1.426 (1.426)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 06:30:26,716 - Train: 5.69% [108300/1902000] [56.9/1000.0] [batch_t 1.431 (1.428)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 06:32:49,482 - Train: 5.70% [108400/1902000] [57.0/1000.0] [batch_t 1.425 (1.428)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-03 06:33:09,439 - ==> Total time: 1 day, 5:32:48 Eta: 20 days, 8:49:13 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 06:35:14,396 - Train: 5.70% [108500/1902000] [57.0/1000.0] [batch_t 1.432 (1.434)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-03 06:37:37,015 - Train: 5.71% [108600/1902000] [57.1/1000.0] [batch_t 1.422 (1.426)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-03 06:39:59,730 - Train: 5.72% [108700/1902000] [57.2/1000.0] [batch_t 1.440 (1.427)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-03 06:42:22,311 - Train: 5.72% [108800/1902000] [57.2/1000.0] [batch_t 1.427 (1.426)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-03 06:44:44,938 - Train: 5.73% [108900/1902000] [57.3/1000.0] [batch_t 1.421 (1.426)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-03 06:47:07,448 - Train: 5.73% [109000/1902000] [57.3/1000.0] [batch_t 1.435 (1.425)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 06:49:29,974 - Train: 5.74% [109100/1902000] [57.4/1000.0] [batch_t 1.431 (1.425)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 06:51:52,654 - Train: 5.74% [109200/1902000] [57.4/1000.0] [batch_t 1.436 (1.427)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 06:54:15,691 - Train: 5.75% [109300/1902000] [57.5/1000.0] [batch_t 1.422 (1.430)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-03 06:56:38,384 - Train: 5.75% [109400/1902000] [57.5/1000.0] [batch_t 1.428 (1.427)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 06:59:00,995 - Train: 5.76% [109500/1902000] [57.6/1000.0] [batch_t 1.426 (1.426)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-03 07:01:23,691 - Train: 5.76% [109600/1902000] [57.6/1000.0] [batch_t 1.431 (1.427)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 07:03:46,464 - Train: 5.77% [109700/1902000] [57.7/1000.0] [batch_t 1.446 (1.428)] [data_t 0.002] [optim_t 1.444] [lr 0.005000] +2024-04-03 07:06:09,214 - Train: 5.77% [109800/1902000] [57.7/1000.0] [batch_t 1.423 (1.427)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-03 07:08:31,870 - Train: 5.78% [109900/1902000] [57.8/1000.0] [batch_t 1.427 (1.426)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-03 07:10:54,527 - Train: 5.78% [110000/1902000] [57.8/1000.0] [batch_t 1.425 (1.426)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-03 07:13:17,170 - Train: 5.79% [110100/1902000] [57.9/1000.0] [batch_t 1.428 (1.426)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-03 07:15:39,657 - Train: 5.79% [110200/1902000] [57.9/1000.0] [batch_t 1.436 (1.425)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 07:18:02,235 - Train: 5.80% [110300/1902000] [58.0/1000.0] [batch_t 1.430 (1.426)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 07:18:25,056 - ==> Total time: 1 day, 6:18:04 Eta: 20 days, 12:08:03 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 07:20:27,094 - Train: 5.80% [110400/1902000] [58.0/1000.0] [batch_t 1.427 (1.428)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-03 07:22:49,573 - Train: 5.81% [110500/1902000] [58.1/1000.0] [batch_t 1.419 (1.425)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-03 07:25:12,459 - Train: 5.81% [110600/1902000] [58.1/1000.0] [batch_t 1.438 (1.429)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-03 07:27:35,093 - Train: 5.82% [110700/1902000] [58.2/1000.0] [batch_t 1.435 (1.426)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-03 07:29:57,686 - Train: 5.83% [110800/1902000] [58.3/1000.0] [batch_t 1.437 (1.426)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-03 07:32:20,461 - Train: 5.83% [110900/1902000] [58.3/1000.0] [batch_t 1.435 (1.428)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-03 07:34:43,166 - Train: 5.84% [111000/1902000] [58.4/1000.0] [batch_t 1.418 (1.427)] [data_t 0.003] [optim_t 1.415] [lr 0.005000] +2024-04-03 07:37:05,830 - Train: 5.84% [111100/1902000] [58.4/1000.0] [batch_t 1.427 (1.427)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-03 07:39:28,577 - Train: 5.85% [111200/1902000] [58.5/1000.0] [batch_t 1.423 (1.427)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-03 07:41:51,252 - Train: 5.85% [111300/1902000] [58.5/1000.0] [batch_t 1.432 (1.427)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-03 07:44:13,714 - Train: 5.86% [111400/1902000] [58.6/1000.0] [batch_t 1.437 (1.425)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-03 07:46:36,500 - Train: 5.86% [111500/1902000] [58.6/1000.0] [batch_t 1.422 (1.428)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-03 07:48:59,172 - Train: 5.87% [111600/1902000] [58.7/1000.0] [batch_t 1.424 (1.427)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 07:51:21,864 - Train: 5.87% [111700/1902000] [58.7/1000.0] [batch_t 1.416 (1.427)] [data_t 0.003] [optim_t 1.413] [lr 0.005000] +2024-04-03 07:53:44,633 - Train: 5.88% [111800/1902000] [58.8/1000.0] [batch_t 1.424 (1.428)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 07:56:07,321 - Train: 5.88% [111900/1902000] [58.8/1000.0] [batch_t 1.424 (1.427)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-03 07:58:29,975 - Train: 5.89% [112000/1902000] [58.9/1000.0] [batch_t 1.417 (1.426)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-03 08:00:52,663 - Train: 5.89% [112100/1902000] [58.9/1000.0] [batch_t 1.423 (1.427)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-03 08:03:15,243 - Train: 5.90% [112200/1902000] [59.0/1000.0] [batch_t 1.434 (1.426)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-03 08:03:40,946 - ==> Total time: 1 day, 7:03:20 Eta: 20 days, 15:18:42 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 08:05:39,895 - Train: 5.90% [112300/1902000] [59.0/1000.0] [batch_t 1.427 (1.431)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 08:08:02,530 - Train: 5.91% [112400/1902000] [59.1/1000.0] [batch_t 1.429 (1.426)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-03 08:10:25,075 - Train: 5.91% [112500/1902000] [59.1/1000.0] [batch_t 1.439 (1.425)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-03 08:12:47,774 - Train: 5.92% [112600/1902000] [59.2/1000.0] [batch_t 1.422 (1.427)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-03 08:15:10,541 - Train: 5.93% [112700/1902000] [59.3/1000.0] [batch_t 1.433 (1.428)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-03 08:17:33,110 - Train: 5.93% [112800/1902000] [59.3/1000.0] [batch_t 1.426 (1.426)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 08:19:55,668 - Train: 5.94% [112900/1902000] [59.4/1000.0] [batch_t 1.421 (1.425)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-03 08:22:18,209 - Train: 5.94% [113000/1902000] [59.4/1000.0] [batch_t 1.417 (1.425)] [data_t 0.003] [optim_t 1.415] [lr 0.005000] +2024-04-03 08:24:40,845 - Train: 5.95% [113100/1902000] [59.5/1000.0] [batch_t 1.432 (1.426)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 08:27:03,301 - Train: 5.95% [113200/1902000] [59.5/1000.0] [batch_t 1.437 (1.424)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-03 08:29:25,951 - Train: 5.96% [113300/1902000] [59.6/1000.0] [batch_t 1.435 (1.426)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-03 08:31:48,530 - Train: 5.96% [113400/1902000] [59.6/1000.0] [batch_t 1.461 (1.426)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-03 08:34:11,163 - Train: 5.97% [113500/1902000] [59.7/1000.0] [batch_t 1.436 (1.426)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 08:36:33,779 - Train: 5.97% [113600/1902000] [59.7/1000.0] [batch_t 1.430 (1.426)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 08:38:56,263 - Train: 5.98% [113700/1902000] [59.8/1000.0] [batch_t 1.421 (1.425)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-03 08:41:18,738 - Train: 5.98% [113800/1902000] [59.8/1000.0] [batch_t 1.441 (1.425)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-03 08:43:41,321 - Train: 5.99% [113900/1902000] [59.9/1000.0] [batch_t 1.433 (1.426)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-03 08:46:03,790 - Train: 5.99% [114000/1902000] [59.9/1000.0] [batch_t 1.430 (1.425)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 08:48:26,372 - Train: 6.00% [114100/1902000] [60.0/1000.0] [batch_t 1.423 (1.426)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-03 08:48:54,913 - ==> Total time: 1 day, 7:48:34 Eta: 20 days, 18:20:59 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 08:50:50,680 - Train: 6.00% [114200/1902000] [60.0/1000.0] [batch_t 1.426 (1.428)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-03 08:53:13,225 - Train: 6.01% [114300/1902000] [60.1/1000.0] [batch_t 1.417 (1.425)] [data_t 0.002] [optim_t 1.415] [lr 0.005000] +2024-04-03 08:55:35,725 - Train: 6.01% [114400/1902000] [60.1/1000.0] [batch_t 1.426 (1.425)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-03 08:57:58,257 - Train: 6.02% [114500/1902000] [60.2/1000.0] [batch_t 1.423 (1.425)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 09:00:20,767 - Train: 6.03% [114600/1902000] [60.3/1000.0] [batch_t 1.438 (1.425)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-03 09:02:43,499 - Train: 6.03% [114700/1902000] [60.3/1000.0] [batch_t 1.416 (1.427)] [data_t 0.003] [optim_t 1.413] [lr 0.005000] +2024-04-03 09:05:06,012 - Train: 6.04% [114800/1902000] [60.4/1000.0] [batch_t 1.424 (1.425)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 09:07:28,530 - Train: 6.04% [114900/1902000] [60.4/1000.0] [batch_t 1.422 (1.425)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-03 09:09:51,036 - Train: 6.05% [115000/1902000] [60.5/1000.0] [batch_t 1.425 (1.425)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-03 09:12:13,490 - Train: 6.05% [115100/1902000] [60.5/1000.0] [batch_t 1.417 (1.424)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-03 09:14:35,998 - Train: 6.06% [115200/1902000] [60.6/1000.0] [batch_t 1.422 (1.425)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-03 09:16:58,443 - Train: 6.06% [115300/1902000] [60.6/1000.0] [batch_t 1.429 (1.424)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-03 09:19:20,996 - Train: 6.07% [115400/1902000] [60.7/1000.0] [batch_t 1.432 (1.425)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 09:21:43,401 - Train: 6.07% [115500/1902000] [60.7/1000.0] [batch_t 1.426 (1.424)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-03 09:24:05,772 - Train: 6.08% [115600/1902000] [60.8/1000.0] [batch_t 1.431 (1.424)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-03 09:26:28,252 - Train: 6.08% [115700/1902000] [60.8/1000.0] [batch_t 1.419 (1.425)] [data_t 0.002] [optim_t 1.417] [lr 0.005000] +2024-04-03 09:28:50,594 - Train: 6.09% [115800/1902000] [60.9/1000.0] [batch_t 1.437 (1.423)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-03 09:31:12,791 - Train: 6.09% [115900/1902000] [60.9/1000.0] [batch_t 1.426 (1.422)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-03 09:33:35,065 - Train: 6.10% [116000/1902000] [61.0/1000.0] [batch_t 1.434 (1.423)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-03 09:34:06,431 - ==> Total time: 1 day, 8:33:45 Eta: 20 days, 21:15:10 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 09:35:59,167 - Train: 6.10% [116100/1902000] [61.0/1000.0] [batch_t 1.440 (1.428)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-03 09:38:21,625 - Train: 6.11% [116200/1902000] [61.1/1000.0] [batch_t 1.431 (1.424)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 09:40:44,047 - Train: 6.11% [116300/1902000] [61.1/1000.0] [batch_t 1.424 (1.424)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 09:43:06,535 - Train: 6.12% [116400/1902000] [61.2/1000.0] [batch_t 1.411 (1.425)] [data_t 0.003] [optim_t 1.409] [lr 0.005000] +2024-04-03 09:45:28,919 - Train: 6.13% [116500/1902000] [61.3/1000.0] [batch_t 1.429 (1.424)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 09:47:51,328 - Train: 6.13% [116600/1902000] [61.3/1000.0] [batch_t 1.423 (1.424)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-03 09:50:13,645 - Train: 6.14% [116700/1902000] [61.4/1000.0] [batch_t 1.437 (1.423)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-03 09:52:36,127 - Train: 6.14% [116800/1902000] [61.4/1000.0] [batch_t 1.419 (1.425)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-03 09:54:58,587 - Train: 6.15% [116900/1902000] [61.5/1000.0] [batch_t 1.435 (1.425)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-03 09:57:20,869 - Train: 6.15% [117000/1902000] [61.5/1000.0] [batch_t 1.419 (1.423)] [data_t 0.002] [optim_t 1.417] [lr 0.005000] +2024-04-03 09:59:43,168 - Train: 6.16% [117100/1902000] [61.6/1000.0] [batch_t 1.427 (1.423)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-03 10:02:05,454 - Train: 6.16% [117200/1902000] [61.6/1000.0] [batch_t 1.432 (1.423)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-03 10:04:27,889 - Train: 6.17% [117300/1902000] [61.7/1000.0] [batch_t 1.436 (1.424)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 10:06:50,389 - Train: 6.17% [117400/1902000] [61.7/1000.0] [batch_t 1.429 (1.425)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-03 10:09:12,872 - Train: 6.18% [117500/1902000] [61.8/1000.0] [batch_t 1.424 (1.425)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-03 10:11:35,335 - Train: 6.18% [117600/1902000] [61.8/1000.0] [batch_t 1.432 (1.425)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-03 10:13:58,308 - Train: 6.19% [117700/1902000] [61.9/1000.0] [batch_t 1.432 (1.430)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 10:16:25,661 - Train: 6.19% [117800/1902000] [61.9/1000.0] [batch_t 1.455 (1.473)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-03 10:18:50,575 - Train: 6.20% [117900/1902000] [62.0/1000.0] [batch_t 1.446 (1.449)] [data_t 0.003] [optim_t 1.443] [lr 0.005000] +2024-04-03 10:19:25,398 - ==> Total time: 1 day, 9:19:04 Eta: 21 days, 0:04:10 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 10:21:37,877 - Train: 6.20% [118000/1902000] [62.0/1000.0] [batch_t 1.446 (1.724)] [data_t 0.003] [optim_t 1.443] [lr 0.005000] +2024-04-03 10:24:02,820 - Train: 6.21% [118100/1902000] [62.1/1000.0] [batch_t 1.456 (1.449)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-03 10:27:32,834 - Train: 6.21% [118200/1902000] [62.1/1000.0] [batch_t 1.450 (2.100)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-03 10:31:01,875 - Train: 6.22% [118300/1902000] [62.2/1000.0] [batch_t 1.444 (2.090)] [data_t 0.003] [optim_t 1.441] [lr 0.005000] +2024-04-03 10:33:34,965 - Train: 6.23% [118400/1902000] [62.3/1000.0] [batch_t 1.449 (1.531)] [data_t 0.003] [optim_t 1.446] [lr 0.005000] +2024-04-03 10:36:05,445 - Train: 6.23% [118500/1902000] [62.3/1000.0] [batch_t 1.455 (1.505)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-03 10:38:30,380 - Train: 6.24% [118600/1902000] [62.4/1000.0] [batch_t 1.450 (1.449)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-03 10:40:55,349 - Train: 6.24% [118700/1902000] [62.4/1000.0] [batch_t 1.451 (1.450)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-03 10:43:20,254 - Train: 6.25% [118800/1902000] [62.5/1000.0] [batch_t 1.455 (1.449)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-03 10:45:45,170 - Train: 6.25% [118900/1902000] [62.5/1000.0] [batch_t 1.445 (1.449)] [data_t 0.003] [optim_t 1.442] [lr 0.005000] +2024-04-03 10:48:09,304 - Train: 6.26% [119000/1902000] [62.6/1000.0] [batch_t 1.431 (1.441)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 10:50:34,638 - Train: 6.26% [119100/1902000] [62.6/1000.0] [batch_t 1.427 (1.453)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-03 10:52:56,851 - Train: 6.27% [119200/1902000] [62.7/1000.0] [batch_t 1.417 (1.422)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-03 10:55:19,279 - Train: 6.27% [119300/1902000] [62.7/1000.0] [batch_t 1.424 (1.424)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-03 10:57:41,451 - Train: 6.28% [119400/1902000] [62.8/1000.0] [batch_t 1.428 (1.422)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 11:00:03,826 - Train: 6.28% [119500/1902000] [62.8/1000.0] [batch_t 1.419 (1.424)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-03 11:02:26,305 - Train: 6.29% [119600/1902000] [62.9/1000.0] [batch_t 1.421 (1.425)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-03 11:04:48,772 - Train: 6.29% [119700/1902000] [62.9/1000.0] [batch_t 1.430 (1.425)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 11:07:11,152 - Train: 6.30% [119800/1902000] [63.0/1000.0] [batch_t 1.431 (1.424)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 11:07:48,257 - ==> Total time: 1 day, 10:07:27 Eta: 21 days, 3:31:56 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 11:09:36,485 - Train: 6.30% [119900/1902000] [63.0/1000.0] [batch_t 1.435 (1.435)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-03 11:11:58,867 - Train: 6.31% [120000/1902000] [63.1/1000.0] [batch_t 1.430 (1.424)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 11:14:21,274 - Train: 6.31% [120100/1902000] [63.1/1000.0] [batch_t 1.423 (1.424)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-03 11:16:43,623 - Train: 6.32% [120200/1902000] [63.2/1000.0] [batch_t 1.418 (1.423)] [data_t 0.003] [optim_t 1.415] [lr 0.005000] +2024-04-03 11:19:06,042 - Train: 6.32% [120300/1902000] [63.2/1000.0] [batch_t 1.437 (1.424)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-03 11:21:28,352 - Train: 6.33% [120400/1902000] [63.3/1000.0] [batch_t 1.426 (1.423)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-03 11:23:50,769 - Train: 6.34% [120500/1902000] [63.4/1000.0] [batch_t 1.423 (1.424)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-03 11:26:13,474 - Train: 6.34% [120600/1902000] [63.4/1000.0] [batch_t 1.425 (1.427)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-03 11:28:36,101 - Train: 6.35% [120700/1902000] [63.5/1000.0] [batch_t 1.432 (1.426)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-03 11:30:58,546 - Train: 6.35% [120800/1902000] [63.5/1000.0] [batch_t 1.424 (1.424)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 11:33:21,145 - Train: 6.36% [120900/1902000] [63.6/1000.0] [batch_t 1.420 (1.426)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-03 11:35:43,542 - Train: 6.36% [121000/1902000] [63.6/1000.0] [batch_t 1.431 (1.424)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 11:38:06,066 - Train: 6.37% [121100/1902000] [63.7/1000.0] [batch_t 1.420 (1.425)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-03 11:40:28,325 - Train: 6.37% [121200/1902000] [63.7/1000.0] [batch_t 1.436 (1.422)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 11:56:36,624 - Train: 6.38% [121300/1902000] [63.8/1000.0] [batch_t 7.752 (9.683)] [data_t 6.322] [optim_t 1.430] [lr 0.005000] +2024-04-03 11:59:35,337 - Train: 6.38% [121400/1902000] [63.8/1000.0] [batch_t 1.421 (1.787)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-03 12:01:57,692 - Train: 6.39% [121500/1902000] [63.9/1000.0] [batch_t 1.425 (1.423)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-03 12:04:20,021 - Train: 6.39% [121600/1902000] [63.9/1000.0] [batch_t 1.433 (1.423)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-03 12:06:42,370 - Train: 6.40% [121700/1902000] [64.0/1000.0] [batch_t 1.427 (1.423)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-03 12:07:22,252 - ==> Total time: 1 day, 11:07:01 Eta: 21 days, 9:35:18 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 12:09:09,034 - Train: 6.40% [121800/1902000] [64.0/1000.0] [batch_t 1.424 (1.452)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 12:11:31,423 - Train: 6.41% [121900/1902000] [64.1/1000.0] [batch_t 1.427 (1.424)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-03 12:13:53,679 - Train: 6.41% [122000/1902000] [64.1/1000.0] [batch_t 1.432 (1.422)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 12:16:16,147 - Train: 6.42% [122100/1902000] [64.2/1000.0] [batch_t 1.426 (1.425)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-03 12:18:38,545 - Train: 6.42% [122200/1902000] [64.2/1000.0] [batch_t 1.445 (1.424)] [data_t 0.002] [optim_t 1.442] [lr 0.005000] +2024-04-03 12:21:00,969 - Train: 6.43% [122300/1902000] [64.3/1000.0] [batch_t 1.412 (1.424)] [data_t 0.002] [optim_t 1.410] [lr 0.005000] +2024-04-03 12:23:23,403 - Train: 6.44% [122400/1902000] [64.4/1000.0] [batch_t 1.415 (1.424)] [data_t 0.003] [optim_t 1.412] [lr 0.005000] +2024-04-03 12:25:45,695 - Train: 6.44% [122500/1902000] [64.4/1000.0] [batch_t 1.415 (1.423)] [data_t 0.003] [optim_t 1.412] [lr 0.005000] +2024-04-03 12:28:07,926 - Train: 6.45% [122600/1902000] [64.5/1000.0] [batch_t 1.411 (1.422)] [data_t 0.003] [optim_t 1.408] [lr 0.005000] +2024-04-03 12:30:30,156 - Train: 6.45% [122700/1902000] [64.5/1000.0] [batch_t 1.435 (1.422)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-03 12:32:52,755 - Train: 6.46% [122800/1902000] [64.6/1000.0] [batch_t 1.431 (1.426)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 12:35:15,116 - Train: 6.46% [122900/1902000] [64.6/1000.0] [batch_t 1.437 (1.424)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-03 12:37:37,570 - Train: 6.47% [123000/1902000] [64.7/1000.0] [batch_t 1.421 (1.424)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-03 12:40:00,000 - Train: 6.47% [123100/1902000] [64.7/1000.0] [batch_t 1.419 (1.424)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-03 12:42:22,147 - Train: 6.48% [123200/1902000] [64.8/1000.0] [batch_t 1.424 (1.421)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 12:44:44,517 - Train: 6.48% [123300/1902000] [64.8/1000.0] [batch_t 1.420 (1.424)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-03 12:47:06,747 - Train: 6.49% [123400/1902000] [64.9/1000.0] [batch_t 1.425 (1.422)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 12:49:29,123 - Train: 6.49% [123500/1902000] [64.9/1000.0] [batch_t 1.408 (1.424)] [data_t 0.002] [optim_t 1.405] [lr 0.005000] +2024-04-03 12:51:51,533 - Train: 6.50% [123600/1902000] [65.0/1000.0] [batch_t 1.408 (1.424)] [data_t 0.003] [optim_t 1.405] [lr 0.005000] +2024-04-03 12:52:34,265 - ==> Total time: 1 day, 11:52:13 Eta: 21 days, 11:58:59 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 12:54:15,721 - Train: 6.50% [123700/1902000] [65.0/1000.0] [batch_t 1.432 (1.427)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 12:56:38,095 - Train: 6.51% [123800/1902000] [65.1/1000.0] [batch_t 1.431 (1.424)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 12:59:00,544 - Train: 6.51% [123900/1902000] [65.1/1000.0] [batch_t 1.417 (1.424)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-03 13:01:22,967 - Train: 6.52% [124000/1902000] [65.2/1000.0] [batch_t 1.420 (1.424)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-03 13:03:45,285 - Train: 6.52% [124100/1902000] [65.2/1000.0] [batch_t 1.435 (1.423)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 13:06:07,905 - Train: 6.53% [124200/1902000] [65.3/1000.0] [batch_t 1.419 (1.426)] [data_t 0.003] [optim_t 1.416] [lr 0.005000] +2024-04-03 13:08:30,231 - Train: 6.54% [124300/1902000] [65.4/1000.0] [batch_t 1.432 (1.423)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-03 13:10:52,550 - Train: 6.54% [124400/1902000] [65.4/1000.0] [batch_t 1.434 (1.423)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-03 13:13:14,830 - Train: 6.55% [124500/1902000] [65.5/1000.0] [batch_t 1.420 (1.423)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-03 13:15:37,329 - Train: 6.55% [124600/1902000] [65.5/1000.0] [batch_t 1.426 (1.425)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-03 13:17:59,605 - Train: 6.56% [124700/1902000] [65.6/1000.0] [batch_t 1.429 (1.423)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 13:20:22,060 - Train: 6.56% [124800/1902000] [65.6/1000.0] [batch_t 1.425 (1.424)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-03 13:22:44,258 - Train: 6.57% [124900/1902000] [65.7/1000.0] [batch_t 1.420 (1.422)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-03 13:25:06,699 - Train: 6.57% [125000/1902000] [65.7/1000.0] [batch_t 1.422 (1.424)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-03 13:27:29,160 - Train: 6.58% [125100/1902000] [65.8/1000.0] [batch_t 1.436 (1.425)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 13:29:51,554 - Train: 6.58% [125200/1902000] [65.8/1000.0] [batch_t 1.420 (1.424)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-03 13:32:14,057 - Train: 6.59% [125300/1902000] [65.9/1000.0] [batch_t 1.417 (1.425)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-03 13:34:36,473 - Train: 6.59% [125400/1902000] [65.9/1000.0] [batch_t 1.435 (1.424)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 13:36:58,911 - Train: 6.60% [125500/1902000] [66.0/1000.0] [batch_t 1.418 (1.424)] [data_t 0.003] [optim_t 1.415] [lr 0.005000] +2024-04-03 13:37:44,429 - ==> Total time: 1 day, 12:37:23 Eta: 21 days, 14:16:31 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 13:39:22,906 - Train: 6.60% [125600/1902000] [66.0/1000.0] [batch_t 1.430 (1.428)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 13:41:45,389 - Train: 6.61% [125700/1902000] [66.1/1000.0] [batch_t 1.421 (1.425)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-03 13:44:07,783 - Train: 6.61% [125800/1902000] [66.1/1000.0] [batch_t 1.424 (1.424)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 13:46:30,164 - Train: 6.62% [125900/1902000] [66.2/1000.0] [batch_t 1.426 (1.424)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-03 13:48:52,590 - Train: 6.62% [126000/1902000] [66.2/1000.0] [batch_t 1.424 (1.424)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-03 13:51:14,987 - Train: 6.63% [126100/1902000] [66.3/1000.0] [batch_t 1.425 (1.424)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-03 13:53:37,391 - Train: 6.64% [126200/1902000] [66.4/1000.0] [batch_t 1.444 (1.424)] [data_t 0.002] [optim_t 1.441] [lr 0.005000] +2024-04-03 13:55:59,720 - Train: 6.64% [126300/1902000] [66.4/1000.0] [batch_t 1.425 (1.423)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-03 13:58:22,074 - Train: 6.65% [126400/1902000] [66.5/1000.0] [batch_t 1.426 (1.423)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 14:00:44,436 - Train: 6.65% [126500/1902000] [66.5/1000.0] [batch_t 1.441 (1.424)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-03 14:03:06,689 - Train: 6.66% [126600/1902000] [66.6/1000.0] [batch_t 1.425 (1.422)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-03 14:05:29,163 - Train: 6.66% [126700/1902000] [66.6/1000.0] [batch_t 1.436 (1.425)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 14:07:51,600 - Train: 6.67% [126800/1902000] [66.7/1000.0] [batch_t 1.429 (1.424)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-03 14:10:13,954 - Train: 6.67% [126900/1902000] [66.7/1000.0] [batch_t 1.419 (1.423)] [data_t 0.002] [optim_t 1.417] [lr 0.005000] +2024-04-03 14:12:36,202 - Train: 6.68% [127000/1902000] [66.8/1000.0] [batch_t 1.429 (1.422)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-03 14:14:58,405 - Train: 6.68% [127100/1902000] [66.8/1000.0] [batch_t 1.420 (1.422)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-03 14:21:28,186 - Train: 6.69% [127200/1902000] [66.9/1000.0] [batch_t 1.420 (3.898)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-03 14:23:50,403 - Train: 6.69% [127300/1902000] [66.9/1000.0] [batch_t 1.431 (1.422)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-03 14:26:12,843 - Train: 6.70% [127400/1902000] [67.0/1000.0] [batch_t 1.424 (1.424)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-03 14:27:01,124 - ==> Total time: 1 day, 13:26:40 Eta: 21 days, 17:25:49 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 14:28:39,185 - Train: 6.70% [127500/1902000] [67.0/1000.0] [batch_t 1.444 (1.435)] [data_t 0.003] [optim_t 1.441] [lr 0.005000] +2024-04-03 14:31:01,478 - Train: 6.71% [127600/1902000] [67.1/1000.0] [batch_t 1.424 (1.423)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 14:42:13,029 - Train: 6.71% [127700/1902000] [67.1/1000.0] [batch_t 1.421 (6.715)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-03 14:44:35,386 - Train: 6.72% [127800/1902000] [67.2/1000.0] [batch_t 1.425 (1.423)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-03 14:46:57,528 - Train: 6.72% [127900/1902000] [67.2/1000.0] [batch_t 1.415 (1.421)] [data_t 0.003] [optim_t 1.412] [lr 0.005000] +2024-04-03 14:49:19,861 - Train: 6.73% [128000/1902000] [67.3/1000.0] [batch_t 1.445 (1.423)] [data_t 0.002] [optim_t 1.443] [lr 0.005000] +2024-04-03 14:51:42,151 - Train: 6.74% [128100/1902000] [67.4/1000.0] [batch_t 1.438 (1.423)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-03 14:54:04,501 - Train: 6.74% [128200/1902000] [67.4/1000.0] [batch_t 1.441 (1.423)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-03 14:56:26,916 - Train: 6.75% [128300/1902000] [67.5/1000.0] [batch_t 1.421 (1.424)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-03 14:58:49,263 - Train: 6.75% [128400/1902000] [67.5/1000.0] [batch_t 1.431 (1.423)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 15:01:11,669 - Train: 6.76% [128500/1902000] [67.6/1000.0] [batch_t 1.420 (1.424)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-03 15:03:33,984 - Train: 6.76% [128600/1902000] [67.6/1000.0] [batch_t 1.416 (1.423)] [data_t 0.002] [optim_t 1.414] [lr 0.005000] +2024-04-03 15:05:56,316 - Train: 6.77% [128700/1902000] [67.7/1000.0] [batch_t 1.430 (1.423)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 15:08:18,741 - Train: 6.77% [128800/1902000] [67.7/1000.0] [batch_t 1.432 (1.424)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-03 15:10:41,103 - Train: 6.78% [128900/1902000] [67.8/1000.0] [batch_t 1.442 (1.424)] [data_t 0.003] [optim_t 1.439] [lr 0.005000] +2024-04-03 15:13:03,457 - Train: 6.78% [129000/1902000] [67.8/1000.0] [batch_t 1.431 (1.423)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 15:15:25,751 - Train: 6.79% [129100/1902000] [67.9/1000.0] [batch_t 1.421 (1.423)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-03 15:17:48,155 - Train: 6.79% [129200/1902000] [67.9/1000.0] [batch_t 1.423 (1.424)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-03 15:20:10,493 - Train: 6.80% [129300/1902000] [68.0/1000.0] [batch_t 1.428 (1.423)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-03 15:21:01,786 - ==> Total time: 1 day, 14:20:41 Eta: 21 days, 21:32:57 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 15:22:36,909 - Train: 6.80% [129400/1902000] [68.0/1000.0] [batch_t 1.424 (1.432)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-03 15:24:59,294 - Train: 6.81% [129500/1902000] [68.1/1000.0] [batch_t 1.425 (1.424)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 15:27:21,771 - Train: 6.81% [129600/1902000] [68.1/1000.0] [batch_t 1.422 (1.425)] [data_t 0.002] [optim_t 1.420] [lr 0.005000] +2024-04-03 15:29:44,147 - Train: 6.82% [129700/1902000] [68.2/1000.0] [batch_t 1.425 (1.424)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-03 15:32:06,610 - Train: 6.82% [129800/1902000] [68.2/1000.0] [batch_t 1.428 (1.425)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 15:34:28,937 - Train: 6.83% [129900/1902000] [68.3/1000.0] [batch_t 1.424 (1.423)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 15:36:51,235 - Train: 6.83% [130000/1902000] [68.3/1000.0] [batch_t 1.421 (1.423)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-03 15:39:13,789 - Train: 6.84% [130100/1902000] [68.4/1000.0] [batch_t 1.413 (1.425)] [data_t 0.003] [optim_t 1.410] [lr 0.005000] +2024-04-03 15:41:36,299 - Train: 6.85% [130200/1902000] [68.5/1000.0] [batch_t 1.424 (1.425)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 15:43:58,649 - Train: 6.85% [130300/1902000] [68.5/1000.0] [batch_t 1.416 (1.423)] [data_t 0.003] [optim_t 1.413] [lr 0.005000] +2024-04-03 15:46:20,961 - Train: 6.86% [130400/1902000] [68.6/1000.0] [batch_t 1.438 (1.423)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-03 15:48:43,352 - Train: 6.86% [130500/1902000] [68.6/1000.0] [batch_t 1.416 (1.424)] [data_t 0.003] [optim_t 1.413] [lr 0.005000] +2024-04-03 15:51:05,721 - Train: 6.87% [130600/1902000] [68.7/1000.0] [batch_t 1.427 (1.424)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-03 15:53:28,009 - Train: 6.87% [130700/1902000] [68.7/1000.0] [batch_t 1.431 (1.423)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-03 15:55:50,477 - Train: 6.88% [130800/1902000] [68.8/1000.0] [batch_t 1.440 (1.425)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-03 15:58:12,873 - Train: 6.88% [130900/1902000] [68.8/1000.0] [batch_t 1.421 (1.424)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-03 16:00:35,318 - Train: 6.89% [131000/1902000] [68.9/1000.0] [batch_t 1.424 (1.424)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-03 16:02:57,671 - Train: 6.89% [131100/1902000] [68.9/1000.0] [batch_t 1.423 (1.423)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-03 16:05:20,004 - Train: 6.90% [131200/1902000] [69.0/1000.0] [batch_t 1.436 (1.423)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 16:06:14,136 - ==> Total time: 1 day, 15:05:53 Eta: 21 days, 23:32:34 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 16:07:43,924 - Train: 6.90% [131300/1902000] [69.0/1000.0] [batch_t 1.429 (1.428)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-03 16:10:06,158 - Train: 6.91% [131400/1902000] [69.1/1000.0] [batch_t 1.434 (1.422)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-03 16:12:28,470 - Train: 6.91% [131500/1902000] [69.1/1000.0] [batch_t 1.409 (1.423)] [data_t 0.003] [optim_t 1.406] [lr 0.005000] +2024-04-03 16:14:50,837 - Train: 6.92% [131600/1902000] [69.2/1000.0] [batch_t 1.414 (1.424)] [data_t 0.002] [optim_t 1.412] [lr 0.005000] +2024-04-03 16:17:13,116 - Train: 6.92% [131700/1902000] [69.2/1000.0] [batch_t 1.435 (1.423)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-03 16:19:35,367 - Train: 6.93% [131800/1902000] [69.3/1000.0] [batch_t 1.422 (1.422)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-03 16:21:57,861 - Train: 6.93% [131900/1902000] [69.3/1000.0] [batch_t 1.440 (1.425)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-03 16:24:20,199 - Train: 6.94% [132000/1902000] [69.4/1000.0] [batch_t 1.435 (1.423)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 16:26:42,587 - Train: 6.95% [132100/1902000] [69.5/1000.0] [batch_t 1.417 (1.424)] [data_t 0.002] [optim_t 1.415] [lr 0.005000] +2024-04-03 16:29:04,912 - Train: 6.95% [132200/1902000] [69.5/1000.0] [batch_t 1.434 (1.423)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-03 16:31:27,494 - Train: 6.96% [132300/1902000] [69.6/1000.0] [batch_t 1.424 (1.426)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-03 16:33:49,803 - Train: 6.96% [132400/1902000] [69.6/1000.0] [batch_t 1.424 (1.423)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 16:36:12,152 - Train: 6.97% [132500/1902000] [69.7/1000.0] [batch_t 1.430 (1.423)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 16:38:34,567 - Train: 6.97% [132600/1902000] [69.7/1000.0] [batch_t 1.433 (1.424)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-03 16:40:56,929 - Train: 6.98% [132700/1902000] [69.8/1000.0] [batch_t 1.429 (1.424)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-03 16:43:19,444 - Train: 6.98% [132800/1902000] [69.8/1000.0] [batch_t 1.423 (1.425)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-03 16:45:41,614 - Train: 6.99% [132900/1902000] [69.9/1000.0] [batch_t 1.416 (1.422)] [data_t 0.003] [optim_t 1.413] [lr 0.005000] +2024-04-03 16:48:03,925 - Train: 6.99% [133000/1902000] [69.9/1000.0] [batch_t 1.431 (1.423)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-03 16:50:26,267 - Train: 7.00% [133100/1902000] [70.0/1000.0] [batch_t 1.431 (1.423)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 16:51:23,210 - ==> Total time: 1 day, 15:51:02 Eta: 22 days, 1:26:44 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 16:52:51,027 - Train: 7.00% [133200/1902000] [70.0/1000.0] [batch_t 1.432 (1.433)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-03 16:55:13,366 - Train: 7.01% [133300/1902000] [70.1/1000.0] [batch_t 1.435 (1.423)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 16:57:35,651 - Train: 7.01% [133400/1902000] [70.1/1000.0] [batch_t 1.425 (1.423)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-03 16:59:58,204 - Train: 7.02% [133500/1902000] [70.2/1000.0] [batch_t 1.428 (1.425)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-03 17:02:20,614 - Train: 7.02% [133600/1902000] [70.2/1000.0] [batch_t 1.423 (1.424)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-03 17:04:42,915 - Train: 7.03% [133700/1902000] [70.3/1000.0] [batch_t 1.415 (1.423)] [data_t 0.003] [optim_t 1.412] [lr 0.005000] +2024-04-03 17:07:05,275 - Train: 7.03% [133800/1902000] [70.3/1000.0] [batch_t 1.430 (1.424)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 17:09:27,546 - Train: 7.04% [133900/1902000] [70.4/1000.0] [batch_t 1.425 (1.423)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-03 17:11:49,772 - Train: 7.05% [134000/1902000] [70.5/1000.0] [batch_t 1.428 (1.422)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-03 17:14:12,195 - Train: 7.05% [134100/1902000] [70.5/1000.0] [batch_t 1.427 (1.424)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-03 17:16:34,527 - Train: 7.06% [134200/1902000] [70.6/1000.0] [batch_t 1.430 (1.423)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 17:18:56,922 - Train: 7.06% [134300/1902000] [70.6/1000.0] [batch_t 1.420 (1.424)] [data_t 0.002] [optim_t 1.417] [lr 0.005000] +2024-04-03 17:21:19,388 - Train: 7.07% [134400/1902000] [70.7/1000.0] [batch_t 1.418 (1.425)] [data_t 0.002] [optim_t 1.416] [lr 0.005000] +2024-04-03 17:23:41,824 - Train: 7.07% [134500/1902000] [70.7/1000.0] [batch_t 1.428 (1.424)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-03 17:26:13,117 - Train: 7.08% [134600/1902000] [70.8/1000.0] [batch_t 1.439 (1.513)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-03 17:28:35,313 - Train: 7.08% [134700/1902000] [70.8/1000.0] [batch_t 1.414 (1.422)] [data_t 0.002] [optim_t 1.412] [lr 0.005000] +2024-04-03 17:30:57,829 - Train: 7.09% [134800/1902000] [70.9/1000.0] [batch_t 1.421 (1.425)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-03 17:33:20,158 - Train: 7.09% [134900/1902000] [70.9/1000.0] [batch_t 1.423 (1.423)] [data_t 0.002] [optim_t 1.420] [lr 0.005000] +2024-04-03 17:35:42,445 - Train: 7.10% [135000/1902000] [71.0/1000.0] [batch_t 1.431 (1.423)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 17:36:42,203 - ==> Total time: 1 day, 16:36:21 Eta: 22 days, 3:18:35 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 17:38:07,122 - Train: 7.10% [135100/1902000] [71.0/1000.0] [batch_t 1.430 (1.435)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 17:40:29,475 - Train: 7.11% [135200/1902000] [71.1/1000.0] [batch_t 1.420 (1.423)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-03 17:42:51,734 - Train: 7.11% [135300/1902000] [71.1/1000.0] [batch_t 1.420 (1.423)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-03 17:45:13,997 - Train: 7.12% [135400/1902000] [71.2/1000.0] [batch_t 1.428 (1.423)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-03 17:47:36,444 - Train: 7.12% [135500/1902000] [71.2/1000.0] [batch_t 1.428 (1.424)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 17:49:58,894 - Train: 7.13% [135600/1902000] [71.3/1000.0] [batch_t 1.421 (1.424)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-03 17:52:45,813 - Train: 7.13% [135700/1902000] [71.3/1000.0] [batch_t 1.434 (1.669)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-03 17:55:08,254 - Train: 7.14% [135800/1902000] [71.4/1000.0] [batch_t 1.416 (1.424)] [data_t 0.002] [optim_t 1.413] [lr 0.005000] +2024-04-03 17:57:30,594 - Train: 7.15% [135900/1902000] [71.5/1000.0] [batch_t 1.412 (1.423)] [data_t 0.003] [optim_t 1.409] [lr 0.005000] +2024-04-03 17:59:52,752 - Train: 7.15% [136000/1902000] [71.5/1000.0] [batch_t 1.422 (1.421)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-03 18:02:15,166 - Train: 7.16% [136100/1902000] [71.6/1000.0] [batch_t 1.427 (1.424)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-03 18:04:37,451 - Train: 7.16% [136200/1902000] [71.6/1000.0] [batch_t 1.442 (1.423)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-03 18:06:59,764 - Train: 7.17% [136300/1902000] [71.7/1000.0] [batch_t 1.439 (1.423)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-03 18:09:22,141 - Train: 7.17% [136400/1902000] [71.7/1000.0] [batch_t 1.430 (1.424)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 18:11:44,377 - Train: 7.18% [136500/1902000] [71.8/1000.0] [batch_t 1.414 (1.422)] [data_t 0.003] [optim_t 1.411] [lr 0.005000] +2024-04-03 18:14:06,858 - Train: 7.18% [136600/1902000] [71.8/1000.0] [batch_t 1.427 (1.425)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-03 18:16:29,298 - Train: 7.19% [136700/1902000] [71.9/1000.0] [batch_t 1.421 (1.424)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-03 18:18:51,717 - Train: 7.19% [136800/1902000] [71.9/1000.0] [batch_t 1.425 (1.424)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-03 18:21:14,073 - Train: 7.20% [136900/1902000] [72.0/1000.0] [batch_t 1.430 (1.423)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-03 18:22:16,723 - ==> Total time: 1 day, 17:21:56 Eta: 22 days, 5:09:24 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 18:23:37,761 - Train: 7.20% [137000/1902000] [72.0/1000.0] [batch_t 1.416 (1.424)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-03 18:26:00,035 - Train: 7.21% [137100/1902000] [72.1/1000.0] [batch_t 1.420 (1.423)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-03 18:28:22,408 - Train: 7.21% [137200/1902000] [72.1/1000.0] [batch_t 1.441 (1.424)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-03 18:30:44,649 - Train: 7.22% [137300/1902000] [72.2/1000.0] [batch_t 1.421 (1.422)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-03 18:33:07,085 - Train: 7.22% [137400/1902000] [72.2/1000.0] [batch_t 1.424 (1.424)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-03 18:35:29,341 - Train: 7.23% [137500/1902000] [72.3/1000.0] [batch_t 1.417 (1.422)] [data_t 0.002] [optim_t 1.415] [lr 0.005000] +2024-04-03 18:37:51,699 - Train: 7.23% [137600/1902000] [72.3/1000.0] [batch_t 1.431 (1.423)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-03 18:40:14,084 - Train: 7.24% [137700/1902000] [72.4/1000.0] [batch_t 1.426 (1.424)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-03 18:42:36,671 - Train: 7.25% [137800/1902000] [72.5/1000.0] [batch_t 1.425 (1.426)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-03 18:44:59,070 - Train: 7.25% [137900/1902000] [72.5/1000.0] [batch_t 1.436 (1.424)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-03 18:47:21,320 - Train: 7.26% [138000/1902000] [72.6/1000.0] [batch_t 1.424 (1.422)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-03 18:49:43,574 - Train: 7.26% [138100/1902000] [72.6/1000.0] [batch_t 1.432 (1.422)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-03 18:52:05,837 - Train: 7.27% [138200/1902000] [72.7/1000.0] [batch_t 1.417 (1.423)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-03 18:54:28,310 - Train: 7.27% [138300/1902000] [72.7/1000.0] [batch_t 1.445 (1.425)] [data_t 0.003] [optim_t 1.442] [lr 0.005000] +2024-04-03 18:56:50,701 - Train: 7.28% [138400/1902000] [72.8/1000.0] [batch_t 1.435 (1.424)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 18:59:13,034 - Train: 7.28% [138500/1902000] [72.8/1000.0] [batch_t 1.431 (1.423)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 19:01:35,537 - Train: 7.29% [138600/1902000] [72.9/1000.0] [batch_t 1.428 (1.425)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-03 19:03:57,752 - Train: 7.29% [138700/1902000] [72.9/1000.0] [batch_t 1.414 (1.422)] [data_t 0.002] [optim_t 1.412] [lr 0.005000] +2024-04-03 19:06:20,049 - Train: 7.30% [138800/1902000] [73.0/1000.0] [batch_t 1.429 (1.423)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-03 19:07:25,598 - ==> Total time: 1 day, 18:07:05 Eta: 22 days, 6:50:31 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 19:08:43,980 - Train: 7.30% [138900/1902000] [73.0/1000.0] [batch_t 1.434 (1.427)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-03 19:11:06,359 - Train: 7.31% [139000/1902000] [73.1/1000.0] [batch_t 1.424 (1.424)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 19:13:28,726 - Train: 7.31% [139100/1902000] [73.1/1000.0] [batch_t 1.420 (1.424)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-03 19:15:51,132 - Train: 7.32% [139200/1902000] [73.2/1000.0] [batch_t 1.435 (1.424)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 19:18:13,449 - Train: 7.32% [139300/1902000] [73.2/1000.0] [batch_t 1.424 (1.423)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-03 19:20:35,905 - Train: 7.33% [139400/1902000] [73.3/1000.0] [batch_t 1.426 (1.424)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-03 19:22:58,324 - Train: 7.33% [139500/1902000] [73.3/1000.0] [batch_t 1.438 (1.424)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-03 19:25:20,588 - Train: 7.34% [139600/1902000] [73.4/1000.0] [batch_t 1.424 (1.423)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-03 19:27:43,072 - Train: 7.34% [139700/1902000] [73.4/1000.0] [batch_t 1.423 (1.425)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 19:30:05,571 - Train: 7.35% [139800/1902000] [73.5/1000.0] [batch_t 1.417 (1.425)] [data_t 0.002] [optim_t 1.415] [lr 0.005000] +2024-04-03 19:32:28,154 - Train: 7.36% [139900/1902000] [73.6/1000.0] [batch_t 1.437 (1.426)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-03 19:34:50,708 - Train: 7.36% [140000/1902000] [73.6/1000.0] [batch_t 1.436 (1.425)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 19:37:13,473 - Train: 7.37% [140100/1902000] [73.7/1000.0] [batch_t 1.418 (1.428)] [data_t 0.003] [optim_t 1.415] [lr 0.005000] +2024-04-03 19:39:35,924 - Train: 7.37% [140200/1902000] [73.7/1000.0] [batch_t 1.427 (1.424)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 19:41:58,354 - Train: 7.38% [140300/1902000] [73.8/1000.0] [batch_t 1.418 (1.424)] [data_t 0.003] [optim_t 1.415] [lr 0.005000] +2024-04-03 19:44:21,062 - Train: 7.38% [140400/1902000] [73.8/1000.0] [batch_t 1.418 (1.427)] [data_t 0.003] [optim_t 1.415] [lr 0.005000] +2024-04-03 19:46:43,631 - Train: 7.39% [140500/1902000] [73.9/1000.0] [batch_t 1.430 (1.426)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 19:49:06,263 - Train: 7.39% [140600/1902000] [73.9/1000.0] [batch_t 1.419 (1.426)] [data_t 0.003] [optim_t 1.416] [lr 0.005000] +2024-04-03 19:51:28,788 - Train: 7.40% [140700/1902000] [74.0/1000.0] [batch_t 1.435 (1.425)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 19:52:37,249 - ==> Total time: 1 day, 18:52:16 Eta: 22 days, 8:28:15 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 19:53:52,848 - Train: 7.40% [140800/1902000] [74.0/1000.0] [batch_t 1.422 (1.428)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-03 19:56:15,472 - Train: 7.41% [140900/1902000] [74.1/1000.0] [batch_t 1.433 (1.426)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-03 19:58:37,895 - Train: 7.41% [141000/1902000] [74.1/1000.0] [batch_t 1.431 (1.424)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-03 20:01:00,452 - Train: 7.42% [141100/1902000] [74.2/1000.0] [batch_t 1.417 (1.425)] [data_t 0.003] [optim_t 1.415] [lr 0.005000] +2024-04-03 20:03:22,886 - Train: 7.42% [141200/1902000] [74.2/1000.0] [batch_t 1.420 (1.424)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-03 20:05:45,325 - Train: 7.43% [141300/1902000] [74.3/1000.0] [batch_t 1.423 (1.424)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-03 20:08:07,691 - Train: 7.43% [141400/1902000] [74.3/1000.0] [batch_t 1.426 (1.424)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 20:10:30,386 - Train: 7.44% [141500/1902000] [74.4/1000.0] [batch_t 1.423 (1.427)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-03 20:12:52,942 - Train: 7.44% [141600/1902000] [74.4/1000.0] [batch_t 1.432 (1.425)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-03 20:15:15,559 - Train: 7.45% [141700/1902000] [74.5/1000.0] [batch_t 1.439 (1.426)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-03 20:17:38,108 - Train: 7.46% [141800/1902000] [74.6/1000.0] [batch_t 1.432 (1.425)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-03 20:20:00,847 - Train: 7.46% [141900/1902000] [74.6/1000.0] [batch_t 1.437 (1.427)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-03 20:22:23,512 - Train: 7.47% [142000/1902000] [74.7/1000.0] [batch_t 1.436 (1.427)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 20:24:46,018 - Train: 7.47% [142100/1902000] [74.7/1000.0] [batch_t 1.432 (1.425)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-03 20:27:08,703 - Train: 7.48% [142200/1902000] [74.8/1000.0] [batch_t 1.436 (1.427)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-03 20:29:31,152 - Train: 7.48% [142300/1902000] [74.8/1000.0] [batch_t 1.421 (1.424)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-03 20:31:53,698 - Train: 7.49% [142400/1902000] [74.9/1000.0] [batch_t 1.420 (1.425)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-03 20:34:16,226 - Train: 7.49% [142500/1902000] [74.9/1000.0] [batch_t 1.427 (1.425)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 20:36:38,822 - Train: 7.50% [142600/1902000] [75.0/1000.0] [batch_t 1.427 (1.426)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 20:37:50,186 - ==> Total time: 1 day, 19:37:29 Eta: 22 days, 10:02:26 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 20:39:03,349 - Train: 7.50% [142700/1902000] [75.0/1000.0] [batch_t 1.441 (1.435)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-03 20:41:26,004 - Train: 7.51% [142800/1902000] [75.1/1000.0] [batch_t 1.437 (1.426)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-03 20:43:48,562 - Train: 7.51% [142900/1902000] [75.1/1000.0] [batch_t 1.443 (1.425)] [data_t 0.003] [optim_t 1.440] [lr 0.005000] +2024-04-03 20:46:11,129 - Train: 7.52% [143000/1902000] [75.2/1000.0] [batch_t 1.436 (1.426)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 20:48:33,561 - Train: 7.52% [143100/1902000] [75.2/1000.0] [batch_t 1.436 (1.424)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-03 20:50:56,054 - Train: 7.53% [143200/1902000] [75.3/1000.0] [batch_t 1.424 (1.425)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-03 20:53:18,634 - Train: 7.53% [143300/1902000] [75.3/1000.0] [batch_t 1.423 (1.426)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-03 20:55:41,219 - Train: 7.54% [143400/1902000] [75.4/1000.0] [batch_t 1.427 (1.426)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-03 20:58:03,882 - Train: 7.54% [143500/1902000] [75.4/1000.0] [batch_t 1.425 (1.427)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-03 21:00:26,420 - Train: 7.55% [143600/1902000] [75.5/1000.0] [batch_t 1.446 (1.425)] [data_t 0.003] [optim_t 1.444] [lr 0.005000] +2024-04-03 21:02:49,194 - Train: 7.56% [143700/1902000] [75.6/1000.0] [batch_t 1.429 (1.428)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-03 21:05:11,728 - Train: 7.56% [143800/1902000] [75.6/1000.0] [batch_t 1.433 (1.425)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-03 21:07:34,327 - Train: 7.57% [143900/1902000] [75.7/1000.0] [batch_t 1.422 (1.426)] [data_t 0.002] [optim_t 1.420] [lr 0.005000] +2024-04-03 21:09:57,003 - Train: 7.57% [144000/1902000] [75.7/1000.0] [batch_t 1.440 (1.427)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-03 21:12:19,460 - Train: 7.58% [144100/1902000] [75.8/1000.0] [batch_t 1.438 (1.424)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-03 21:14:42,068 - Train: 7.58% [144200/1902000] [75.8/1000.0] [batch_t 1.428 (1.426)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 21:17:04,516 - Train: 7.59% [144300/1902000] [75.9/1000.0] [batch_t 1.421 (1.424)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-03 21:19:27,000 - Train: 7.59% [144400/1902000] [75.9/1000.0] [batch_t 1.416 (1.425)] [data_t 0.003] [optim_t 1.413] [lr 0.005000] +2024-04-03 21:21:49,472 - Train: 7.60% [144500/1902000] [76.0/1000.0] [batch_t 1.436 (1.425)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-03 21:23:03,526 - ==> Total time: 1 day, 20:22:43 Eta: 22 days, 11:33:02 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 21:24:14,520 - Train: 7.60% [144600/1902000] [76.0/1000.0] [batch_t 1.433 (1.429)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-03 21:26:37,016 - Train: 7.61% [144700/1902000] [76.1/1000.0] [batch_t 1.421 (1.425)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-03 21:28:59,624 - Train: 7.61% [144800/1902000] [76.1/1000.0] [batch_t 1.417 (1.426)] [data_t 0.002] [optim_t 1.415] [lr 0.005000] +2024-04-03 21:31:22,140 - Train: 7.62% [144900/1902000] [76.2/1000.0] [batch_t 1.437 (1.425)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-03 21:33:44,557 - Train: 7.62% [145000/1902000] [76.2/1000.0] [batch_t 1.433 (1.424)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-03 21:36:07,129 - Train: 7.63% [145100/1902000] [76.3/1000.0] [batch_t 1.417 (1.426)] [data_t 0.002] [optim_t 1.414] [lr 0.005000] +2024-04-03 21:38:29,626 - Train: 7.63% [145200/1902000] [76.3/1000.0] [batch_t 1.435 (1.425)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-03 21:40:52,190 - Train: 7.64% [145300/1902000] [76.4/1000.0] [batch_t 1.421 (1.426)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-03 21:43:14,630 - Train: 7.64% [145400/1902000] [76.4/1000.0] [batch_t 1.432 (1.424)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-03 21:45:37,236 - Train: 7.65% [145500/1902000] [76.5/1000.0] [batch_t 1.428 (1.426)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-03 21:47:59,712 - Train: 7.66% [145600/1902000] [76.6/1000.0] [batch_t 1.421 (1.425)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-03 21:50:22,223 - Train: 7.66% [145700/1902000] [76.6/1000.0] [batch_t 1.425 (1.425)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-03 21:52:44,670 - Train: 7.67% [145800/1902000] [76.7/1000.0] [batch_t 1.413 (1.424)] [data_t 0.003] [optim_t 1.410] [lr 0.005000] +2024-04-03 21:55:07,251 - Train: 7.67% [145900/1902000] [76.7/1000.0] [batch_t 1.422 (1.426)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-03 21:57:29,587 - Train: 7.68% [146000/1902000] [76.8/1000.0] [batch_t 1.427 (1.423)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-03 21:59:52,101 - Train: 7.68% [146100/1902000] [76.8/1000.0] [batch_t 1.430 (1.425)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 22:02:14,629 - Train: 7.69% [146200/1902000] [76.9/1000.0] [batch_t 1.436 (1.425)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 22:04:37,203 - Train: 7.69% [146300/1902000] [76.9/1000.0] [batch_t 1.434 (1.426)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-03 22:06:59,678 - Train: 7.70% [146400/1902000] [77.0/1000.0] [batch_t 1.431 (1.425)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 22:08:16,648 - ==> Total time: 1 day, 21:07:56 Eta: 22 days, 13:00:03 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 22:09:23,736 - Train: 7.70% [146500/1902000] [77.0/1000.0] [batch_t 1.432 (1.430)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-03 22:11:46,160 - Train: 7.71% [146600/1902000] [77.1/1000.0] [batch_t 1.427 (1.424)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-03 22:14:08,811 - Train: 7.71% [146700/1902000] [77.1/1000.0] [batch_t 1.424 (1.426)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-03 22:16:31,265 - Train: 7.72% [146800/1902000] [77.2/1000.0] [batch_t 1.422 (1.424)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-03 22:18:53,866 - Train: 7.72% [146900/1902000] [77.2/1000.0] [batch_t 1.437 (1.426)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-03 22:21:16,443 - Train: 7.73% [147000/1902000] [77.3/1000.0] [batch_t 1.431 (1.426)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-03 22:23:38,994 - Train: 7.73% [147100/1902000] [77.3/1000.0] [batch_t 1.413 (1.425)] [data_t 0.003] [optim_t 1.411] [lr 0.005000] +2024-04-03 22:26:01,558 - Train: 7.74% [147200/1902000] [77.4/1000.0] [batch_t 1.429 (1.426)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-03 22:28:24,182 - Train: 7.74% [147300/1902000] [77.4/1000.0] [batch_t 1.418 (1.426)] [data_t 0.002] [optim_t 1.415] [lr 0.005000] +2024-04-03 22:30:46,772 - Train: 7.75% [147400/1902000] [77.5/1000.0] [batch_t 1.435 (1.426)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 22:33:09,410 - Train: 7.75% [147500/1902000] [77.5/1000.0] [batch_t 1.422 (1.426)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-03 22:35:32,083 - Train: 7.76% [147600/1902000] [77.6/1000.0] [batch_t 1.411 (1.427)] [data_t 0.002] [optim_t 1.409] [lr 0.005000] +2024-04-03 22:37:54,777 - Train: 7.77% [147700/1902000] [77.7/1000.0] [batch_t 1.435 (1.427)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-03 22:40:17,435 - Train: 7.77% [147800/1902000] [77.7/1000.0] [batch_t 1.436 (1.426)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-03 22:42:39,966 - Train: 7.78% [147900/1902000] [77.8/1000.0] [batch_t 1.425 (1.425)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-03 22:45:02,559 - Train: 7.78% [148000/1902000] [77.8/1000.0] [batch_t 1.435 (1.426)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-03 22:47:25,219 - Train: 7.79% [148100/1902000] [77.9/1000.0] [batch_t 1.422 (1.427)] [data_t 0.002] [optim_t 1.420] [lr 0.005000] +2024-04-03 22:49:47,881 - Train: 7.79% [148200/1902000] [77.9/1000.0] [batch_t 1.409 (1.427)] [data_t 0.003] [optim_t 1.406] [lr 0.005000] +2024-04-03 22:52:10,507 - Train: 7.80% [148300/1902000] [78.0/1000.0] [batch_t 1.435 (1.426)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-03 22:53:30,345 - ==> Total time: 1 day, 21:53:09 Eta: 22 days, 14:23:48 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 22:54:35,572 - Train: 7.80% [148400/1902000] [78.0/1000.0] [batch_t 1.422 (1.447)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-03 22:56:58,076 - Train: 7.81% [148500/1902000] [78.1/1000.0] [batch_t 1.431 (1.425)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 22:59:20,531 - Train: 7.81% [148600/1902000] [78.1/1000.0] [batch_t 1.422 (1.424)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-03 23:01:43,264 - Train: 7.82% [148700/1902000] [78.2/1000.0] [batch_t 1.441 (1.427)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-03 23:04:05,685 - Train: 7.82% [148800/1902000] [78.2/1000.0] [batch_t 1.429 (1.424)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-03 23:06:28,247 - Train: 7.83% [148900/1902000] [78.3/1000.0] [batch_t 1.435 (1.426)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-03 23:08:50,762 - Train: 7.83% [149000/1902000] [78.3/1000.0] [batch_t 1.427 (1.425)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-03 23:11:13,408 - Train: 7.84% [149100/1902000] [78.4/1000.0] [batch_t 1.439 (1.426)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-03 23:13:36,072 - Train: 7.84% [149200/1902000] [78.4/1000.0] [batch_t 1.427 (1.427)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-03 23:15:58,615 - Train: 7.85% [149300/1902000] [78.5/1000.0] [batch_t 1.430 (1.425)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-03 23:18:21,120 - Train: 7.85% [149400/1902000] [78.5/1000.0] [batch_t 1.432 (1.425)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-03 23:20:43,762 - Train: 7.86% [149500/1902000] [78.6/1000.0] [batch_t 1.433 (1.426)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-03 23:23:06,265 - Train: 7.87% [149600/1902000] [78.7/1000.0] [batch_t 1.436 (1.425)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-03 23:25:28,786 - Train: 7.87% [149700/1902000] [78.7/1000.0] [batch_t 1.440 (1.425)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-03 23:27:51,314 - Train: 7.88% [149800/1902000] [78.8/1000.0] [batch_t 1.427 (1.425)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-03 23:30:13,891 - Train: 7.88% [149900/1902000] [78.8/1000.0] [batch_t 1.425 (1.426)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 23:32:36,539 - Train: 7.89% [150000/1902000] [78.9/1000.0] [batch_t 1.409 (1.426)] [data_t 0.003] [optim_t 1.406] [lr 0.005000] +2024-04-03 23:34:59,057 - Train: 7.89% [150100/1902000] [78.9/1000.0] [batch_t 1.425 (1.425)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 23:37:21,533 - Train: 7.90% [150200/1902000] [79.0/1000.0] [batch_t 1.426 (1.425)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-03 23:38:44,125 - ==> Total time: 1 day, 22:38:23 Eta: 22 days, 15:44:18 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-03 23:39:45,658 - Train: 7.90% [150300/1902000] [79.0/1000.0] [batch_t 1.425 (1.431)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-03 23:42:08,398 - Train: 7.91% [150400/1902000] [79.1/1000.0] [batch_t 1.429 (1.427)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-03 23:44:31,136 - Train: 7.91% [150500/1902000] [79.1/1000.0] [batch_t 1.436 (1.427)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-03 23:46:53,531 - Train: 7.92% [150600/1902000] [79.2/1000.0] [batch_t 1.426 (1.424)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-03 23:49:16,062 - Train: 7.92% [150700/1902000] [79.2/1000.0] [batch_t 1.426 (1.425)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-03 23:51:38,658 - Train: 7.93% [150800/1902000] [79.3/1000.0] [batch_t 1.432 (1.426)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-03 23:54:01,054 - Train: 7.93% [150900/1902000] [79.3/1000.0] [batch_t 1.436 (1.424)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-03 23:56:23,570 - Train: 7.94% [151000/1902000] [79.4/1000.0] [batch_t 1.435 (1.425)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-03 23:58:46,195 - Train: 7.94% [151100/1902000] [79.4/1000.0] [batch_t 1.434 (1.426)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-04 00:01:08,790 - Train: 7.95% [151200/1902000] [79.5/1000.0] [batch_t 1.437 (1.426)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-04 00:03:31,240 - Train: 7.95% [151300/1902000] [79.5/1000.0] [batch_t 1.427 (1.424)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-04 00:05:53,696 - Train: 7.96% [151400/1902000] [79.6/1000.0] [batch_t 1.412 (1.424)] [data_t 0.003] [optim_t 1.409] [lr 0.005000] +2024-04-04 00:08:16,079 - Train: 7.97% [151500/1902000] [79.7/1000.0] [batch_t 1.426 (1.424)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-04 00:10:38,554 - Train: 7.97% [151600/1902000] [79.7/1000.0] [batch_t 1.425 (1.425)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-04 00:13:00,898 - Train: 7.98% [151700/1902000] [79.8/1000.0] [batch_t 1.426 (1.423)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-04 00:15:23,449 - Train: 7.98% [151800/1902000] [79.8/1000.0] [batch_t 1.436 (1.425)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-04 00:17:45,810 - Train: 7.99% [151900/1902000] [79.9/1000.0] [batch_t 1.432 (1.424)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-04 00:20:08,374 - Train: 7.99% [152000/1902000] [79.9/1000.0] [batch_t 1.427 (1.426)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-04 00:22:30,850 - Train: 8.00% [152100/1902000] [80.0/1000.0] [batch_t 1.427 (1.425)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-04 00:23:56,253 - ==> Total time: 1 day, 23:23:35 Eta: 22 days, 17:01:21 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 00:24:54,815 - Train: 8.00% [152200/1902000] [80.0/1000.0] [batch_t 1.421 (1.430)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-04 00:27:17,297 - Train: 8.01% [152300/1902000] [80.1/1000.0] [batch_t 1.422 (1.425)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-04 00:29:39,852 - Train: 8.01% [152400/1902000] [80.1/1000.0] [batch_t 1.427 (1.425)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-04 00:32:02,332 - Train: 8.02% [152500/1902000] [80.2/1000.0] [batch_t 1.437 (1.425)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-04 00:34:24,759 - Train: 8.02% [152600/1902000] [80.2/1000.0] [batch_t 1.436 (1.424)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-04 00:36:47,126 - Train: 8.03% [152700/1902000] [80.3/1000.0] [batch_t 1.426 (1.424)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-04 00:39:09,795 - Train: 8.03% [152800/1902000] [80.3/1000.0] [batch_t 1.430 (1.427)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-04 00:41:32,399 - Train: 8.04% [152900/1902000] [80.4/1000.0] [batch_t 1.437 (1.426)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-04 00:43:54,936 - Train: 8.04% [153000/1902000] [80.4/1000.0] [batch_t 1.415 (1.425)] [data_t 0.003] [optim_t 1.412] [lr 0.005000] +2024-04-04 00:46:17,570 - Train: 8.05% [153100/1902000] [80.5/1000.0] [batch_t 1.426 (1.426)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-04 00:48:40,126 - Train: 8.05% [153200/1902000] [80.5/1000.0] [batch_t 1.445 (1.425)] [data_t 0.002] [optim_t 1.442] [lr 0.005000] +2024-04-04 00:51:02,708 - Train: 8.06% [153300/1902000] [80.6/1000.0] [batch_t 1.423 (1.426)] [data_t 0.002] [optim_t 1.420] [lr 0.005000] +2024-04-04 00:53:25,225 - Train: 8.07% [153400/1902000] [80.7/1000.0] [batch_t 1.432 (1.425)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-04 00:55:47,644 - Train: 8.07% [153500/1902000] [80.7/1000.0] [batch_t 1.435 (1.424)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-04 00:58:10,228 - Train: 8.08% [153600/1902000] [80.8/1000.0] [batch_t 1.421 (1.426)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-04 01:00:32,869 - Train: 8.08% [153700/1902000] [80.8/1000.0] [batch_t 1.423 (1.426)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-04 01:02:55,473 - Train: 8.09% [153800/1902000] [80.9/1000.0] [batch_t 1.434 (1.426)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-04 01:05:18,058 - Train: 8.09% [153900/1902000] [80.9/1000.0] [batch_t 1.420 (1.426)] [data_t 0.003] [optim_t 1.416] [lr 0.005000] +2024-04-04 01:07:40,730 - Train: 8.10% [154000/1902000] [81.0/1000.0] [batch_t 1.430 (1.427)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-04 01:09:09,201 - ==> Total time: 2 days, 0:08:48 Eta: 22 days, 18:15:32 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 01:10:04,817 - Train: 8.10% [154100/1902000] [81.0/1000.0] [batch_t 1.442 (1.430)] [data_t 0.003] [optim_t 1.439] [lr 0.005000] +2024-04-04 01:12:27,358 - Train: 8.11% [154200/1902000] [81.1/1000.0] [batch_t 1.417 (1.425)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-04 01:14:49,880 - Train: 8.11% [154300/1902000] [81.1/1000.0] [batch_t 1.423 (1.425)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-04 01:17:12,358 - Train: 8.12% [154400/1902000] [81.2/1000.0] [batch_t 1.433 (1.425)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-04 01:19:35,032 - Train: 8.12% [154500/1902000] [81.2/1000.0] [batch_t 1.432 (1.427)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-04 01:21:57,577 - Train: 8.13% [154600/1902000] [81.3/1000.0] [batch_t 1.427 (1.425)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-04 01:24:20,202 - Train: 8.13% [154700/1902000] [81.3/1000.0] [batch_t 1.411 (1.426)] [data_t 0.003] [optim_t 1.408] [lr 0.005000] +2024-04-04 01:26:42,691 - Train: 8.14% [154800/1902000] [81.4/1000.0] [batch_t 1.421 (1.425)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-04 01:29:05,263 - Train: 8.14% [154900/1902000] [81.4/1000.0] [batch_t 1.426 (1.426)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-04 01:31:27,657 - Train: 8.15% [155000/1902000] [81.5/1000.0] [batch_t 1.439 (1.424)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-04 01:33:50,102 - Train: 8.15% [155100/1902000] [81.5/1000.0] [batch_t 1.421 (1.424)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-04 01:36:12,667 - Train: 8.16% [155200/1902000] [81.6/1000.0] [batch_t 1.422 (1.426)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-04 01:38:35,324 - Train: 8.17% [155300/1902000] [81.7/1000.0] [batch_t 1.418 (1.426)] [data_t 0.002] [optim_t 1.416] [lr 0.005000] +2024-04-04 01:40:57,860 - Train: 8.17% [155400/1902000] [81.7/1000.0] [batch_t 1.434 (1.425)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-04 01:43:20,415 - Train: 8.18% [155500/1902000] [81.8/1000.0] [batch_t 1.425 (1.425)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-04 01:45:42,863 - Train: 8.18% [155600/1902000] [81.8/1000.0] [batch_t 1.428 (1.424)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-04 01:48:05,444 - Train: 8.19% [155700/1902000] [81.9/1000.0] [batch_t 1.431 (1.426)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-04 01:50:27,898 - Train: 8.19% [155800/1902000] [81.9/1000.0] [batch_t 1.417 (1.424)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-04 01:52:50,330 - Train: 8.20% [155900/1902000] [82.0/1000.0] [batch_t 1.420 (1.424)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-04 01:54:21,578 - ==> Total time: 2 days, 0:54:01 Eta: 22 days, 19:26:41 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 01:55:14,516 - Train: 8.20% [156000/1902000] [82.0/1000.0] [batch_t 1.430 (1.431)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-04 01:57:37,080 - Train: 8.21% [156100/1902000] [82.1/1000.0] [batch_t 1.426 (1.426)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-04 01:59:59,649 - Train: 8.21% [156200/1902000] [82.1/1000.0] [batch_t 1.428 (1.426)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-04 02:02:22,213 - Train: 8.22% [156300/1902000] [82.2/1000.0] [batch_t 1.428 (1.426)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-04 02:04:44,886 - Train: 8.22% [156400/1902000] [82.2/1000.0] [batch_t 1.442 (1.427)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-04 02:07:07,601 - Train: 8.23% [156500/1902000] [82.3/1000.0] [batch_t 1.420 (1.427)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-04 02:09:02,813 - Train: 8.23% [156600/1902000] [82.3/1000.0] [batch_t 0.605 (1.152)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:10:03,302 - Train: 8.24% [156700/1902000] [82.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 02:11:03,732 - Train: 8.24% [156800/1902000] [82.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:12:04,120 - Train: 8.25% [156900/1902000] [82.5/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 02:13:04,617 - Train: 8.25% [157000/1902000] [82.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 02:14:05,049 - Train: 8.26% [157100/1902000] [82.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:15:05,446 - Train: 8.26% [157200/1902000] [82.6/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 02:16:05,908 - Train: 8.27% [157300/1902000] [82.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:17:06,331 - Train: 8.28% [157400/1902000] [82.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:18:06,822 - Train: 8.28% [157500/1902000] [82.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:19:07,259 - Train: 8.29% [157600/1902000] [82.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 02:20:07,680 - Train: 8.29% [157700/1902000] [82.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:21:08,116 - Train: 8.30% [157800/1902000] [83.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 02:21:47,982 - ==> Total time: 2 days, 1:21:27 Eta: 22 days, 17:18:45 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 02:22:10,216 - Train: 8.30% [157900/1902000] [83.0/1000.0] [batch_t 0.605 (0.611)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 02:23:10,629 - Train: 8.31% [158000/1902000] [83.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:24:11,021 - Train: 8.31% [158100/1902000] [83.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:25:11,533 - Train: 8.32% [158200/1902000] [83.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:26:11,963 - Train: 8.32% [158300/1902000] [83.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:27:12,365 - Train: 8.33% [158400/1902000] [83.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:28:12,754 - Train: 8.33% [158500/1902000] [83.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 02:29:13,212 - Train: 8.34% [158600/1902000] [83.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 02:30:13,708 - Train: 8.34% [158700/1902000] [83.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 02:31:14,139 - Train: 8.35% [158800/1902000] [83.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 02:32:14,535 - Train: 8.35% [158900/1902000] [83.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 02:33:14,941 - Train: 8.36% [159000/1902000] [83.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 02:34:15,340 - Train: 8.36% [159100/1902000] [83.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:35:15,721 - Train: 8.37% [159200/1902000] [83.7/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 02:36:16,134 - Train: 8.38% [159300/1902000] [83.8/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 02:37:16,572 - Train: 8.38% [159400/1902000] [83.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 02:38:17,062 - Train: 8.39% [159500/1902000] [83.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 02:39:17,470 - Train: 8.39% [159600/1902000] [83.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:40:17,855 - Train: 8.40% [159700/1902000] [84.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 02:40:58,938 - ==> Total time: 2 days, 1:40:38 Eta: 22 days, 13:43:10 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 02:41:20,007 - Train: 8.40% [159800/1902000] [84.0/1000.0] [batch_t 0.603 (0.614)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 02:42:20,432 - Train: 8.41% [159900/1902000] [84.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:43:20,967 - Train: 8.41% [160000/1902000] [84.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 02:44:21,386 - Train: 8.42% [160100/1902000] [84.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 02:45:21,806 - Train: 8.42% [160200/1902000] [84.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 02:46:22,258 - Train: 8.43% [160300/1902000] [84.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 02:47:22,662 - Train: 8.43% [160400/1902000] [84.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:48:23,145 - Train: 8.44% [160500/1902000] [84.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 02:49:23,597 - Train: 8.44% [160600/1902000] [84.4/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 02:50:24,001 - Train: 8.45% [160700/1902000] [84.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 02:51:24,421 - Train: 8.45% [160800/1902000] [84.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 02:52:24,919 - Train: 8.46% [160900/1902000] [84.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:53:25,465 - Train: 8.46% [161000/1902000] [84.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 02:54:25,990 - Train: 8.47% [161100/1902000] [84.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:55:26,577 - Train: 8.48% [161200/1902000] [84.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-04 02:56:27,264 - Train: 8.48% [161300/1902000] [84.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-04 02:57:27,808 - Train: 8.49% [161400/1902000] [84.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-04 02:58:28,354 - Train: 8.49% [161500/1902000] [84.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-04 02:59:28,893 - Train: 8.50% [161600/1902000] [85.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:00:11,292 - ==> Total time: 2 days, 1:59:50 Eta: 22 days, 10:12:28 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 03:00:30,992 - Train: 8.50% [161700/1902000] [85.0/1000.0] [batch_t 0.604 (0.611)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-04 03:01:31,678 - Train: 8.51% [161800/1902000] [85.1/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 03:02:32,219 - Train: 8.51% [161900/1902000] [85.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-04 03:03:32,768 - Train: 8.52% [162000/1902000] [85.2/1000.0] [batch_t 0.609 (0.605)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-04 03:04:33,333 - Train: 8.52% [162100/1902000] [85.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-04 03:05:33,856 - Train: 8.53% [162200/1902000] [85.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-04 03:06:34,379 - Train: 8.53% [162300/1902000] [85.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-04 03:07:34,935 - Train: 8.54% [162400/1902000] [85.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 03:08:35,482 - Train: 8.54% [162500/1902000] [85.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-04 03:09:36,115 - Train: 8.55% [162600/1902000] [85.5/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 03:10:36,648 - Train: 8.55% [162700/1902000] [85.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-04 03:11:37,199 - Train: 8.56% [162800/1902000] [85.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:12:37,757 - Train: 8.56% [162900/1902000] [85.6/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 03:13:38,330 - Train: 8.57% [163000/1902000] [85.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:14:39,009 - Train: 8.58% [163100/1902000] [85.8/1000.0] [batch_t 0.604 (0.607)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-04 03:15:39,584 - Train: 8.58% [163200/1902000] [85.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:16:40,107 - Train: 8.59% [163300/1902000] [85.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 03:17:40,611 - Train: 8.59% [163400/1902000] [85.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:18:41,173 - Train: 8.60% [163500/1902000] [86.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-04 03:19:24,809 - ==> Total time: 2 days, 2:19:04 Eta: 22 days, 6:46:25 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 03:19:43,449 - Train: 8.60% [163600/1902000] [86.0/1000.0] [batch_t 0.608 (0.611)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-04 03:20:44,027 - Train: 8.61% [163700/1902000] [86.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 03:21:44,690 - Train: 8.61% [163800/1902000] [86.1/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 03:22:45,172 - Train: 8.62% [163900/1902000] [86.2/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 03:23:45,610 - Train: 8.62% [164000/1902000] [86.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:24:46,045 - Train: 8.63% [164100/1902000] [86.3/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 03:25:46,494 - Train: 8.63% [164200/1902000] [86.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:26:47,040 - Train: 8.64% [164300/1902000] [86.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 03:27:47,530 - Train: 8.64% [164400/1902000] [86.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 03:28:48,004 - Train: 8.65% [164500/1902000] [86.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:29:48,478 - Train: 8.65% [164600/1902000] [86.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:30:48,918 - Train: 8.66% [164700/1902000] [86.6/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 03:31:49,399 - Train: 8.66% [164800/1902000] [86.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:32:49,843 - Train: 8.67% [164900/1902000] [86.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:33:50,332 - Train: 8.68% [165000/1902000] [86.8/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 03:34:50,955 - Train: 8.68% [165100/1902000] [86.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 03:35:51,409 - Train: 8.69% [165200/1902000] [86.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:36:51,921 - Train: 8.69% [165300/1902000] [86.9/1000.0] [batch_t 0.608 (0.605)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-04 03:37:52,379 - Train: 8.70% [165400/1902000] [87.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:38:37,146 - ==> Total time: 2 days, 2:38:16 Eta: 22 days, 3:24:27 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 03:38:54,475 - Train: 8.70% [165500/1902000] [87.0/1000.0] [batch_t 0.606 (0.613)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 03:39:55,049 - Train: 8.71% [165600/1902000] [87.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:40:55,504 - Train: 8.71% [165700/1902000] [87.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 03:41:55,964 - Train: 8.72% [165800/1902000] [87.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:42:56,558 - Train: 8.72% [165900/1902000] [87.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:43:57,043 - Train: 8.73% [166000/1902000] [87.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 03:44:57,563 - Train: 8.73% [166100/1902000] [87.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:45:58,044 - Train: 8.74% [166200/1902000] [87.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 03:46:58,543 - Train: 8.74% [166300/1902000] [87.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 03:47:59,107 - Train: 8.75% [166400/1902000] [87.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:48:59,573 - Train: 8.75% [166500/1902000] [87.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:50:00,042 - Train: 8.76% [166600/1902000] [87.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 03:51:00,517 - Train: 8.76% [166700/1902000] [87.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 03:52:01,008 - Train: 8.77% [166800/1902000] [87.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 03:53:01,542 - Train: 8.77% [166900/1902000] [87.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 03:54:01,968 - Train: 8.78% [167000/1902000] [87.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 03:55:02,426 - Train: 8.79% [167100/1902000] [87.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 03:56:02,867 - Train: 8.79% [167200/1902000] [87.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 03:57:03,308 - Train: 8.80% [167300/1902000] [88.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 03:57:49,268 - ==> Total time: 2 days, 2:57:28 Eta: 22 days, 0:06:36 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 03:58:05,330 - Train: 8.80% [167400/1902000] [88.0/1000.0] [batch_t 0.607 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 03:59:05,836 - Train: 8.81% [167500/1902000] [88.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:00:06,410 - Train: 8.81% [167600/1902000] [88.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 04:01:06,875 - Train: 8.82% [167700/1902000] [88.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:02:07,366 - Train: 8.82% [167800/1902000] [88.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:03:07,843 - Train: 8.83% [167900/1902000] [88.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:04:08,252 - Train: 8.83% [168000/1902000] [88.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:05:08,711 - Train: 8.84% [168100/1902000] [88.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:06:09,256 - Train: 8.84% [168200/1902000] [88.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 04:07:09,710 - Train: 8.85% [168300/1902000] [88.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 04:08:10,168 - Train: 8.85% [168400/1902000] [88.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:09:10,683 - Train: 8.86% [168500/1902000] [88.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:10:11,170 - Train: 8.86% [168600/1902000] [88.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:11:11,605 - Train: 8.87% [168700/1902000] [88.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 04:12:12,087 - Train: 8.87% [168800/1902000] [88.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:13:12,644 - Train: 8.88% [168900/1902000] [88.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:14:13,119 - Train: 8.89% [169000/1902000] [88.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:15:13,585 - Train: 8.89% [169100/1902000] [88.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 04:16:14,073 - Train: 8.90% [169200/1902000] [89.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 04:17:01,224 - ==> Total time: 2 days, 3:16:40 Eta: 21 days, 20:52:44 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 04:17:17,969 - Train: 8.90% [169300/1902000] [89.0/1000.0] [batch_t 0.606 (0.629)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:18:18,563 - Train: 8.91% [169400/1902000] [89.1/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 04:19:19,010 - Train: 8.91% [169500/1902000] [89.1/1000.0] [batch_t 0.610 (0.604)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-04 04:20:19,467 - Train: 8.92% [169600/1902000] [89.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:21:19,926 - Train: 8.92% [169700/1902000] [89.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:22:20,393 - Train: 8.93% [169800/1902000] [89.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:23:20,846 - Train: 8.93% [169900/1902000] [89.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:24:21,254 - Train: 8.94% [170000/1902000] [89.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:25:21,684 - Train: 8.94% [170100/1902000] [89.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 04:26:22,241 - Train: 8.95% [170200/1902000] [89.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:27:22,657 - Train: 8.95% [170300/1902000] [89.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 04:28:23,120 - Train: 8.96% [170400/1902000] [89.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:29:23,551 - Train: 8.96% [170500/1902000] [89.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:30:24,002 - Train: 8.97% [170600/1902000] [89.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:31:24,567 - Train: 8.97% [170700/1902000] [89.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:32:25,017 - Train: 8.98% [170800/1902000] [89.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:33:25,478 - Train: 8.99% [170900/1902000] [89.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:34:25,914 - Train: 8.99% [171000/1902000] [89.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 04:35:26,372 - Train: 9.00% [171100/1902000] [90.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:36:14,709 - ==> Total time: 2 days, 3:35:54 Eta: 21 days, 17:43:01 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 04:36:28,253 - Train: 9.00% [171200/1902000] [90.0/1000.0] [batch_t 0.605 (0.615)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:37:28,673 - Train: 9.01% [171300/1902000] [90.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 04:38:29,294 - Train: 9.01% [171400/1902000] [90.1/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 04:39:29,730 - Train: 9.02% [171500/1902000] [90.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:40:30,195 - Train: 9.02% [171600/1902000] [90.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:41:30,664 - Train: 9.03% [171700/1902000] [90.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:42:31,125 - Train: 9.03% [171800/1902000] [90.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:43:31,667 - Train: 9.04% [171900/1902000] [90.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 04:44:32,080 - Train: 9.04% [172000/1902000] [90.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:45:32,536 - Train: 9.05% [172100/1902000] [90.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:46:32,997 - Train: 9.05% [172200/1902000] [90.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:47:33,449 - Train: 9.06% [172300/1902000] [90.6/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 04:48:33,871 - Train: 9.06% [172400/1902000] [90.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:49:34,324 - Train: 9.07% [172500/1902000] [90.7/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 04:50:34,771 - Train: 9.07% [172600/1902000] [90.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:51:35,230 - Train: 9.08% [172700/1902000] [90.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:52:35,659 - Train: 9.09% [172800/1902000] [90.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:53:36,892 - Train: 9.09% [172900/1902000] [90.9/1000.0] [batch_t 0.604 (0.612)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-04 04:54:37,312 - Train: 9.10% [173000/1902000] [91.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 04:55:26,869 - ==> Total time: 2 days, 3:55:06 Eta: 21 days, 14:36:49 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 04:55:39,374 - Train: 9.10% [173100/1902000] [91.0/1000.0] [batch_t 0.605 (0.618)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 04:56:39,931 - Train: 9.11% [173200/1902000] [91.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 04:57:40,380 - Train: 9.11% [173300/1902000] [91.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 04:58:40,845 - Train: 9.12% [173400/1902000] [91.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 04:59:41,291 - Train: 9.12% [173500/1902000] [91.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:00:41,742 - Train: 9.13% [173600/1902000] [91.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:01:42,260 - Train: 9.13% [173700/1902000] [91.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 05:02:42,675 - Train: 9.14% [173800/1902000] [91.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 05:03:43,153 - Train: 9.14% [173900/1902000] [91.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:04:43,604 - Train: 9.15% [174000/1902000] [91.5/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 05:05:44,083 - Train: 9.15% [174100/1902000] [91.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:06:44,501 - Train: 9.16% [174200/1902000] [91.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 05:07:44,939 - Train: 9.16% [174300/1902000] [91.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:08:45,392 - Train: 9.17% [174400/1902000] [91.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:09:45,847 - Train: 9.17% [174500/1902000] [91.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 05:10:46,225 - Train: 9.18% [174600/1902000] [91.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 05:11:46,622 - Train: 9.19% [174700/1902000] [91.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:12:47,042 - Train: 9.19% [174800/1902000] [91.9/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 05:13:47,443 - Train: 9.20% [174900/1902000] [92.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 05:14:38,251 - ==> Total time: 2 days, 4:14:17 Eta: 21 days, 11:34:08 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 05:14:49,502 - Train: 9.20% [175000/1902000] [92.0/1000.0] [batch_t 0.606 (0.618)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:15:49,888 - Train: 9.21% [175100/1902000] [92.1/1000.0] [batch_t 0.610 (0.604)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-04 05:16:50,308 - Train: 9.21% [175200/1902000] [92.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 05:17:50,687 - Train: 9.22% [175300/1902000] [92.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 05:18:51,127 - Train: 9.22% [175400/1902000] [92.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 05:19:51,680 - Train: 9.23% [175500/1902000] [92.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 05:20:52,066 - Train: 9.23% [175600/1902000] [92.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 05:21:52,430 - Train: 9.24% [175700/1902000] [92.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 05:22:52,862 - Train: 9.24% [175800/1902000] [92.4/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 05:23:53,237 - Train: 9.25% [175900/1902000] [92.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 05:24:53,589 - Train: 9.25% [176000/1902000] [92.5/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 05:25:53,980 - Train: 9.26% [176100/1902000] [92.6/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 05:26:54,330 - Train: 9.26% [176200/1902000] [92.6/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 05:27:54,745 - Train: 9.27% [176300/1902000] [92.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 05:28:55,126 - Train: 9.27% [176400/1902000] [92.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 05:29:55,538 - Train: 9.28% [176500/1902000] [92.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 05:30:55,886 - Train: 9.28% [176600/1902000] [92.8/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 05:31:56,290 - Train: 9.29% [176700/1902000] [92.9/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 05:32:56,754 - Train: 9.30% [176800/1902000] [93.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:33:48,690 - ==> Total time: 2 days, 4:33:28 Eta: 21 days, 8:34:48 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 05:33:59,091 - Train: 9.30% [176900/1902000] [93.0/1000.0] [batch_t 0.604 (0.620)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 05:34:59,455 - Train: 9.31% [177000/1902000] [93.1/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 05:35:59,986 - Train: 9.31% [177100/1902000] [93.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 05:37:00,383 - Train: 9.32% [177200/1902000] [93.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 05:38:00,721 - Train: 9.32% [177300/1902000] [93.2/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 05:39:01,103 - Train: 9.33% [177400/1902000] [93.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 05:40:01,605 - Train: 9.33% [177500/1902000] [93.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:41:01,982 - Train: 9.34% [177600/1902000] [93.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 05:42:02,356 - Train: 9.34% [177700/1902000] [93.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:43:02,706 - Train: 9.35% [177800/1902000] [93.5/1000.0] [batch_t 0.601 (0.603)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 05:44:03,058 - Train: 9.35% [177900/1902000] [93.5/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 05:45:03,507 - Train: 9.36% [178000/1902000] [93.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 05:46:03,907 - Train: 9.36% [178100/1902000] [93.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 05:47:04,269 - Train: 9.37% [178200/1902000] [93.7/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 05:48:04,638 - Train: 9.37% [178300/1902000] [93.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 05:49:05,016 - Train: 9.38% [178400/1902000] [93.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:50:05,404 - Train: 9.38% [178500/1902000] [93.8/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 05:51:05,794 - Train: 9.39% [178600/1902000] [93.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 05:52:06,157 - Train: 9.40% [178700/1902000] [94.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 05:52:59,359 - ==> Total time: 2 days, 4:52:38 Eta: 21 days, 5:38:55 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 05:53:08,304 - Train: 9.40% [178800/1902000] [94.0/1000.0] [batch_t 0.606 (0.636)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 05:54:08,687 - Train: 9.41% [178900/1902000] [94.1/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 05:55:09,056 - Train: 9.41% [179000/1902000] [94.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 05:56:09,444 - Train: 9.42% [179100/1902000] [94.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 05:57:09,855 - Train: 9.42% [179200/1902000] [94.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 05:58:10,230 - Train: 9.43% [179300/1902000] [94.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 05:59:10,623 - Train: 9.43% [179400/1902000] [94.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:00:11,183 - Train: 9.44% [179500/1902000] [94.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:01:11,571 - Train: 9.44% [179600/1902000] [94.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:02:11,967 - Train: 9.45% [179700/1902000] [94.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 06:03:12,416 - Train: 9.45% [179800/1902000] [94.5/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 06:04:12,826 - Train: 9.46% [179900/1902000] [94.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:05:13,254 - Train: 9.46% [180000/1902000] [94.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:06:13,654 - Train: 9.47% [180100/1902000] [94.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 06:07:14,055 - Train: 9.47% [180200/1902000] [94.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:08:14,442 - Train: 9.48% [180300/1902000] [94.8/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 06:09:14,872 - Train: 9.48% [180400/1902000] [94.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 06:10:15,287 - Train: 9.49% [180500/1902000] [94.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:11:15,658 - Train: 9.50% [180600/1902000] [95.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:12:10,013 - ==> Total time: 2 days, 5:11:49 Eta: 21 days, 2:46:20 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 06:12:17,911 - Train: 9.50% [180700/1902000] [95.0/1000.0] [batch_t 0.604 (0.634)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:13:18,343 - Train: 9.51% [180800/1902000] [95.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:14:18,701 - Train: 9.51% [180900/1902000] [95.1/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:15:19,068 - Train: 9.52% [181000/1902000] [95.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:16:19,463 - Train: 9.52% [181100/1902000] [95.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 06:17:19,882 - Train: 9.53% [181200/1902000] [95.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:18:20,414 - Train: 9.53% [181300/1902000] [95.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:19:20,807 - Train: 9.54% [181400/1902000] [95.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:20:21,191 - Train: 9.54% [181500/1902000] [95.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:21:21,560 - Train: 9.55% [181600/1902000] [95.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 06:22:21,956 - Train: 9.55% [181700/1902000] [95.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 06:23:22,384 - Train: 9.56% [181800/1902000] [95.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:24:22,776 - Train: 9.56% [181900/1902000] [95.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:25:23,293 - Train: 9.57% [182000/1902000] [95.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:26:23,699 - Train: 9.57% [182100/1902000] [95.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:27:24,094 - Train: 9.58% [182200/1902000] [95.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:28:24,502 - Train: 9.58% [182300/1902000] [95.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 06:29:24,896 - Train: 9.59% [182400/1902000] [95.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:30:25,306 - Train: 9.60% [182500/1902000] [96.0/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 06:31:20,850 - ==> Total time: 2 days, 5:31:00 Eta: 20 days, 23:56:58 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 06:31:27,468 - Train: 9.60% [182600/1902000] [96.0/1000.0] [batch_t 0.604 (0.659)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:32:27,851 - Train: 9.61% [182700/1902000] [96.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:33:28,238 - Train: 9.61% [182800/1902000] [96.1/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 06:34:28,600 - Train: 9.62% [182900/1902000] [96.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:35:28,970 - Train: 9.62% [183000/1902000] [96.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:36:29,367 - Train: 9.63% [183100/1902000] [96.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:37:29,816 - Train: 9.63% [183200/1902000] [96.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:38:30,202 - Train: 9.64% [183300/1902000] [96.4/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 06:39:30,642 - Train: 9.64% [183400/1902000] [96.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:40:31,064 - Train: 9.65% [183500/1902000] [96.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:41:31,457 - Train: 9.65% [183600/1902000] [96.5/1000.0] [batch_t 0.599 (0.604)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-04 06:42:31,832 - Train: 9.66% [183700/1902000] [96.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:43:32,223 - Train: 9.66% [183800/1902000] [96.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:44:32,648 - Train: 9.67% [183900/1902000] [96.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:45:33,047 - Train: 9.67% [184000/1902000] [96.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:46:33,457 - Train: 9.68% [184100/1902000] [96.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:47:33,855 - Train: 9.68% [184200/1902000] [96.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:48:34,214 - Train: 9.69% [184300/1902000] [96.9/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 06:49:34,605 - Train: 9.70% [184400/1902000] [97.0/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 06:50:31,488 - ==> Total time: 2 days, 5:50:11 Eta: 20 days, 21:10:40 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 06:50:36,795 - Train: 9.70% [184500/1902000] [97.0/1000.0] [batch_t 0.602 (0.661)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 06:51:37,166 - Train: 9.71% [184600/1902000] [97.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:52:37,502 - Train: 9.71% [184700/1902000] [97.1/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:53:37,878 - Train: 9.72% [184800/1902000] [97.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:54:38,265 - Train: 9.72% [184900/1902000] [97.2/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:55:38,622 - Train: 9.73% [185000/1902000] [97.3/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 06:56:38,986 - Train: 9.73% [185100/1902000] [97.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 06:57:39,500 - Train: 9.74% [185200/1902000] [97.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:58:39,877 - Train: 9.74% [185300/1902000] [97.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 06:59:40,306 - Train: 9.75% [185400/1902000] [97.5/1000.0] [batch_t 0.609 (0.604)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 07:00:40,715 - Train: 9.75% [185500/1902000] [97.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:01:41,112 - Train: 9.76% [185600/1902000] [97.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:02:41,586 - Train: 9.76% [185700/1902000] [97.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:03:42,000 - Train: 9.77% [185800/1902000] [97.7/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 07:04:42,388 - Train: 9.77% [185900/1902000] [97.7/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 07:05:42,763 - Train: 9.78% [186000/1902000] [97.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:06:43,171 - Train: 9.78% [186100/1902000] [97.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:07:43,543 - Train: 9.79% [186200/1902000] [97.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 07:08:43,909 - Train: 9.79% [186300/1902000] [97.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 07:09:41,886 - ==> Total time: 2 days, 6:09:21 Eta: 20 days, 18:27:20 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 07:09:46,308 - Train: 9.80% [186400/1902000] [98.0/1000.0] [batch_t 0.608 (0.701)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 07:10:46,756 - Train: 9.81% [186500/1902000] [98.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:11:47,147 - Train: 9.81% [186600/1902000] [98.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 07:12:47,547 - Train: 9.82% [186700/1902000] [98.2/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 07:13:47,970 - Train: 9.82% [186800/1902000] [98.2/1000.0] [batch_t 0.609 (0.604)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 07:14:48,349 - Train: 9.83% [186900/1902000] [98.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 07:15:48,745 - Train: 9.83% [187000/1902000] [98.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 07:16:49,139 - Train: 9.84% [187100/1902000] [98.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 07:17:49,696 - Train: 9.84% [187200/1902000] [98.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 07:18:50,112 - Train: 9.85% [187300/1902000] [98.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:19:50,574 - Train: 9.85% [187400/1902000] [98.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 07:20:50,968 - Train: 9.86% [187500/1902000] [98.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 07:21:51,357 - Train: 9.86% [187600/1902000] [98.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 07:22:51,907 - Train: 9.87% [187700/1902000] [98.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:23:52,317 - Train: 9.87% [187800/1902000] [98.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 07:24:52,688 - Train: 9.88% [187900/1902000] [98.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 07:25:53,068 - Train: 9.88% [188000/1902000] [98.8/1000.0] [batch_t 0.600 (0.604)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-04 07:26:53,452 - Train: 9.89% [188100/1902000] [98.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 07:27:53,809 - Train: 9.89% [188200/1902000] [98.9/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 07:28:52,982 - ==> Total time: 2 days, 6:28:32 Eta: 20 days, 15:47:01 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 07:28:55,647 - Train: 9.90% [188300/1902000] [99.0/1000.0] [batch_t 0.604 (0.683)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:29:56,143 - Train: 9.91% [188400/1902000] [99.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 07:30:56,504 - Train: 9.91% [188500/1902000] [99.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 07:31:56,865 - Train: 9.92% [188600/1902000] [99.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:32:57,247 - Train: 9.92% [188700/1902000] [99.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:33:57,647 - Train: 9.93% [188800/1902000] [99.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:34:58,110 - Train: 9.93% [188900/1902000] [99.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:35:58,500 - Train: 9.94% [189000/1902000] [99.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:36:58,891 - Train: 9.94% [189100/1902000] [99.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 07:37:59,285 - Train: 9.95% [189200/1902000] [99.5/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 07:38:59,661 - Train: 9.95% [189300/1902000] [99.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 07:40:00,068 - Train: 9.96% [189400/1902000] [99.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:41:00,469 - Train: 9.96% [189500/1902000] [99.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 07:42:00,829 - Train: 9.97% [189600/1902000] [99.7/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 07:43:01,283 - Train: 9.97% [189700/1902000] [99.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:44:01,674 - Train: 9.98% [189800/1902000] [99.8/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 07:45:02,041 - Train: 9.98% [189900/1902000] [99.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:46:02,408 - Train: 9.99% [190000/1902000] [99.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:47:02,822 - Train: 9.99% [190100/1902000] [99.9/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 07:48:03,283 - Train: 10.00% [190200/1902000] [100.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:48:03,286 - ==> Total time: 2 days, 6:47:42 Eta: 20 days, 13:09:25 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 07:49:05,787 - Train: 10.01% [190300/1902000] [100.1/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 07:50:06,234 - Train: 10.01% [190400/1902000] [100.1/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 07:51:06,605 - Train: 10.02% [190500/1902000] [100.2/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 07:52:06,939 - Train: 10.02% [190600/1902000] [100.2/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 07:53:07,297 - Train: 10.03% [190700/1902000] [100.3/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 07:54:07,683 - Train: 10.03% [190800/1902000] [100.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 07:55:08,172 - Train: 10.04% [190900/1902000] [100.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 07:56:08,554 - Train: 10.04% [191000/1902000] [100.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 07:57:08,955 - Train: 10.05% [191100/1902000] [100.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 07:58:09,307 - Train: 10.05% [191200/1902000] [100.5/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 07:59:09,698 - Train: 10.06% [191300/1902000] [100.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:00:10,212 - Train: 10.06% [191400/1902000] [100.6/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 08:01:10,607 - Train: 10.07% [191500/1902000] [100.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:02:10,972 - Train: 10.07% [191600/1902000] [100.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:03:11,330 - Train: 10.08% [191700/1902000] [100.8/1000.0] [batch_t 0.607 (0.603)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 08:04:11,727 - Train: 10.08% [191800/1902000] [100.8/1000.0] [batch_t 0.600 (0.604)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-04 08:05:12,114 - Train: 10.09% [191900/1902000] [100.9/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 08:06:12,495 - Train: 10.09% [192000/1902000] [100.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:07:12,897 - Train: 10.10% [192100/1902000] [101.0/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:07:14,107 - ==> Total time: 2 days, 7:06:53 Eta: 20 days, 10:34:37 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 08:08:14,963 - Train: 10.11% [192200/1902000] [101.1/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 08:09:15,352 - Train: 10.11% [192300/1902000] [101.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:10:15,724 - Train: 10.12% [192400/1902000] [101.2/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 08:11:16,118 - Train: 10.12% [192500/1902000] [101.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:12:16,507 - Train: 10.13% [192600/1902000] [101.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:13:16,893 - Train: 10.13% [192700/1902000] [101.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:14:17,286 - Train: 10.14% [192800/1902000] [101.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:15:17,749 - Train: 10.14% [192900/1902000] [101.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:16:18,124 - Train: 10.15% [193000/1902000] [101.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:17:18,543 - Train: 10.15% [193100/1902000] [101.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:18:18,943 - Train: 10.16% [193200/1902000] [101.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:19:19,327 - Train: 10.16% [193300/1902000] [101.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 08:20:19,840 - Train: 10.17% [193400/1902000] [101.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:21:20,248 - Train: 10.17% [193500/1902000] [101.7/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 08:22:21,432 - Train: 10.18% [193600/1902000] [101.8/1000.0] [batch_t 0.606 (0.612)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:23:21,841 - Train: 10.18% [193700/1902000] [101.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:24:22,298 - Train: 10.19% [193800/1902000] [101.9/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 08:25:22,714 - Train: 10.19% [193900/1902000] [101.9/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 08:26:23,292 - Train: 10.20% [194000/1902000] [102.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:26:25,712 - ==> Total time: 2 days, 7:26:05 Eta: 20 days, 8:02:36 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 08:27:25,518 - Train: 10.21% [194100/1902000] [102.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:28:25,929 - Train: 10.21% [194200/1902000] [102.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 08:29:26,314 - Train: 10.22% [194300/1902000] [102.2/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 08:30:26,753 - Train: 10.22% [194400/1902000] [102.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:31:27,133 - Train: 10.23% [194500/1902000] [102.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:32:27,541 - Train: 10.23% [194600/1902000] [102.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 08:33:28,106 - Train: 10.24% [194700/1902000] [102.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 08:34:28,513 - Train: 10.24% [194800/1902000] [102.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:35:28,907 - Train: 10.25% [194900/1902000] [102.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:36:29,304 - Train: 10.25% [195000/1902000] [102.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:37:29,726 - Train: 10.26% [195100/1902000] [102.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:38:30,126 - Train: 10.26% [195200/1902000] [102.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:39:30,622 - Train: 10.27% [195300/1902000] [102.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:40:31,023 - Train: 10.27% [195400/1902000] [102.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:41:31,446 - Train: 10.28% [195500/1902000] [102.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:42:31,828 - Train: 10.28% [195600/1902000] [102.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:43:32,222 - Train: 10.29% [195700/1902000] [102.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:44:32,646 - Train: 10.29% [195800/1902000] [102.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 08:45:33,058 - Train: 10.30% [195900/1902000] [103.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:45:36,686 - ==> Total time: 2 days, 7:45:16 Eta: 20 days, 5:33:04 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 08:46:35,059 - Train: 10.30% [196000/1902000] [103.0/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 08:47:35,468 - Train: 10.31% [196100/1902000] [103.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:48:35,859 - Train: 10.32% [196200/1902000] [103.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:49:36,262 - Train: 10.32% [196300/1902000] [103.2/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 08:50:36,657 - Train: 10.33% [196400/1902000] [103.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 08:51:37,141 - Train: 10.33% [196500/1902000] [103.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:52:37,512 - Train: 10.34% [196600/1902000] [103.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 08:53:37,891 - Train: 10.34% [196700/1902000] [103.4/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 08:54:38,293 - Train: 10.35% [196800/1902000] [103.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 08:55:38,704 - Train: 10.35% [196900/1902000] [103.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:56:39,114 - Train: 10.36% [197000/1902000] [103.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 08:57:39,537 - Train: 10.36% [197100/1902000] [103.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 08:58:39,969 - Train: 10.37% [197200/1902000] [103.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 08:59:40,468 - Train: 10.37% [197300/1902000] [103.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 09:00:40,874 - Train: 10.38% [197400/1902000] [103.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 09:01:41,277 - Train: 10.38% [197500/1902000] [103.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:02:41,665 - Train: 10.39% [197600/1902000] [103.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 09:03:42,118 - Train: 10.39% [197700/1902000] [103.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:04:42,567 - Train: 10.40% [197800/1902000] [104.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 09:04:47,399 - ==> Total time: 2 days, 8:04:26 Eta: 20 days, 3:06:01 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 09:05:44,883 - Train: 10.40% [197900/1902000] [104.0/1000.0] [batch_t 0.603 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 09:06:45,276 - Train: 10.41% [198000/1902000] [104.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:07:45,685 - Train: 10.42% [198100/1902000] [104.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 09:08:46,079 - Train: 10.42% [198200/1902000] [104.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:09:46,502 - Train: 10.43% [198300/1902000] [104.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:10:46,906 - Train: 10.43% [198400/1902000] [104.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 09:11:47,308 - Train: 10.44% [198500/1902000] [104.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 09:12:47,823 - Train: 10.44% [198600/1902000] [104.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 09:13:48,212 - Train: 10.45% [198700/1902000] [104.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 09:14:48,651 - Train: 10.45% [198800/1902000] [104.5/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 09:15:49,027 - Train: 10.46% [198900/1902000] [104.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:16:49,419 - Train: 10.46% [199000/1902000] [104.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 09:17:49,865 - Train: 10.47% [199100/1902000] [104.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:18:50,352 - Train: 10.47% [199200/1902000] [104.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:19:50,740 - Train: 10.48% [199300/1902000] [104.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:20:51,149 - Train: 10.48% [199400/1902000] [104.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:21:51,529 - Train: 10.49% [199500/1902000] [104.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:22:51,894 - Train: 10.49% [199600/1902000] [104.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 09:23:52,310 - Train: 10.50% [199700/1902000] [105.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:23:58,373 - ==> Total time: 2 days, 8:23:37 Eta: 20 days, 0:41:25 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 09:24:54,363 - Train: 10.50% [199800/1902000] [105.0/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 09:25:54,727 - Train: 10.51% [199900/1902000] [105.1/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 09:26:55,156 - Train: 10.52% [200000/1902000] [105.2/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 09:27:55,574 - Train: 10.52% [200100/1902000] [105.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 09:28:55,947 - Train: 10.53% [200200/1902000] [105.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 09:29:56,321 - Train: 10.53% [200300/1902000] [105.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 09:30:56,709 - Train: 10.54% [200400/1902000] [105.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:31:57,097 - Train: 10.54% [200500/1902000] [105.4/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 09:32:57,564 - Train: 10.55% [200600/1902000] [105.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:33:57,984 - Train: 10.55% [200700/1902000] [105.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 09:34:58,395 - Train: 10.56% [200800/1902000] [105.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:35:58,781 - Train: 10.56% [200900/1902000] [105.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 09:36:59,168 - Train: 10.57% [201000/1902000] [105.7/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 09:37:59,640 - Train: 10.57% [201100/1902000] [105.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 09:39:00,006 - Train: 10.58% [201200/1902000] [105.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:40:00,411 - Train: 10.58% [201300/1902000] [105.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:41:00,823 - Train: 10.59% [201400/1902000] [105.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:42:01,228 - Train: 10.59% [201500/1902000] [105.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 09:43:01,635 - Train: 10.60% [201600/1902000] [106.0/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 09:43:08,885 - ==> Total time: 2 days, 8:42:48 Eta: 19 days, 22:19:08 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 09:44:03,592 - Train: 10.60% [201700/1902000] [106.0/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 09:45:04,131 - Train: 10.61% [201800/1902000] [106.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 09:46:04,508 - Train: 10.62% [201900/1902000] [106.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:47:04,894 - Train: 10.62% [202000/1902000] [106.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:48:05,326 - Train: 10.63% [202100/1902000] [106.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:49:05,724 - Train: 10.63% [202200/1902000] [106.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:50:06,126 - Train: 10.64% [202300/1902000] [106.4/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 09:51:06,604 - Train: 10.64% [202400/1902000] [106.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 09:52:06,948 - Train: 10.65% [202500/1902000] [106.5/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:53:07,313 - Train: 10.65% [202600/1902000] [106.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:54:07,718 - Train: 10.66% [202700/1902000] [106.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:55:08,104 - Train: 10.66% [202800/1902000] [106.6/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 09:56:08,487 - Train: 10.67% [202900/1902000] [106.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 09:57:08,895 - Train: 10.67% [203000/1902000] [106.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 09:58:09,359 - Train: 10.68% [203100/1902000] [106.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 09:59:09,747 - Train: 10.68% [203200/1902000] [106.8/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 10:00:10,131 - Train: 10.69% [203300/1902000] [106.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:01:10,532 - Train: 10.69% [203400/1902000] [106.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:02:10,907 - Train: 10.70% [203500/1902000] [107.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:02:19,364 - ==> Total time: 2 days, 9:01:58 Eta: 19 days, 19:59:09 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 10:03:13,067 - Train: 10.70% [203600/1902000] [107.0/1000.0] [batch_t 0.604 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:04:13,467 - Train: 10.71% [203700/1902000] [107.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:05:13,845 - Train: 10.72% [203800/1902000] [107.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:06:14,261 - Train: 10.72% [203900/1902000] [107.2/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 10:07:14,645 - Train: 10.73% [204000/1902000] [107.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:08:15,024 - Train: 10.73% [204100/1902000] [107.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:09:15,375 - Train: 10.74% [204200/1902000] [107.4/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:10:15,769 - Train: 10.74% [204300/1902000] [107.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:11:16,228 - Train: 10.75% [204400/1902000] [107.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:12:16,624 - Train: 10.75% [204500/1902000] [107.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 10:13:17,037 - Train: 10.76% [204600/1902000] [107.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:14:17,408 - Train: 10.76% [204700/1902000] [107.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:15:17,808 - Train: 10.77% [204800/1902000] [107.7/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 10:16:18,310 - Train: 10.77% [204900/1902000] [107.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:17:18,724 - Train: 10.78% [205000/1902000] [107.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:18:19,112 - Train: 10.78% [205100/1902000] [107.8/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 10:19:19,504 - Train: 10.79% [205200/1902000] [107.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 10:20:19,905 - Train: 10.79% [205300/1902000] [107.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:21:20,290 - Train: 10.80% [205400/1902000] [108.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:21:29,955 - ==> Total time: 2 days, 9:21:09 Eta: 19 days, 17:41:24 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 10:22:22,275 - Train: 10.80% [205500/1902000] [108.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:23:22,821 - Train: 10.81% [205600/1902000] [108.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:24:23,207 - Train: 10.81% [205700/1902000] [108.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 10:25:23,604 - Train: 10.82% [205800/1902000] [108.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:26:24,010 - Train: 10.83% [205900/1902000] [108.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:27:24,394 - Train: 10.83% [206000/1902000] [108.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:28:24,892 - Train: 10.84% [206100/1902000] [108.4/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 10:29:25,291 - Train: 10.84% [206200/1902000] [108.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:30:25,713 - Train: 10.85% [206300/1902000] [108.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:31:26,096 - Train: 10.85% [206400/1902000] [108.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 10:32:26,518 - Train: 10.86% [206500/1902000] [108.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:33:26,917 - Train: 10.86% [206600/1902000] [108.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 10:34:27,320 - Train: 10.87% [206700/1902000] [108.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:35:27,715 - Train: 10.87% [206800/1902000] [108.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:36:28,202 - Train: 10.88% [206900/1902000] [108.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 10:37:28,614 - Train: 10.88% [207000/1902000] [108.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:38:29,025 - Train: 10.89% [207100/1902000] [108.9/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 10:39:29,411 - Train: 10.89% [207200/1902000] [108.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:40:29,828 - Train: 10.90% [207300/1902000] [109.0/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 10:40:40,695 - ==> Total time: 2 days, 9:40:20 Eta: 19 days, 15:25:52 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 10:41:31,876 - Train: 10.90% [207400/1902000] [109.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:42:32,262 - Train: 10.91% [207500/1902000] [109.1/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 10:43:32,623 - Train: 10.91% [207600/1902000] [109.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:44:33,006 - Train: 10.92% [207700/1902000] [109.2/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 10:45:33,356 - Train: 10.93% [207800/1902000] [109.3/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:46:33,867 - Train: 10.93% [207900/1902000] [109.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:47:34,247 - Train: 10.94% [208000/1902000] [109.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 10:48:34,649 - Train: 10.94% [208100/1902000] [109.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:49:35,055 - Train: 10.95% [208200/1902000] [109.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:50:35,428 - Train: 10.95% [208300/1902000] [109.5/1000.0] [batch_t 0.600 (0.604)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-04 10:51:35,836 - Train: 10.96% [208400/1902000] [109.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:52:36,186 - Train: 10.96% [208500/1902000] [109.6/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:53:36,584 - Train: 10.97% [208600/1902000] [109.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:54:37,060 - Train: 10.97% [208700/1902000] [109.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:55:37,455 - Train: 10.98% [208800/1902000] [109.8/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 10:56:37,846 - Train: 10.98% [208900/1902000] [109.8/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 10:57:38,252 - Train: 10.99% [209000/1902000] [109.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 10:58:38,665 - Train: 10.99% [209100/1902000] [109.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 10:59:39,151 - Train: 11.00% [209200/1902000] [110.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 10:59:51,230 - ==> Total time: 2 days, 9:59:30 Eta: 19 days, 13:12:25 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 11:00:41,125 - Train: 11.00% [209300/1902000] [110.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:01:41,497 - Train: 11.01% [209400/1902000] [110.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:02:41,907 - Train: 11.01% [209500/1902000] [110.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 11:03:42,258 - Train: 11.02% [209600/1902000] [110.2/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:04:42,627 - Train: 11.03% [209700/1902000] [110.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 11:05:43,010 - Train: 11.03% [209800/1902000] [110.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 11:06:43,521 - Train: 11.04% [209900/1902000] [110.4/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 11:07:43,901 - Train: 11.04% [210000/1902000] [110.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 11:08:44,289 - Train: 11.05% [210100/1902000] [110.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:09:44,664 - Train: 11.05% [210200/1902000] [110.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 11:10:45,020 - Train: 11.06% [210300/1902000] [110.6/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 11:11:45,489 - Train: 11.06% [210400/1902000] [110.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 11:12:45,876 - Train: 11.07% [210500/1902000] [110.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:13:46,271 - Train: 11.07% [210600/1902000] [110.7/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 11:14:46,651 - Train: 11.08% [210700/1902000] [110.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 11:15:47,044 - Train: 11.08% [210800/1902000] [110.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:16:47,426 - Train: 11.09% [210900/1902000] [110.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:17:47,818 - Train: 11.09% [211000/1902000] [110.9/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 11:18:48,168 - Train: 11.10% [211100/1902000] [111.0/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 11:19:01,452 - ==> Total time: 2 days, 10:18:40 Eta: 19 days, 11:00:58 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 11:19:53,078 - Train: 11.10% [211200/1902000] [111.0/1000.0] [batch_t 0.605 (0.623)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:20:53,452 - Train: 11.11% [211300/1902000] [111.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 11:21:53,865 - Train: 11.11% [211400/1902000] [111.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 11:22:54,256 - Train: 11.12% [211500/1902000] [111.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 11:23:54,644 - Train: 11.13% [211600/1902000] [111.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 11:24:55,034 - Train: 11.13% [211700/1902000] [111.3/1000.0] [batch_t 0.610 (0.604)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-04 11:25:55,417 - Train: 11.14% [211800/1902000] [111.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 11:26:55,953 - Train: 11.14% [211900/1902000] [111.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:27:56,351 - Train: 11.15% [212000/1902000] [111.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 11:28:56,714 - Train: 11.15% [212100/1902000] [111.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 11:29:57,095 - Train: 11.16% [212200/1902000] [111.6/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 11:30:57,558 - Train: 11.16% [212300/1902000] [111.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:31:58,252 - Train: 11.17% [212400/1902000] [111.7/1000.0] [batch_t 0.606 (0.607)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-04 11:32:58,822 - Train: 11.17% [212500/1902000] [111.7/1000.0] [batch_t 0.608 (0.606)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-04 11:33:59,379 - Train: 11.18% [212600/1902000] [111.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-04 11:34:59,924 - Train: 11.18% [212700/1902000] [111.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:36:00,467 - Train: 11.19% [212800/1902000] [111.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-04 11:37:01,034 - Train: 11.19% [212900/1902000] [111.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-04 11:38:01,547 - Train: 11.20% [213000/1902000] [112.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:38:16,089 - ==> Total time: 2 days, 10:37:55 Eta: 19 days, 8:52:08 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 11:39:03,812 - Train: 11.20% [213100/1902000] [112.0/1000.0] [batch_t 0.605 (0.610)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-04 11:40:04,344 - Train: 11.21% [213200/1902000] [112.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-04 11:41:04,824 - Train: 11.21% [213300/1902000] [112.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 11:42:05,316 - Train: 11.22% [213400/1902000] [112.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:43:05,825 - Train: 11.23% [213500/1902000] [112.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:44:06,418 - Train: 11.23% [213600/1902000] [112.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-04 11:45:06,919 - Train: 11.24% [213700/1902000] [112.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 11:46:07,457 - Train: 11.24% [213800/1902000] [112.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 11:47:07,982 - Train: 11.25% [213900/1902000] [112.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-04 11:48:08,490 - Train: 11.25% [214000/1902000] [112.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-04 11:49:09,035 - Train: 11.26% [214100/1902000] [112.6/1000.0] [batch_t 0.609 (0.605)] [data_t 0.003] [optim_t 0.607] [lr 0.005000] +2024-04-04 11:50:09,578 - Train: 11.26% [214200/1902000] [112.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-04 11:51:10,120 - Train: 11.27% [214300/1902000] [112.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:52:10,695 - Train: 11.27% [214400/1902000] [112.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:53:11,240 - Train: 11.28% [214500/1902000] [112.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-04 11:54:11,774 - Train: 11.28% [214600/1902000] [112.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 11:55:12,315 - Train: 11.29% [214700/1902000] [112.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 11:56:12,847 - Train: 11.29% [214800/1902000] [112.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-04 11:57:13,490 - Train: 11.30% [214900/1902000] [113.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-04 11:57:29,225 - ==> Total time: 2 days, 10:57:08 Eta: 19 days, 6:45:01 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 11:58:15,494 - Train: 11.30% [215000/1902000] [113.0/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 11:59:16,014 - Train: 11.31% [215100/1902000] [113.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-04 12:00:16,510 - Train: 11.31% [215200/1902000] [113.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:01:16,981 - Train: 11.32% [215300/1902000] [113.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:02:17,492 - Train: 11.32% [215400/1902000] [113.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:03:17,917 - Train: 11.33% [215500/1902000] [113.3/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:04:18,461 - Train: 11.34% [215600/1902000] [113.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 12:05:18,892 - Train: 11.34% [215700/1902000] [113.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:06:19,307 - Train: 11.35% [215800/1902000] [113.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:07:19,749 - Train: 11.35% [215900/1902000] [113.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:08:20,174 - Train: 11.36% [216000/1902000] [113.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:09:20,614 - Train: 11.36% [216100/1902000] [113.6/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 12:10:21,141 - Train: 11.37% [216200/1902000] [113.7/1000.0] [batch_t 0.600 (0.605)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-04 12:11:21,584 - Train: 11.37% [216300/1902000] [113.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:12:22,031 - Train: 11.38% [216400/1902000] [113.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:13:22,494 - Train: 11.38% [216500/1902000] [113.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 12:14:22,931 - Train: 11.39% [216600/1902000] [113.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:15:23,389 - Train: 11.39% [216700/1902000] [113.9/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 12:16:23,855 - Train: 11.40% [216800/1902000] [114.0/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 12:16:40,792 - ==> Total time: 2 days, 11:16:20 Eta: 19 days, 4:39:36 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 12:17:26,190 - Train: 11.40% [216900/1902000] [114.0/1000.0] [batch_t 0.605 (0.613)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:18:26,626 - Train: 11.41% [217000/1902000] [114.1/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:19:27,074 - Train: 11.41% [217100/1902000] [114.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 12:20:27,522 - Train: 11.42% [217200/1902000] [114.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:21:27,959 - Train: 11.42% [217300/1902000] [114.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 12:22:28,512 - Train: 11.43% [217400/1902000] [114.3/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 12:23:28,943 - Train: 11.44% [217500/1902000] [114.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 12:24:29,352 - Train: 11.44% [217600/1902000] [114.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:25:29,815 - Train: 11.45% [217700/1902000] [114.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 12:26:30,277 - Train: 11.45% [217800/1902000] [114.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 12:27:30,702 - Train: 11.46% [217900/1902000] [114.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:28:31,175 - Train: 11.46% [218000/1902000] [114.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:29:31,615 - Train: 11.47% [218100/1902000] [114.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:30:32,133 - Train: 11.47% [218200/1902000] [114.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:31:32,575 - Train: 11.48% [218300/1902000] [114.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:32:33,002 - Train: 11.48% [218400/1902000] [114.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:33:33,427 - Train: 11.49% [218500/1902000] [114.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:34:33,843 - Train: 11.49% [218600/1902000] [114.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:35:34,282 - Train: 11.50% [218700/1902000] [115.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 12:35:52,412 - ==> Total time: 2 days, 11:35:31 Eta: 19 days, 2:36:03 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 12:36:36,470 - Train: 11.50% [218800/1902000] [115.0/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:37:36,883 - Train: 11.51% [218900/1902000] [115.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 12:38:37,303 - Train: 11.51% [219000/1902000] [115.1/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 12:39:37,734 - Train: 11.52% [219100/1902000] [115.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 12:40:38,158 - Train: 11.52% [219200/1902000] [115.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:41:38,596 - Train: 11.53% [219300/1902000] [115.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:42:39,043 - Train: 11.54% [219400/1902000] [115.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:43:39,594 - Train: 11.54% [219500/1902000] [115.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:44:40,010 - Train: 11.55% [219600/1902000] [115.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 12:45:40,439 - Train: 11.55% [219700/1902000] [115.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 12:46:40,850 - Train: 11.56% [219800/1902000] [115.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:47:41,280 - Train: 11.56% [219900/1902000] [115.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:48:41,710 - Train: 11.57% [220000/1902000] [115.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:49:42,226 - Train: 11.57% [220100/1902000] [115.7/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 12:50:42,644 - Train: 11.58% [220200/1902000] [115.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 12:51:43,079 - Train: 11.58% [220300/1902000] [115.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:52:43,512 - Train: 11.59% [220400/1902000] [115.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:53:43,945 - Train: 11.59% [220500/1902000] [115.9/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 12:54:44,410 - Train: 11.60% [220600/1902000] [116.0/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 12:55:03,768 - ==> Total time: 2 days, 11:54:43 Eta: 19 days, 0:34:15 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 12:55:47,200 - Train: 11.60% [220700/1902000] [116.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:56:47,755 - Train: 11.61% [220800/1902000] [116.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 12:57:48,194 - Train: 11.61% [220900/1902000] [116.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:58:48,649 - Train: 11.62% [221000/1902000] [116.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 12:59:49,081 - Train: 11.62% [221100/1902000] [116.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:00:49,563 - Train: 11.63% [221200/1902000] [116.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:01:49,992 - Train: 11.64% [221300/1902000] [116.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:02:50,439 - Train: 11.64% [221400/1902000] [116.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:03:50,978 - Train: 11.65% [221500/1902000] [116.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 13:04:51,436 - Train: 11.65% [221600/1902000] [116.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:05:51,898 - Train: 11.66% [221700/1902000] [116.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:06:52,330 - Train: 11.66% [221800/1902000] [116.6/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 13:07:52,776 - Train: 11.67% [221900/1902000] [116.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:08:53,286 - Train: 11.67% [222000/1902000] [116.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:09:53,729 - Train: 11.68% [222100/1902000] [116.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:10:54,168 - Train: 11.68% [222200/1902000] [116.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:11:54,578 - Train: 11.69% [222300/1902000] [116.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:12:55,009 - Train: 11.69% [222400/1902000] [116.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:13:55,441 - Train: 11.70% [222500/1902000] [117.0/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:14:15,992 - ==> Total time: 2 days, 12:13:55 Eta: 18 days, 22:34:19 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 13:14:57,742 - Train: 11.70% [222600/1902000] [117.0/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:15:58,269 - Train: 11.71% [222700/1902000] [117.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 13:16:58,707 - Train: 11.71% [222800/1902000] [117.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:17:59,142 - Train: 11.72% [222900/1902000] [117.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:18:59,594 - Train: 11.72% [223000/1902000] [117.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:20:00,008 - Train: 11.73% [223100/1902000] [117.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:21:00,546 - Train: 11.74% [223200/1902000] [117.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:22:00,988 - Train: 11.74% [223300/1902000] [117.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:23:01,428 - Train: 11.75% [223400/1902000] [117.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:24:01,861 - Train: 11.75% [223500/1902000] [117.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 13:25:02,294 - Train: 11.76% [223600/1902000] [117.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:26:02,721 - Train: 11.76% [223700/1902000] [117.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:27:03,142 - Train: 11.77% [223800/1902000] [117.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:28:03,556 - Train: 11.77% [223900/1902000] [117.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:29:04,063 - Train: 11.78% [224000/1902000] [117.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:30:04,500 - Train: 11.78% [224100/1902000] [117.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:31:04,931 - Train: 11.79% [224200/1902000] [117.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:32:05,350 - Train: 11.79% [224300/1902000] [117.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:33:05,786 - Train: 11.80% [224400/1902000] [118.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 13:33:27,553 - ==> Total time: 2 days, 12:33:07 Eta: 18 days, 20:36:00 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 13:34:07,910 - Train: 11.80% [224500/1902000] [118.0/1000.0] [batch_t 0.603 (0.611)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 13:35:08,343 - Train: 11.81% [224600/1902000] [118.1/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:36:08,787 - Train: 11.81% [224700/1902000] [118.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:37:09,246 - Train: 11.82% [224800/1902000] [118.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:38:09,673 - Train: 11.82% [224900/1902000] [118.2/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 13:39:10,094 - Train: 11.83% [225000/1902000] [118.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:40:10,531 - Train: 11.83% [225100/1902000] [118.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:41:10,988 - Train: 11.84% [225200/1902000] [118.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 13:42:11,478 - Train: 11.85% [225300/1902000] [118.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:43:11,906 - Train: 11.85% [225400/1902000] [118.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:44:12,354 - Train: 11.86% [225500/1902000] [118.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:45:12,782 - Train: 11.86% [225600/1902000] [118.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:46:13,225 - Train: 11.87% [225700/1902000] [118.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 13:47:13,771 - Train: 11.87% [225800/1902000] [118.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:48:14,198 - Train: 11.88% [225900/1902000] [118.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:49:14,635 - Train: 11.88% [226000/1902000] [118.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:50:15,056 - Train: 11.89% [226100/1902000] [118.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:51:15,488 - Train: 11.89% [226200/1902000] [118.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:52:15,887 - Train: 11.90% [226300/1902000] [119.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:52:38,853 - ==> Total time: 2 days, 12:52:18 Eta: 18 days, 18:39:20 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 13:53:17,865 - Train: 11.90% [226400/1902000] [119.0/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 13:54:18,436 - Train: 11.91% [226500/1902000] [119.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 13:55:18,861 - Train: 11.91% [226600/1902000] [119.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 13:56:19,295 - Train: 11.92% [226700/1902000] [119.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 13:57:19,735 - Train: 11.92% [226800/1902000] [119.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 13:58:20,143 - Train: 11.93% [226900/1902000] [119.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-04 13:59:20,623 - Train: 11.93% [227000/1902000] [119.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 14:00:21,096 - Train: 11.94% [227100/1902000] [119.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 14:01:21,605 - Train: 11.95% [227200/1902000] [119.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:02:22,023 - Train: 11.95% [227300/1902000] [119.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:03:22,454 - Train: 11.96% [227400/1902000] [119.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:04:22,891 - Train: 11.96% [227500/1902000] [119.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:05:23,304 - Train: 11.97% [227600/1902000] [119.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:06:23,734 - Train: 11.97% [227700/1902000] [119.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 14:07:24,262 - Train: 11.98% [227800/1902000] [119.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 14:08:24,684 - Train: 11.98% [227900/1902000] [119.8/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 14:09:25,126 - Train: 11.99% [228000/1902000] [119.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:10:25,518 - Train: 11.99% [228100/1902000] [119.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:11:26,071 - Train: 12.00% [228200/1902000] [120.0/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 14:11:50,234 - ==> Total time: 2 days, 13:11:29 Eta: 18 days, 16:44:18 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 14:12:27,953 - Train: 12.00% [228300/1902000] [120.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:13:28,433 - Train: 12.01% [228400/1902000] [120.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 14:14:28,831 - Train: 12.01% [228500/1902000] [120.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:15:29,225 - Train: 12.02% [228600/1902000] [120.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:16:29,594 - Train: 12.02% [228700/1902000] [120.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 14:17:29,987 - Train: 12.03% [228800/1902000] [120.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:18:30,386 - Train: 12.03% [228900/1902000] [120.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 14:19:30,880 - Train: 12.04% [229000/1902000] [120.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 14:20:31,246 - Train: 12.05% [229100/1902000] [120.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:21:31,658 - Train: 12.05% [229200/1902000] [120.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:22:32,062 - Train: 12.06% [229300/1902000] [120.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:23:32,450 - Train: 12.06% [229400/1902000] [120.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:24:32,827 - Train: 12.07% [229500/1902000] [120.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:25:33,209 - Train: 12.07% [229600/1902000] [120.7/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 14:26:33,655 - Train: 12.08% [229700/1902000] [120.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 14:27:34,043 - Train: 12.08% [229800/1902000] [120.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:28:34,443 - Train: 12.09% [229900/1902000] [120.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 14:29:34,857 - Train: 12.09% [230000/1902000] [120.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:30:35,253 - Train: 12.10% [230100/1902000] [121.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:31:00,613 - ==> Total time: 2 days, 13:30:40 Eta: 18 days, 14:50:43 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 14:31:39,018 - Train: 12.10% [230200/1902000] [121.0/1000.0] [batch_t 0.605 (0.640)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:32:39,393 - Train: 12.11% [230300/1902000] [121.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 14:33:39,801 - Train: 12.11% [230400/1902000] [121.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:34:40,197 - Train: 12.12% [230500/1902000] [121.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 14:35:40,591 - Train: 12.12% [230600/1902000] [121.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 14:36:40,988 - Train: 12.13% [230700/1902000] [121.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 14:37:41,371 - Train: 12.13% [230800/1902000] [121.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:38:41,755 - Train: 12.14% [230900/1902000] [121.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 14:39:42,228 - Train: 12.15% [231000/1902000] [121.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 14:40:42,608 - Train: 12.15% [231100/1902000] [121.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:41:43,011 - Train: 12.16% [231200/1902000] [121.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 14:42:43,400 - Train: 12.16% [231300/1902000] [121.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:43:43,801 - Train: 12.17% [231400/1902000] [121.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:44:44,252 - Train: 12.17% [231500/1902000] [121.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 14:45:44,618 - Train: 12.18% [231600/1902000] [121.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:46:44,991 - Train: 12.18% [231700/1902000] [121.8/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 14:47:45,368 - Train: 12.19% [231800/1902000] [121.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 14:48:45,739 - Train: 12.19% [231900/1902000] [121.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 14:49:46,108 - Train: 12.20% [232000/1902000] [122.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 14:50:12,672 - ==> Total time: 2 days, 13:49:52 Eta: 18 days, 12:58:53 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 14:50:48,037 - Train: 12.20% [232100/1902000] [122.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:51:48,530 - Train: 12.21% [232200/1902000] [122.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:52:48,903 - Train: 12.21% [232300/1902000] [122.1/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 14:53:49,311 - Train: 12.22% [232400/1902000] [122.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:54:49,709 - Train: 12.22% [232500/1902000] [122.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 14:55:50,092 - Train: 12.23% [232600/1902000] [122.3/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 14:56:50,471 - Train: 12.23% [232700/1902000] [122.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 14:57:50,867 - Train: 12.24% [232800/1902000] [122.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:58:51,325 - Train: 12.25% [232900/1902000] [122.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 14:59:51,704 - Train: 12.25% [233000/1902000] [122.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 15:00:52,086 - Train: 12.26% [233100/1902000] [122.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:01:52,478 - Train: 12.26% [233200/1902000] [122.6/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 15:02:52,876 - Train: 12.27% [233300/1902000] [122.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:03:53,287 - Train: 12.27% [233400/1902000] [122.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:04:53,792 - Train: 12.28% [233500/1902000] [122.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 15:05:54,198 - Train: 12.28% [233600/1902000] [122.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:06:54,609 - Train: 12.29% [233700/1902000] [122.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:07:55,038 - Train: 12.29% [233800/1902000] [122.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:08:55,425 - Train: 12.30% [233900/1902000] [123.0/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 15:09:23,211 - ==> Total time: 2 days, 14:09:02 Eta: 18 days, 11:08:23 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 15:09:57,384 - Train: 12.30% [234000/1902000] [123.0/1000.0] [batch_t 0.602 (0.609)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 15:10:57,781 - Train: 12.31% [234100/1902000] [123.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 15:11:58,283 - Train: 12.31% [234200/1902000] [123.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 15:12:58,678 - Train: 12.32% [234300/1902000] [123.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 15:13:59,088 - Train: 12.32% [234400/1902000] [123.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 15:14:59,466 - Train: 12.33% [234500/1902000] [123.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:15:59,874 - Train: 12.33% [234600/1902000] [123.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 15:17:00,372 - Train: 12.34% [234700/1902000] [123.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 15:18:00,744 - Train: 12.34% [234800/1902000] [123.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 15:19:01,106 - Train: 12.35% [234900/1902000] [123.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 15:20:01,513 - Train: 12.36% [235000/1902000] [123.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:21:01,886 - Train: 12.36% [235100/1902000] [123.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:22:02,267 - Train: 12.37% [235200/1902000] [123.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 15:23:02,670 - Train: 12.37% [235300/1902000] [123.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 15:24:03,052 - Train: 12.38% [235400/1902000] [123.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 15:25:03,508 - Train: 12.38% [235500/1902000] [123.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:26:03,885 - Train: 12.39% [235600/1902000] [123.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 15:27:04,256 - Train: 12.39% [235700/1902000] [123.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 15:28:04,660 - Train: 12.40% [235800/1902000] [124.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:28:33,643 - ==> Total time: 2 days, 14:28:13 Eta: 18 days, 9:19:21 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 15:29:06,653 - Train: 12.40% [235900/1902000] [124.0/1000.0] [batch_t 0.603 (0.609)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 15:30:07,048 - Train: 12.41% [236000/1902000] [124.1/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 15:31:07,437 - Train: 12.41% [236100/1902000] [124.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 15:32:07,827 - Train: 12.42% [236200/1902000] [124.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 15:33:08,220 - Train: 12.42% [236300/1902000] [124.2/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 15:34:08,622 - Train: 12.43% [236400/1902000] [124.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:35:09,006 - Train: 12.43% [236500/1902000] [124.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 15:36:09,404 - Train: 12.44% [236600/1902000] [124.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 15:37:09,844 - Train: 12.44% [236700/1902000] [124.4/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 15:38:10,208 - Train: 12.45% [236800/1902000] [124.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:39:10,582 - Train: 12.46% [236900/1902000] [124.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:40:10,965 - Train: 12.46% [237000/1902000] [124.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 15:41:11,337 - Train: 12.47% [237100/1902000] [124.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 15:42:11,783 - Train: 12.47% [237200/1902000] [124.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 15:43:12,158 - Train: 12.48% [237300/1902000] [124.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:44:12,555 - Train: 12.48% [237400/1902000] [124.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:45:12,933 - Train: 12.49% [237500/1902000] [124.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 15:46:13,319 - Train: 12.49% [237600/1902000] [124.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:47:13,689 - Train: 12.50% [237700/1902000] [125.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:47:43,884 - ==> Total time: 2 days, 14:47:23 Eta: 18 days, 7:31:43 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 15:48:15,538 - Train: 12.50% [237800/1902000] [125.0/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:49:16,036 - Train: 12.51% [237900/1902000] [125.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:50:16,422 - Train: 12.51% [238000/1902000] [125.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:51:16,827 - Train: 12.52% [238100/1902000] [125.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:52:17,193 - Train: 12.52% [238200/1902000] [125.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 15:53:17,576 - Train: 12.53% [238300/1902000] [125.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:54:17,955 - Train: 12.53% [238400/1902000] [125.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:55:18,478 - Train: 12.54% [238500/1902000] [125.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 15:56:18,852 - Train: 12.54% [238600/1902000] [125.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:57:19,227 - Train: 12.55% [238700/1902000] [125.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 15:58:19,590 - Train: 12.56% [238800/1902000] [125.6/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 15:59:19,968 - Train: 12.56% [238900/1902000] [125.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 16:00:20,358 - Train: 12.57% [239000/1902000] [125.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 16:01:20,753 - Train: 12.57% [239100/1902000] [125.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:02:21,219 - Train: 12.58% [239200/1902000] [125.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 16:03:21,595 - Train: 12.58% [239300/1902000] [125.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 16:04:21,959 - Train: 12.59% [239400/1902000] [125.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:05:22,339 - Train: 12.59% [239500/1902000] [125.9/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 16:06:22,721 - Train: 12.60% [239600/1902000] [126.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 16:06:54,125 - ==> Total time: 2 days, 15:06:33 Eta: 18 days, 5:45:30 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 16:07:24,923 - Train: 12.60% [239700/1902000] [126.0/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:08:25,322 - Train: 12.61% [239800/1902000] [126.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 16:09:25,680 - Train: 12.61% [239900/1902000] [126.1/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 16:10:26,057 - Train: 12.62% [240000/1902000] [126.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:11:26,437 - Train: 12.62% [240100/1902000] [126.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 16:12:26,888 - Train: 12.63% [240200/1902000] [126.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:13:27,274 - Train: 12.63% [240300/1902000] [126.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:14:27,653 - Train: 12.64% [240400/1902000] [126.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 16:15:28,025 - Train: 12.64% [240500/1902000] [126.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:16:28,412 - Train: 12.65% [240600/1902000] [126.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:17:35,770 - Train: 12.66% [240700/1902000] [126.6/1000.0] [batch_t 0.602 (0.673)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 16:18:36,161 - Train: 12.66% [240800/1902000] [126.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 16:19:36,537 - Train: 12.67% [240900/1902000] [126.7/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 16:20:37,071 - Train: 12.67% [241000/1902000] [126.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:21:37,444 - Train: 12.68% [241100/1902000] [126.8/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 16:22:37,835 - Train: 12.68% [241200/1902000] [126.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:23:38,230 - Train: 12.69% [241300/1902000] [126.9/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 16:24:38,635 - Train: 12.69% [241400/1902000] [126.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:25:39,171 - Train: 12.70% [241500/1902000] [127.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:26:11,772 - ==> Total time: 2 days, 15:25:51 Eta: 18 days, 4:01:30 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 16:26:41,009 - Train: 12.70% [241600/1902000] [127.0/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 16:27:41,400 - Train: 12.71% [241700/1902000] [127.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:28:41,819 - Train: 12.71% [241800/1902000] [127.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:29:44,346 - Train: 12.72% [241900/1902000] [127.2/1000.0] [batch_t 0.606 (0.625)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:30:44,745 - Train: 12.72% [242000/1902000] [127.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:31:45,161 - Train: 12.73% [242100/1902000] [127.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:32:45,712 - Train: 12.73% [242200/1902000] [127.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:33:46,103 - Train: 12.74% [242300/1902000] [127.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:34:46,512 - Train: 12.74% [242400/1902000] [127.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:35:46,912 - Train: 12.75% [242500/1902000] [127.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:36:47,282 - Train: 12.75% [242600/1902000] [127.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 16:37:47,674 - Train: 12.76% [242700/1902000] [127.6/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 16:38:48,176 - Train: 12.77% [242800/1902000] [127.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:39:48,581 - Train: 12.77% [242900/1902000] [127.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:40:48,986 - Train: 12.78% [243000/1902000] [127.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:41:49,354 - Train: 12.78% [243100/1902000] [127.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:42:49,731 - Train: 12.79% [243200/1902000] [127.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:43:50,144 - Train: 12.79% [243300/1902000] [127.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:44:50,535 - Train: 12.80% [243400/1902000] [128.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:45:24,361 - ==> Total time: 2 days, 15:45:03 Eta: 18 days, 2:18:15 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 16:45:52,790 - Train: 12.80% [243500/1902000] [128.0/1000.0] [batch_t 0.606 (0.612)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:46:53,178 - Train: 12.81% [243600/1902000] [128.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:47:53,570 - Train: 12.81% [243700/1902000] [128.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:48:53,958 - Train: 12.82% [243800/1902000] [128.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:49:54,338 - Train: 12.82% [243900/1902000] [128.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:50:54,881 - Train: 12.83% [244000/1902000] [128.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:51:55,238 - Train: 12.83% [244100/1902000] [128.3/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:52:55,626 - Train: 12.84% [244200/1902000] [128.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 16:53:56,028 - Train: 12.84% [244300/1902000] [128.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 16:54:56,405 - Train: 12.85% [244400/1902000] [128.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:55:56,763 - Train: 12.85% [244500/1902000] [128.5/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 16:56:57,139 - Train: 12.86% [244600/1902000] [128.6/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 16:57:57,512 - Train: 12.87% [244700/1902000] [128.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:58:57,957 - Train: 12.87% [244800/1902000] [128.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 16:59:58,335 - Train: 12.88% [244900/1902000] [128.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:00:58,736 - Train: 12.88% [245000/1902000] [128.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:01:59,106 - Train: 12.89% [245100/1902000] [128.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:02:59,502 - Train: 12.89% [245200/1902000] [128.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 17:03:59,983 - Train: 12.90% [245300/1902000] [129.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 17:04:35,015 - ==> Total time: 2 days, 16:04:14 Eta: 18 days, 0:36:05 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 17:05:02,106 - Train: 12.90% [245400/1902000] [129.0/1000.0] [batch_t 0.604 (0.612)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:06:02,870 - Train: 12.91% [245500/1902000] [129.1/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 17:07:03,282 - Train: 12.91% [245600/1902000] [129.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:08:03,664 - Train: 12.92% [245700/1902000] [129.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:09:04,204 - Train: 12.92% [245800/1902000] [129.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 17:10:04,952 - Train: 12.93% [245900/1902000] [129.3/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:11:05,358 - Train: 12.93% [246000/1902000] [129.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:12:05,745 - Train: 12.94% [246100/1902000] [129.4/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 17:13:06,141 - Train: 12.94% [246200/1902000] [129.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:14:06,544 - Train: 12.95% [246300/1902000] [129.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:15:06,927 - Train: 12.95% [246400/1902000] [129.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:16:07,301 - Train: 12.96% [246500/1902000] [129.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:17:08,230 - Train: 12.97% [246600/1902000] [129.7/1000.0] [batch_t 0.603 (0.609)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 17:18:08,608 - Train: 12.97% [246700/1902000] [129.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:19:08,979 - Train: 12.98% [246800/1902000] [129.8/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 17:20:09,346 - Train: 12.98% [246900/1902000] [129.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 17:21:09,716 - Train: 12.99% [247000/1902000] [129.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:22:10,188 - Train: 12.99% [247100/1902000] [129.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:23:10,608 - Train: 13.00% [247200/1902000] [130.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:23:46,860 - ==> Total time: 2 days, 16:23:26 Eta: 17 days, 22:55:19 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 17:24:12,499 - Train: 13.00% [247300/1902000] [130.0/1000.0] [batch_t 0.609 (0.609)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 17:25:12,888 - Train: 13.01% [247400/1902000] [130.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:26:13,522 - Train: 13.01% [247500/1902000] [130.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:27:13,913 - Train: 13.02% [247600/1902000] [130.2/1000.0] [batch_t 0.600 (0.604)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-04 17:28:14,332 - Train: 13.02% [247700/1902000] [130.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:29:14,887 - Train: 13.03% [247800/1902000] [130.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:30:15,279 - Train: 13.03% [247900/1902000] [130.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 17:31:15,699 - Train: 13.04% [248000/1902000] [130.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:32:16,103 - Train: 13.04% [248100/1902000] [130.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:33:16,502 - Train: 13.05% [248200/1902000] [130.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 17:34:16,895 - Train: 13.05% [248300/1902000] [130.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:35:17,395 - Train: 13.06% [248400/1902000] [130.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 17:36:17,803 - Train: 13.07% [248500/1902000] [130.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:37:18,218 - Train: 13.07% [248600/1902000] [130.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 17:38:18,597 - Train: 13.08% [248700/1902000] [130.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 17:39:18,995 - Train: 13.08% [248800/1902000] [130.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:40:19,463 - Train: 13.09% [248900/1902000] [130.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:41:19,873 - Train: 13.09% [249000/1902000] [130.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:42:23,857 - Train: 13.10% [249100/1902000] [131.0/1000.0] [batch_t 0.603 (0.640)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 17:43:01,300 - ==> Total time: 2 days, 16:42:40 Eta: 17 days, 21:16:06 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 17:43:28,886 - Train: 13.10% [249200/1902000] [131.0/1000.0] [batch_t 0.605 (0.628)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:44:29,277 - Train: 13.11% [249300/1902000] [131.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 17:45:29,840 - Train: 13.11% [249400/1902000] [131.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:46:31,342 - Train: 13.12% [249500/1902000] [131.2/1000.0] [batch_t 0.604 (0.615)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 17:47:31,835 - Train: 13.12% [249600/1902000] [131.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 17:48:32,629 - Train: 13.13% [249700/1902000] [131.3/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 17:49:32,997 - Train: 13.13% [249800/1902000] [131.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 17:50:33,760 - Train: 13.14% [249900/1902000] [131.4/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 17:51:34,105 - Train: 13.14% [250000/1902000] [131.4/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:52:34,434 - Train: 13.15% [250100/1902000] [131.5/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:53:34,866 - Train: 13.15% [250200/1902000] [131.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:54:35,909 - Train: 13.16% [250300/1902000] [131.6/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:55:36,278 - Train: 13.17% [250400/1902000] [131.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:56:36,919 - Train: 13.17% [250500/1902000] [131.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 17:57:37,312 - Train: 13.18% [250600/1902000] [131.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 17:58:37,692 - Train: 13.18% [250700/1902000] [131.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 17:59:38,084 - Train: 13.19% [250800/1902000] [131.9/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 18:00:38,612 - Train: 13.19% [250900/1902000] [131.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:01:39,000 - Train: 13.20% [251000/1902000] [132.0/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 18:02:17,654 - ==> Total time: 2 days, 17:01:57 Eta: 17 days, 19:38:17 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 18:02:40,953 - Train: 13.20% [251100/1902000] [132.0/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:03:41,343 - Train: 13.21% [251200/1902000] [132.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 18:04:41,741 - Train: 13.21% [251300/1902000] [132.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 18:05:42,272 - Train: 13.22% [251400/1902000] [132.2/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 18:06:42,654 - Train: 13.22% [251500/1902000] [132.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:07:43,043 - Train: 13.23% [251600/1902000] [132.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 18:08:43,475 - Train: 13.23% [251700/1902000] [132.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 18:09:43,874 - Train: 13.24% [251800/1902000] [132.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:10:44,240 - Train: 13.24% [251900/1902000] [132.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:11:44,622 - Train: 13.25% [252000/1902000] [132.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:12:45,000 - Train: 13.25% [252100/1902000] [132.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 18:13:45,600 - Train: 13.26% [252200/1902000] [132.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 18:14:45,994 - Train: 13.26% [252300/1902000] [132.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:15:46,367 - Train: 13.27% [252400/1902000] [132.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:16:46,739 - Train: 13.28% [252500/1902000] [132.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:17:47,099 - Train: 13.28% [252600/1902000] [132.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:18:47,525 - Train: 13.29% [252700/1902000] [132.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 18:19:47,881 - Train: 13.29% [252800/1902000] [132.9/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 18:20:48,253 - Train: 13.30% [252900/1902000] [133.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:21:28,088 - ==> Total time: 2 days, 17:21:07 Eta: 17 days, 18:01:01 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 18:21:50,464 - Train: 13.30% [253000/1902000] [133.0/1000.0] [batch_t 0.603 (0.609)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 18:22:50,840 - Train: 13.31% [253100/1902000] [133.1/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 18:23:51,226 - Train: 13.31% [253200/1902000] [133.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 18:24:51,734 - Train: 13.32% [253300/1902000] [133.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:25:52,133 - Train: 13.32% [253400/1902000] [133.2/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 18:26:52,522 - Train: 13.33% [253500/1902000] [133.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:27:52,966 - Train: 13.33% [253600/1902000] [133.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:28:53,363 - Train: 13.34% [253700/1902000] [133.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:29:53,751 - Train: 13.34% [253800/1902000] [133.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:30:54,162 - Train: 13.35% [253900/1902000] [133.5/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 18:31:54,606 - Train: 13.35% [254000/1902000] [133.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 18:32:55,000 - Train: 13.36% [254100/1902000] [133.6/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 18:33:55,390 - Train: 13.36% [254200/1902000] [133.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:34:55,770 - Train: 13.37% [254300/1902000] [133.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 18:35:56,141 - Train: 13.38% [254400/1902000] [133.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 18:36:56,522 - Train: 13.38% [254500/1902000] [133.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 18:37:56,918 - Train: 13.39% [254600/1902000] [133.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 18:38:57,365 - Train: 13.39% [254700/1902000] [133.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 18:39:57,749 - Train: 13.40% [254800/1902000] [134.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:40:38,815 - ==> Total time: 2 days, 17:40:18 Eta: 17 days, 16:24:57 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 18:40:59,598 - Train: 13.40% [254900/1902000] [134.0/1000.0] [batch_t 0.609 (0.609)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 18:42:00,009 - Train: 13.41% [255000/1902000] [134.1/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 18:43:00,399 - Train: 13.41% [255100/1902000] [134.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:44:00,933 - Train: 13.42% [255200/1902000] [134.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 18:45:01,300 - Train: 13.42% [255300/1902000] [134.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 18:46:01,684 - Train: 13.43% [255400/1902000] [134.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 18:47:02,079 - Train: 13.43% [255500/1902000] [134.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 18:48:02,455 - Train: 13.44% [255600/1902000] [134.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:49:02,825 - Train: 13.44% [255700/1902000] [134.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 18:50:03,229 - Train: 13.45% [255800/1902000] [134.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 18:51:03,686 - Train: 13.45% [255900/1902000] [134.5/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 18:52:04,036 - Train: 13.46% [256000/1902000] [134.6/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 18:53:04,399 - Train: 13.46% [256100/1902000] [134.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 18:54:04,793 - Train: 13.47% [256200/1902000] [134.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 18:55:05,163 - Train: 13.48% [256300/1902000] [134.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 18:56:05,547 - Train: 13.48% [256400/1902000] [134.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 18:57:05,911 - Train: 13.49% [256500/1902000] [134.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 18:58:06,285 - Train: 13.49% [256600/1902000] [134.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 18:59:06,751 - Train: 13.50% [256700/1902000] [135.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 18:59:49,027 - ==> Total time: 2 days, 17:59:28 Eta: 17 days, 14:49:58 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 19:00:08,669 - Train: 13.50% [256800/1902000] [135.0/1000.0] [batch_t 0.605 (0.611)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:01:09,042 - Train: 13.51% [256900/1902000] [135.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:02:09,389 - Train: 13.51% [257000/1902000] [135.1/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 19:03:09,753 - Train: 13.52% [257100/1902000] [135.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:04:10,270 - Train: 13.52% [257200/1902000] [135.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 19:05:10,634 - Train: 13.53% [257300/1902000] [135.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:06:10,996 - Train: 13.53% [257400/1902000] [135.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:07:11,364 - Train: 13.54% [257500/1902000] [135.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:08:11,749 - Train: 13.54% [257600/1902000] [135.4/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 19:09:12,125 - Train: 13.55% [257700/1902000] [135.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 19:10:12,522 - Train: 13.55% [257800/1902000] [135.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 19:11:12,958 - Train: 13.56% [257900/1902000] [135.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 19:12:13,347 - Train: 13.56% [258000/1902000] [135.6/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 19:13:13,745 - Train: 13.57% [258100/1902000] [135.7/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 19:14:14,116 - Train: 13.58% [258200/1902000] [135.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 19:15:14,509 - Train: 13.58% [258300/1902000] [135.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 19:16:14,902 - Train: 13.59% [258400/1902000] [135.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:17:15,427 - Train: 13.59% [258500/1902000] [135.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 19:18:16,101 - Train: 13.60% [258600/1902000] [136.0/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:18:59,565 - ==> Total time: 2 days, 18:18:39 Eta: 17 days, 13:16:08 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 19:19:18,119 - Train: 13.60% [258700/1902000] [136.0/1000.0] [batch_t 0.604 (0.612)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:20:18,525 - Train: 13.61% [258800/1902000] [136.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 19:21:18,912 - Train: 13.61% [258900/1902000] [136.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 19:22:19,452 - Train: 13.62% [259000/1902000] [136.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:23:19,835 - Train: 13.62% [259100/1902000] [136.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 19:24:20,208 - Train: 13.63% [259200/1902000] [136.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 19:25:20,571 - Train: 13.63% [259300/1902000] [136.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 19:26:20,958 - Train: 13.64% [259400/1902000] [136.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 19:27:21,344 - Train: 13.64% [259500/1902000] [136.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 19:28:21,727 - Train: 13.65% [259600/1902000] [136.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 19:29:22,102 - Train: 13.65% [259700/1902000] [136.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 19:30:22,539 - Train: 13.66% [259800/1902000] [136.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 19:31:22,918 - Train: 13.66% [259900/1902000] [136.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:32:23,322 - Train: 13.67% [260000/1902000] [136.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:33:23,707 - Train: 13.68% [260100/1902000] [136.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:34:24,084 - Train: 13.68% [260200/1902000] [136.8/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 19:35:24,538 - Train: 13.69% [260300/1902000] [136.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 19:36:24,941 - Train: 13.69% [260400/1902000] [136.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 19:37:25,301 - Train: 13.70% [260500/1902000] [137.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 19:38:09,961 - ==> Total time: 2 days, 18:37:49 Eta: 17 days, 11:43:22 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 19:38:27,116 - Train: 13.70% [260600/1902000] [137.0/1000.0] [batch_t 0.606 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 19:39:27,527 - Train: 13.71% [260700/1902000] [137.1/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 19:40:27,884 - Train: 13.71% [260800/1902000] [137.1/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 19:41:28,404 - Train: 13.72% [260900/1902000] [137.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 19:42:28,775 - Train: 13.72% [261000/1902000] [137.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 19:43:29,153 - Train: 13.73% [261100/1902000] [137.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 19:44:29,550 - Train: 13.73% [261200/1902000] [137.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 19:45:29,900 - Train: 13.74% [261300/1902000] [137.4/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 19:46:30,308 - Train: 13.74% [261400/1902000] [137.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:47:30,702 - Train: 13.75% [261500/1902000] [137.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:48:31,059 - Train: 13.75% [261600/1902000] [137.5/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 19:49:31,504 - Train: 13.76% [261700/1902000] [137.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 19:50:31,862 - Train: 13.76% [261800/1902000] [137.6/1000.0] [batch_t 0.610 (0.603)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-04 19:51:32,220 - Train: 13.77% [261900/1902000] [137.7/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:52:32,574 - Train: 13.77% [262000/1902000] [137.7/1000.0] [batch_t 0.606 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 19:53:32,944 - Train: 13.78% [262100/1902000] [137.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 19:54:33,414 - Train: 13.79% [262200/1902000] [137.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:55:33,786 - Train: 13.79% [262300/1902000] [137.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 19:56:34,168 - Train: 13.80% [262400/1902000] [138.0/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 19:57:20,063 - ==> Total time: 2 days, 18:56:59 Eta: 17 days, 10:11:39 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 19:57:36,162 - Train: 13.80% [262500/1902000] [138.0/1000.0] [batch_t 0.605 (0.613)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:58:36,545 - Train: 13.81% [262600/1902000] [138.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 19:59:36,923 - Train: 13.81% [262700/1902000] [138.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 20:00:37,306 - Train: 13.82% [262800/1902000] [138.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 20:01:37,819 - Train: 13.82% [262900/1902000] [138.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 20:02:38,194 - Train: 13.83% [263000/1902000] [138.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 20:03:38,568 - Train: 13.83% [263100/1902000] [138.3/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 20:04:38,965 - Train: 13.84% [263200/1902000] [138.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 20:05:39,347 - Train: 13.84% [263300/1902000] [138.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 20:06:39,884 - Train: 13.85% [263400/1902000] [138.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 20:07:42,709 - Train: 13.85% [263500/1902000] [138.5/1000.0] [batch_t 0.604 (0.628)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 20:08:43,095 - Train: 13.86% [263600/1902000] [138.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 20:09:43,511 - Train: 13.86% [263700/1902000] [138.6/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 20:10:43,915 - Train: 13.87% [263800/1902000] [138.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 20:11:45,465 - Train: 13.87% [263900/1902000] [138.7/1000.0] [batch_t 0.605 (0.615)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 20:12:45,850 - Train: 13.88% [264000/1902000] [138.8/1000.0] [batch_t 0.609 (0.604)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 20:13:46,359 - Train: 13.89% [264100/1902000] [138.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 20:14:46,748 - Train: 13.89% [264200/1902000] [138.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 20:15:47,159 - Train: 13.90% [264300/1902000] [139.0/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 20:16:34,261 - ==> Total time: 2 days, 19:16:13 Eta: 17 days, 8:41:23 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 20:16:49,026 - Train: 13.90% [264400/1902000] [139.0/1000.0] [batch_t 0.602 (0.614)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 20:17:49,443 - Train: 13.91% [264500/1902000] [139.1/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 20:18:49,993 - Train: 13.91% [264600/1902000] [139.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 20:19:50,407 - Train: 13.92% [264700/1902000] [139.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 20:20:50,791 - Train: 13.92% [264800/1902000] [139.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 20:21:51,189 - Train: 13.93% [264900/1902000] [139.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 20:22:51,635 - Train: 13.93% [265000/1902000] [139.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.003] [optim_t 0.600] [lr 0.005000] +2024-04-04 20:23:52,147 - Train: 13.94% [265100/1902000] [139.4/1000.0] [batch_t 0.602 (0.605)] [data_t 0.003] [optim_t 0.600] [lr 0.005000] +2024-04-04 20:24:52,664 - Train: 13.94% [265200/1902000] [139.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-04 20:25:53,279 - Train: 13.95% [265300/1902000] [139.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 20:26:53,836 - Train: 13.95% [265400/1902000] [139.5/1000.0] [batch_t 0.608 (0.605)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-04 20:27:54,413 - Train: 13.96% [265500/1902000] [139.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-04 20:28:54,970 - Train: 13.96% [265600/1902000] [139.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-04 20:29:55,539 - Train: 13.97% [265700/1902000] [139.7/1000.0] [batch_t 0.608 (0.606)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-04 20:30:56,076 - Train: 13.97% [265800/1902000] [139.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-04 20:31:56,606 - Train: 13.98% [265900/1902000] [139.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-04 20:32:57,218 - Train: 13.99% [266000/1902000] [139.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 20:33:57,744 - Train: 13.99% [266100/1902000] [139.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-04 20:34:58,275 - Train: 14.00% [266200/1902000] [140.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-04 20:35:46,700 - ==> Total time: 2 days, 19:35:26 Eta: 17 days, 7:11:58 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 20:36:00,326 - Train: 14.00% [266300/1902000] [140.0/1000.0] [batch_t 0.606 (0.613)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 20:37:00,864 - Train: 14.01% [266400/1902000] [140.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-04 20:38:01,393 - Train: 14.01% [266500/1902000] [140.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-04 20:39:02,039 - Train: 14.02% [266600/1902000] [140.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-04 20:40:02,566 - Train: 14.02% [266700/1902000] [140.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-04 20:41:03,111 - Train: 14.03% [266800/1902000] [140.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 20:42:03,654 - Train: 14.03% [266900/1902000] [140.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-04 20:43:04,183 - Train: 14.04% [267000/1902000] [140.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-04 20:44:04,808 - Train: 14.04% [267100/1902000] [140.4/1000.0] [batch_t 0.603 (0.606)] [data_t 0.003] [optim_t 0.600] [lr 0.005000] +2024-04-04 20:45:05,342 - Train: 14.05% [267200/1902000] [140.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-04 20:46:05,868 - Train: 14.05% [267300/1902000] [140.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 20:47:06,415 - Train: 14.06% [267400/1902000] [140.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-04 20:48:06,928 - Train: 14.06% [267500/1902000] [140.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-04 20:49:07,440 - Train: 14.07% [267600/1902000] [140.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-04 20:50:07,967 - Train: 14.07% [267700/1902000] [140.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-04 20:51:08,627 - Train: 14.08% [267800/1902000] [140.8/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 20:52:09,150 - Train: 14.09% [267900/1902000] [140.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-04 20:53:09,659 - Train: 14.09% [268000/1902000] [140.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 20:54:10,160 - Train: 14.10% [268100/1902000] [141.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 20:54:59,711 - ==> Total time: 2 days, 19:54:39 Eta: 17 days, 5:43:36 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 20:55:12,100 - Train: 14.10% [268200/1902000] [141.0/1000.0] [batch_t 0.605 (0.616)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 20:56:12,555 - Train: 14.11% [268300/1902000] [141.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 20:57:13,015 - Train: 14.11% [268400/1902000] [141.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 20:58:13,607 - Train: 14.12% [268500/1902000] [141.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 20:59:14,027 - Train: 14.12% [268600/1902000] [141.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:00:14,469 - Train: 14.13% [268700/1902000] [141.3/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 21:01:14,934 - Train: 14.13% [268800/1902000] [141.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:02:15,366 - Train: 14.14% [268900/1902000] [141.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:03:15,986 - Train: 14.14% [269000/1902000] [141.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 21:04:16,466 - Train: 14.15% [269100/1902000] [141.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 21:05:16,941 - Train: 14.15% [269200/1902000] [141.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 21:06:17,394 - Train: 14.16% [269300/1902000] [141.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:07:17,875 - Train: 14.16% [269400/1902000] [141.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:08:18,317 - Train: 14.17% [269500/1902000] [141.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 21:09:18,959 - Train: 14.17% [269600/1902000] [141.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 21:10:19,426 - Train: 14.18% [269700/1902000] [141.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 21:11:20,047 - Train: 14.19% [269800/1902000] [141.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:12:20,523 - Train: 14.19% [269900/1902000] [141.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 21:13:20,985 - Train: 14.20% [270000/1902000] [142.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 21:14:12,859 - ==> Total time: 2 days, 20:13:52 Eta: 17 days, 4:16:13 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 21:14:24,141 - Train: 14.20% [270100/1902000] [142.0/1000.0] [batch_t 0.607 (0.620)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 21:15:24,546 - Train: 14.21% [270200/1902000] [142.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 21:16:25,114 - Train: 14.21% [270300/1902000] [142.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 21:17:26,419 - Train: 14.22% [270400/1902000] [142.2/1000.0] [batch_t 0.603 (0.613)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 21:18:26,883 - Train: 14.22% [270500/1902000] [142.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:19:40,963 - Train: 14.23% [270600/1902000] [142.3/1000.0] [batch_t 0.605 (0.741)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:20:41,410 - Train: 14.23% [270700/1902000] [142.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 21:21:41,862 - Train: 14.24% [270800/1902000] [142.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:22:42,345 - Train: 14.24% [270900/1902000] [142.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 21:23:42,872 - Train: 14.25% [271000/1902000] [142.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 21:24:54,864 - Train: 14.25% [271100/1902000] [142.5/1000.0] [batch_t 0.607 (0.720)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 21:25:55,341 - Train: 14.26% [271200/1902000] [142.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:26:55,841 - Train: 14.26% [271300/1902000] [142.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:27:56,314 - Train: 14.27% [271400/1902000] [142.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:28:56,785 - Train: 14.27% [271500/1902000] [142.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:29:57,387 - Train: 14.28% [271600/1902000] [142.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:30:57,893 - Train: 14.28% [271700/1902000] [142.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 21:31:58,365 - Train: 14.29% [271800/1902000] [142.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:32:58,859 - Train: 14.30% [271900/1902000] [143.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 21:33:50,866 - ==> Total time: 2 days, 20:33:30 Eta: 17 days, 2:52:16 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 21:34:00,976 - Train: 14.30% [272000/1902000] [143.0/1000.0] [batch_t 0.605 (0.620)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:35:01,862 - Train: 14.31% [272100/1902000] [143.1/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:36:02,294 - Train: 14.31% [272200/1902000] [143.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:37:02,751 - Train: 14.32% [272300/1902000] [143.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:38:03,206 - Train: 14.32% [272400/1902000] [143.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 21:39:03,667 - Train: 14.33% [272500/1902000] [143.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:40:04,143 - Train: 14.33% [272600/1902000] [143.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 21:41:05,640 - Train: 14.34% [272700/1902000] [143.4/1000.0] [batch_t 0.605 (0.615)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:42:06,138 - Train: 14.34% [272800/1902000] [143.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 21:43:06,622 - Train: 14.35% [272900/1902000] [143.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 21:44:10,915 - Train: 14.35% [273000/1902000] [143.5/1000.0] [batch_t 4.435 (0.643)] [data_t 3.834] [optim_t 0.601] [lr 0.005000] +2024-04-04 21:45:14,391 - Train: 14.36% [273100/1902000] [143.6/1000.0] [batch_t 0.606 (0.635)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 21:46:14,952 - Train: 14.36% [273200/1902000] [143.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 21:47:15,439 - Train: 14.37% [273300/1902000] [143.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-04 21:48:16,040 - Train: 14.37% [273400/1902000] [143.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:49:16,521 - Train: 14.38% [273500/1902000] [143.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:50:17,961 - Train: 14.38% [273600/1902000] [143.8/1000.0] [batch_t 0.609 (0.614)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 21:51:18,466 - Train: 14.39% [273700/1902000] [143.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 21:52:18,933 - Train: 14.40% [273800/1902000] [144.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:53:12,149 - ==> Total time: 2 days, 20:52:51 Eta: 17 days, 1:27:33 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 21:53:21,332 - Train: 14.40% [273900/1902000] [144.0/1000.0] [batch_t 0.602 (0.623)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-04 21:54:21,832 - Train: 14.41% [274000/1902000] [144.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 21:55:22,331 - Train: 14.41% [274100/1902000] [144.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 21:56:22,791 - Train: 14.42% [274200/1902000] [144.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 21:57:23,261 - Train: 14.42% [274300/1902000] [144.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-04 21:58:23,730 - Train: 14.43% [274400/1902000] [144.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 21:59:24,390 - Train: 14.43% [274500/1902000] [144.3/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:00:24,866 - Train: 14.44% [274600/1902000] [144.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.003] [optim_t 0.601] [lr 0.005000] +2024-04-04 22:01:25,334 - Train: 14.44% [274700/1902000] [144.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 22:02:25,839 - Train: 14.45% [274800/1902000] [144.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:03:26,305 - Train: 14.45% [274900/1902000] [144.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 22:04:26,789 - Train: 14.46% [275000/1902000] [144.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 22:05:27,451 - Train: 14.46% [275100/1902000] [144.6/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 22:06:27,987 - Train: 14.47% [275200/1902000] [144.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 22:07:28,494 - Train: 14.47% [275300/1902000] [144.7/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-04 22:08:28,964 - Train: 14.48% [275400/1902000] [144.8/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 22:09:29,459 - Train: 14.48% [275500/1902000] [144.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 22:10:29,927 - Train: 14.49% [275600/1902000] [144.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 22:11:30,435 - Train: 14.50% [275700/1902000] [145.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 22:12:24,965 - ==> Total time: 2 days, 21:12:04 Eta: 17 days, 0:02:55 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 22:12:32,505 - Train: 14.50% [275800/1902000] [145.0/1000.0] [batch_t 0.604 (0.624)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 22:13:32,954 - Train: 14.51% [275900/1902000] [145.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 22:14:33,396 - Train: 14.51% [276000/1902000] [145.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 22:15:33,881 - Train: 14.52% [276100/1902000] [145.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 22:16:34,342 - Train: 14.52% [276200/1902000] [145.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:17:35,197 - Train: 14.53% [276300/1902000] [145.3/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:18:35,815 - Train: 14.53% [276400/1902000] [145.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 22:19:36,272 - Train: 14.54% [276500/1902000] [145.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:20:36,743 - Train: 14.54% [276600/1902000] [145.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 22:21:37,190 - Train: 14.55% [276700/1902000] [145.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 22:22:37,653 - Train: 14.55% [276800/1902000] [145.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:23:38,118 - Train: 14.56% [276900/1902000] [145.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 22:24:38,582 - Train: 14.56% [277000/1902000] [145.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 22:25:39,114 - Train: 14.57% [277100/1902000] [145.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 22:26:39,562 - Train: 14.57% [277200/1902000] [145.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 22:27:40,019 - Train: 14.58% [277300/1902000] [145.8/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 22:28:40,511 - Train: 14.58% [277400/1902000] [145.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:29:41,036 - Train: 14.59% [277500/1902000] [145.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 22:30:41,482 - Train: 14.60% [277600/1902000] [146.0/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 22:31:37,223 - ==> Total time: 2 days, 21:31:16 Eta: 16 days, 22:39:07 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 22:31:43,861 - Train: 14.60% [277700/1902000] [146.0/1000.0] [batch_t 0.605 (0.636)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:32:44,481 - Train: 14.61% [277800/1902000] [146.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 22:33:44,940 - Train: 14.61% [277900/1902000] [146.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 22:34:45,387 - Train: 14.62% [278000/1902000] [146.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 22:35:45,867 - Train: 14.62% [278100/1902000] [146.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:36:46,329 - Train: 14.63% [278200/1902000] [146.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.003] [optim_t 0.601] [lr 0.005000] +2024-04-04 22:37:46,767 - Train: 14.63% [278300/1902000] [146.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:38:47,334 - Train: 14.64% [278400/1902000] [146.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:39:47,821 - Train: 14.64% [278500/1902000] [146.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 22:40:48,283 - Train: 14.65% [278600/1902000] [146.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:41:48,724 - Train: 14.65% [278700/1902000] [146.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 22:42:49,211 - Train: 14.66% [278800/1902000] [146.6/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-04 22:43:49,766 - Train: 14.66% [278900/1902000] [146.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:44:50,208 - Train: 14.67% [279000/1902000] [146.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 22:45:50,660 - Train: 14.67% [279100/1902000] [146.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:46:51,082 - Train: 14.68% [279200/1902000] [146.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:47:51,550 - Train: 14.68% [279300/1902000] [146.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 22:48:51,995 - Train: 14.69% [279400/1902000] [146.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:49:52,451 - Train: 14.70% [279500/1902000] [147.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 22:50:49,317 - ==> Total time: 2 days, 21:50:28 Eta: 16 days, 21:16:11 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 22:50:55,021 - Train: 14.70% [279600/1902000] [147.0/1000.0] [batch_t 0.605 (0.630)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:51:55,484 - Train: 14.71% [279700/1902000] [147.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:52:55,960 - Train: 14.71% [279800/1902000] [147.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 22:53:56,431 - Train: 14.72% [279900/1902000] [147.2/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-04 22:54:56,918 - Train: 14.72% [280000/1902000] [147.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:55:57,371 - Train: 14.73% [280100/1902000] [147.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:56:57,848 - Train: 14.73% [280200/1902000] [147.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 22:57:58,443 - Train: 14.74% [280300/1902000] [147.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:58:58,995 - Train: 14.74% [280400/1902000] [147.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 22:59:59,464 - Train: 14.75% [280500/1902000] [147.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 23:00:59,940 - Train: 14.75% [280600/1902000] [147.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 23:02:02,638 - Train: 14.76% [280700/1902000] [147.6/1000.0] [batch_t 0.605 (0.627)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:03:03,172 - Train: 14.76% [280800/1902000] [147.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 23:04:04,923 - Train: 14.77% [280900/1902000] [147.7/1000.0] [batch_t 0.605 (0.617)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:05:05,580 - Train: 14.77% [281000/1902000] [147.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 23:06:26,461 - Train: 14.78% [281100/1902000] [147.8/1000.0] [batch_t 0.610 (0.809)] [data_t 0.003] [optim_t 0.608] [lr 0.005000] +2024-04-04 23:07:27,850 - Train: 14.78% [281200/1902000] [147.8/1000.0] [batch_t 0.605 (0.614)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:08:40,427 - Train: 14.79% [281300/1902000] [147.9/1000.0] [batch_t 0.605 (0.726)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:09:45,006 - Train: 14.79% [281400/1902000] [147.9/1000.0] [batch_t 0.605 (0.646)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:11:24,484 - ==> Total time: 2 days, 22:11:04 Eta: 16 days, 20:02:05 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 23:11:28,793 - Train: 14.80% [281500/1902000] [148.0/1000.0] [batch_t 0.603 (0.688)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 23:13:00,180 - Train: 14.81% [281600/1902000] [148.1/1000.0] [batch_t 0.606 (0.914)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 23:14:37,024 - Train: 14.81% [281700/1902000] [148.1/1000.0] [batch_t 0.603 (0.968)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 23:15:38,940 - Train: 14.82% [281800/1902000] [148.2/1000.0] [batch_t 0.603 (0.619)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 23:16:40,941 - Train: 14.82% [281900/1902000] [148.2/1000.0] [batch_t 0.604 (0.620)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 23:17:42,978 - Train: 14.83% [282000/1902000] [148.3/1000.0] [batch_t 0.606 (0.620)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 23:18:44,413 - Train: 14.83% [282100/1902000] [148.3/1000.0] [batch_t 0.605 (0.614)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:19:45,536 - Train: 14.84% [282200/1902000] [148.4/1000.0] [batch_t 0.606 (0.611)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 23:20:46,405 - Train: 14.84% [282300/1902000] [148.4/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:21:47,107 - Train: 14.85% [282400/1902000] [148.5/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:22:47,516 - Train: 14.85% [282500/1902000] [148.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 23:23:49,086 - Train: 14.86% [282600/1902000] [148.6/1000.0] [batch_t 0.605 (0.616)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:24:49,508 - Train: 14.86% [282700/1902000] [148.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:25:50,024 - Train: 14.87% [282800/1902000] [148.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:26:50,592 - Train: 14.87% [282900/1902000] [148.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:27:51,045 - Train: 14.88% [283000/1902000] [148.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 23:28:52,350 - Train: 14.88% [283100/1902000] [148.8/1000.0] [batch_t 0.603 (0.613)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 23:29:52,810 - Train: 14.89% [283200/1902000] [148.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 23:30:53,480 - Train: 14.89% [283300/1902000] [148.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 23:31:52,710 - ==> Total time: 2 days, 22:31:32 Eta: 16 days, 18:48:02 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 23:31:55,788 - Train: 14.90% [283400/1902000] [149.0/1000.0] [batch_t 0.606 (0.695)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 23:32:57,083 - Train: 14.91% [283500/1902000] [149.1/1000.0] [batch_t 0.605 (0.613)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:33:57,516 - Train: 14.91% [283600/1902000] [149.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:34:57,910 - Train: 14.92% [283700/1902000] [149.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 23:35:58,343 - Train: 14.92% [283800/1902000] [149.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:36:58,765 - Train: 14.93% [283900/1902000] [149.3/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 23:37:59,181 - Train: 14.93% [284000/1902000] [149.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:38:59,722 - Train: 14.94% [284100/1902000] [149.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:40:00,130 - Train: 14.94% [284200/1902000] [149.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 23:41:00,546 - Train: 14.95% [284300/1902000] [149.5/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-04 23:42:00,964 - Train: 14.95% [284400/1902000] [149.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 23:43:01,383 - Train: 14.96% [284500/1902000] [149.6/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-04 23:44:01,768 - Train: 14.96% [284600/1902000] [149.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:45:02,263 - Train: 14.97% [284700/1902000] [149.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-04 23:46:03,462 - Train: 14.97% [284800/1902000] [149.7/1000.0] [batch_t 0.604 (0.612)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 23:47:03,868 - Train: 14.98% [284900/1902000] [149.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:48:04,256 - Train: 14.98% [285000/1902000] [149.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:49:04,701 - Train: 14.99% [285100/1902000] [149.9/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 23:50:05,112 - Train: 14.99% [285200/1902000] [149.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:51:05,535 - Train: 15.00% [285300/1902000] [150.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:51:05,538 - ==> Total time: 2 days, 22:50:45 Eta: 16 days, 17:27:35 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-04 23:52:07,805 - Train: 15.01% [285400/1902000] [150.1/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:53:08,186 - Train: 15.01% [285500/1902000] [150.1/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 23:54:08,592 - Train: 15.02% [285600/1902000] [150.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:55:08,969 - Train: 15.02% [285700/1902000] [150.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 23:56:09,334 - Train: 15.03% [285800/1902000] [150.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-04 23:57:09,815 - Train: 15.03% [285900/1902000] [150.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-04 23:58:10,171 - Train: 15.04% [286000/1902000] [150.4/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-04 23:59:10,545 - Train: 15.04% [286100/1902000] [150.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:00:10,941 - Train: 15.05% [286200/1902000] [150.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:01:11,341 - Train: 15.05% [286300/1902000] [150.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:02:11,693 - Train: 15.06% [286400/1902000] [150.6/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:03:12,119 - Train: 15.06% [286500/1902000] [150.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:04:12,528 - Train: 15.07% [286600/1902000] [150.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:05:13,839 - Train: 15.07% [286700/1902000] [150.7/1000.0] [batch_t 0.602 (0.613)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 00:06:14,261 - Train: 15.08% [286800/1902000] [150.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:07:14,648 - Train: 15.08% [286900/1902000] [150.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:08:15,444 - Train: 15.09% [287000/1902000] [150.9/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:09:15,856 - Train: 15.09% [287100/1902000] [150.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:10:16,276 - Train: 15.10% [287200/1902000] [151.0/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:10:17,486 - ==> Total time: 2 days, 23:09:57 Eta: 16 days, 16:07:51 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 00:11:18,514 - Train: 15.11% [287300/1902000] [151.1/1000.0] [batch_t 0.735 (0.609)] [data_t 0.002] [optim_t 0.733] [lr 0.005000] +2024-04-05 00:12:18,916 - Train: 15.11% [287400/1902000] [151.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:13:19,372 - Train: 15.12% [287500/1902000] [151.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 00:14:19,776 - Train: 15.12% [287600/1902000] [151.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:15:20,151 - Train: 15.13% [287700/1902000] [151.3/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 00:16:20,559 - Train: 15.13% [287800/1902000] [151.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:17:21,086 - Train: 15.14% [287900/1902000] [151.4/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 00:18:21,480 - Train: 15.14% [288000/1902000] [151.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 00:19:21,914 - Train: 15.15% [288100/1902000] [151.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:20:22,310 - Train: 15.15% [288200/1902000] [151.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:21:22,724 - Train: 15.16% [288300/1902000] [151.6/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 00:22:23,111 - Train: 15.16% [288400/1902000] [151.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:23:23,536 - Train: 15.17% [288500/1902000] [151.7/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 00:24:24,080 - Train: 15.17% [288600/1902000] [151.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:25:24,457 - Train: 15.18% [288700/1902000] [151.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:26:24,843 - Train: 15.18% [288800/1902000] [151.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:27:25,263 - Train: 15.19% [288900/1902000] [151.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 00:28:25,658 - Train: 15.19% [289000/1902000] [151.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:29:26,087 - Train: 15.20% [289100/1902000] [152.0/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 00:29:28,504 - ==> Total time: 2 days, 23:29:08 Eta: 16 days, 14:48:51 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 00:30:28,475 - Train: 15.21% [289200/1902000] [152.1/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:31:28,904 - Train: 15.21% [289300/1902000] [152.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:32:29,335 - Train: 15.22% [289400/1902000] [152.2/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 00:33:29,784 - Train: 15.22% [289500/1902000] [152.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:34:30,208 - Train: 15.23% [289600/1902000] [152.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:35:30,661 - Train: 15.23% [289700/1902000] [152.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:36:31,070 - Train: 15.24% [289800/1902000] [152.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 00:37:31,604 - Train: 15.24% [289900/1902000] [152.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:38:32,036 - Train: 15.25% [290000/1902000] [152.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:39:32,445 - Train: 15.25% [290100/1902000] [152.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:40:32,893 - Train: 15.26% [290200/1902000] [152.6/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 00:41:33,330 - Train: 15.26% [290300/1902000] [152.6/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 00:42:33,843 - Train: 15.27% [290400/1902000] [152.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:43:34,252 - Train: 15.27% [290500/1902000] [152.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:44:34,655 - Train: 15.28% [290600/1902000] [152.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:45:35,073 - Train: 15.28% [290700/1902000] [152.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 00:46:35,490 - Train: 15.29% [290800/1902000] [152.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:47:35,895 - Train: 15.29% [290900/1902000] [152.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:48:36,319 - Train: 15.30% [291000/1902000] [153.0/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 00:48:39,956 - ==> Total time: 2 days, 23:48:19 Eta: 16 days, 13:30:39 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 00:49:38,573 - Train: 15.30% [291100/1902000] [153.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 00:50:38,983 - Train: 15.31% [291200/1902000] [153.1/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 00:51:39,365 - Train: 15.32% [291300/1902000] [153.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:52:39,768 - Train: 15.32% [291400/1902000] [153.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:53:40,176 - Train: 15.33% [291500/1902000] [153.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:54:40,576 - Train: 15.33% [291600/1902000] [153.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:55:41,072 - Train: 15.34% [291700/1902000] [153.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 00:56:41,480 - Train: 15.34% [291800/1902000] [153.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:57:41,874 - Train: 15.35% [291900/1902000] [153.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 00:58:42,267 - Train: 15.35% [292000/1902000] [153.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 00:59:42,662 - Train: 15.36% [292100/1902000] [153.6/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 01:00:43,083 - Train: 15.36% [292200/1902000] [153.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 01:01:43,505 - Train: 15.37% [292300/1902000] [153.7/1000.0] [batch_t 0.601 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 01:02:43,998 - Train: 15.37% [292400/1902000] [153.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:03:44,406 - Train: 15.38% [292500/1902000] [153.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:04:44,944 - Train: 15.38% [292600/1902000] [153.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 01:05:45,342 - Train: 15.39% [292700/1902000] [153.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 01:06:45,739 - Train: 15.39% [292800/1902000] [153.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 01:07:46,141 - Train: 15.40% [292900/1902000] [154.0/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 01:07:51,062 - ==> Total time: 3 days, 0:07:30 Eta: 16 days, 12:13:12 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 01:08:48,480 - Train: 15.40% [293000/1902000] [154.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 01:09:48,924 - Train: 15.41% [293100/1902000] [154.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:10:49,380 - Train: 15.42% [293200/1902000] [154.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 01:11:49,757 - Train: 15.42% [293300/1902000] [154.2/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 01:12:50,328 - Train: 15.43% [293400/1902000] [154.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 01:13:50,758 - Train: 15.43% [293500/1902000] [154.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 01:14:51,146 - Train: 15.44% [293600/1902000] [154.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 01:15:51,583 - Train: 15.44% [293700/1902000] [154.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:16:51,999 - Train: 15.45% [293800/1902000] [154.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:17:52,429 - Train: 15.45% [293900/1902000] [154.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 01:18:52,831 - Train: 15.46% [294000/1902000] [154.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:19:53,429 - Train: 15.46% [294100/1902000] [154.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:20:53,949 - Train: 15.47% [294200/1902000] [154.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 01:21:54,338 - Train: 15.47% [294300/1902000] [154.7/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 01:22:54,751 - Train: 15.48% [294400/1902000] [154.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:23:55,138 - Train: 15.48% [294500/1902000] [154.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:24:55,856 - Train: 15.49% [294600/1902000] [154.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 01:25:56,440 - Train: 15.49% [294700/1902000] [154.9/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 01:26:56,868 - Train: 15.50% [294800/1902000] [155.0/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 01:27:02,906 - ==> Total time: 3 days, 0:26:42 Eta: 16 days, 10:56:33 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 01:28:00,541 - Train: 15.50% [294900/1902000] [155.0/1000.0] [batch_t 0.605 (0.611)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:29:00,933 - Train: 15.51% [295000/1902000] [155.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:30:01,372 - Train: 15.52% [295100/1902000] [155.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 01:31:01,760 - Train: 15.52% [295200/1902000] [155.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:32:02,201 - Train: 15.53% [295300/1902000] [155.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:33:02,909 - Train: 15.53% [295400/1902000] [155.3/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:34:03,324 - Train: 15.54% [295500/1902000] [155.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 01:35:03,738 - Train: 15.54% [295600/1902000] [155.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:36:04,132 - Train: 15.55% [295700/1902000] [155.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:37:04,531 - Train: 15.55% [295800/1902000] [155.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 01:38:04,938 - Train: 15.56% [295900/1902000] [155.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:39:05,434 - Train: 15.56% [296000/1902000] [155.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:40:05,823 - Train: 15.57% [296100/1902000] [155.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:41:06,209 - Train: 15.57% [296200/1902000] [155.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:42:06,578 - Train: 15.58% [296300/1902000] [155.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:43:07,011 - Train: 15.58% [296400/1902000] [155.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:44:07,437 - Train: 15.59% [296500/1902000] [155.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 01:45:07,831 - Train: 15.59% [296600/1902000] [155.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 01:46:08,300 - Train: 15.60% [296700/1902000] [156.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:46:15,543 - ==> Total time: 3 days, 0:45:55 Eta: 16 days, 9:40:44 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 01:47:10,437 - Train: 15.60% [296800/1902000] [156.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 01:48:10,834 - Train: 15.61% [296900/1902000] [156.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 01:49:11,256 - Train: 15.62% [297000/1902000] [156.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:50:11,666 - Train: 15.62% [297100/1902000] [156.2/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 01:51:12,207 - Train: 15.63% [297200/1902000] [156.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:52:12,570 - Train: 15.63% [297300/1902000] [156.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:53:13,104 - Train: 15.64% [297400/1902000] [156.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 01:54:13,504 - Train: 15.64% [297500/1902000] [156.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:55:13,910 - Train: 15.65% [297600/1902000] [156.5/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 01:56:14,323 - Train: 15.65% [297700/1902000] [156.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 01:57:14,821 - Train: 15.66% [297800/1902000] [156.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 01:58:15,203 - Train: 15.66% [297900/1902000] [156.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 01:59:15,667 - Train: 15.67% [298000/1902000] [156.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 02:00:16,055 - Train: 15.67% [298100/1902000] [156.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 02:01:16,477 - Train: 15.68% [298200/1902000] [156.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:02:16,863 - Train: 15.68% [298300/1902000] [156.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:03:17,952 - Train: 15.69% [298400/1902000] [156.9/1000.0] [batch_t 0.605 (0.611)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:04:18,437 - Train: 15.69% [298500/1902000] [156.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:05:18,824 - Train: 15.70% [298600/1902000] [157.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:05:27,290 - ==> Total time: 3 days, 1:05:06 Eta: 16 days, 8:25:32 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 02:06:21,052 - Train: 15.70% [298700/1902000] [157.0/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:07:21,493 - Train: 15.71% [298800/1902000] [157.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 02:08:21,880 - Train: 15.72% [298900/1902000] [157.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:09:24,019 - Train: 15.72% [299000/1902000] [157.2/1000.0] [batch_t 0.603 (0.621)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 02:10:24,400 - Train: 15.73% [299100/1902000] [157.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:11:24,768 - Train: 15.73% [299200/1902000] [157.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 02:12:25,162 - Train: 15.74% [299300/1902000] [157.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 02:13:25,529 - Train: 15.74% [299400/1902000] [157.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 02:14:25,948 - Train: 15.75% [299500/1902000] [157.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 02:15:26,343 - Train: 15.75% [299600/1902000] [157.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 02:16:26,768 - Train: 15.76% [299700/1902000] [157.6/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 02:17:27,345 - Train: 15.76% [299800/1902000] [157.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:18:28,065 - Train: 15.77% [299900/1902000] [157.7/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 02:19:28,489 - Train: 15.77% [300000/1902000] [157.7/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 02:20:28,926 - Train: 15.78% [300100/1902000] [157.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:21:29,369 - Train: 15.78% [300200/1902000] [157.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 02:22:29,979 - Train: 15.79% [300300/1902000] [157.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:23:30,409 - Train: 15.79% [300400/1902000] [157.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 02:24:30,837 - Train: 15.80% [300500/1902000] [158.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 02:24:40,495 - ==> Total time: 3 days, 1:24:20 Eta: 16 days, 7:11:11 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 02:25:33,021 - Train: 15.80% [300600/1902000] [158.0/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:26:33,498 - Train: 15.81% [300700/1902000] [158.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 02:27:33,925 - Train: 15.81% [300800/1902000] [158.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:28:34,320 - Train: 15.82% [300900/1902000] [158.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:29:34,876 - Train: 15.83% [301000/1902000] [158.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:30:35,311 - Train: 15.83% [301100/1902000] [158.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:31:35,684 - Train: 15.84% [301200/1902000] [158.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 02:32:36,086 - Train: 15.84% [301300/1902000] [158.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 02:33:36,495 - Train: 15.85% [301400/1902000] [158.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:34:36,914 - Train: 15.85% [301500/1902000] [158.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:35:37,390 - Train: 15.86% [301600/1902000] [158.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 02:36:37,825 - Train: 15.86% [301700/1902000] [158.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:37:38,201 - Train: 15.87% [301800/1902000] [158.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 02:38:38,600 - Train: 15.87% [301900/1902000] [158.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 02:39:38,996 - Train: 15.88% [302000/1902000] [158.8/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 02:40:39,358 - Train: 15.88% [302100/1902000] [158.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 02:41:39,759 - Train: 15.89% [302200/1902000] [158.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:42:40,261 - Train: 15.89% [302300/1902000] [158.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 02:43:40,645 - Train: 15.90% [302400/1902000] [159.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 02:43:51,525 - ==> Total time: 3 days, 1:43:31 Eta: 16 days, 5:57:20 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 02:44:42,606 - Train: 15.90% [302500/1902000] [159.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:45:43,011 - Train: 15.91% [302600/1902000] [159.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:46:43,425 - Train: 15.91% [302700/1902000] [159.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 02:47:43,964 - Train: 15.92% [302800/1902000] [159.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 02:48:44,373 - Train: 15.93% [302900/1902000] [159.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 02:49:44,792 - Train: 15.93% [303000/1902000] [159.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:50:45,185 - Train: 15.94% [303100/1902000] [159.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 02:51:45,619 - Train: 15.94% [303200/1902000] [159.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 02:52:45,978 - Train: 15.95% [303300/1902000] [159.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 02:53:46,411 - Train: 15.95% [303400/1902000] [159.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:54:46,827 - Train: 15.96% [303500/1902000] [159.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 02:55:47,330 - Train: 15.96% [303600/1902000] [159.6/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 02:56:47,748 - Train: 15.97% [303700/1902000] [159.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:57:48,133 - Train: 15.97% [303800/1902000] [159.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:58:48,517 - Train: 15.98% [303900/1902000] [159.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 02:59:48,949 - Train: 15.98% [304000/1902000] [159.8/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 03:00:49,434 - Train: 15.99% [304100/1902000] [159.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 03:01:49,818 - Train: 15.99% [304200/1902000] [159.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:02:50,190 - Train: 16.00% [304300/1902000] [160.0/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 03:03:02,267 - ==> Total time: 3 days, 2:02:41 Eta: 16 days, 4:44:09 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 03:03:54,834 - Train: 16.00% [304400/1902000] [160.0/1000.0] [batch_t 0.606 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:04:55,224 - Train: 16.01% [304500/1902000] [160.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:05:55,625 - Train: 16.01% [304600/1902000] [160.1/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 03:06:56,028 - Train: 16.02% [304700/1902000] [160.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 03:07:56,522 - Train: 16.03% [304800/1902000] [160.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:08:56,916 - Train: 16.03% [304900/1902000] [160.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 03:09:57,336 - Train: 16.04% [305000/1902000] [160.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 03:10:57,730 - Train: 16.04% [305100/1902000] [160.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:11:58,114 - Train: 16.05% [305200/1902000] [160.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:12:58,485 - Train: 16.05% [305300/1902000] [160.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 03:13:58,952 - Train: 16.06% [305400/1902000] [160.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:14:59,331 - Train: 16.06% [305500/1902000] [160.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 03:15:59,746 - Train: 16.07% [305600/1902000] [160.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:17:00,132 - Train: 16.07% [305700/1902000] [160.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:18:00,545 - Train: 16.08% [305800/1902000] [160.8/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 03:19:00,977 - Train: 16.08% [305900/1902000] [160.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:20:01,374 - Train: 16.09% [306000/1902000] [160.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:21:01,902 - Train: 16.09% [306100/1902000] [160.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 03:22:02,298 - Train: 16.10% [306200/1902000] [161.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:22:15,586 - ==> Total time: 3 days, 2:21:55 Eta: 16 days, 3:31:51 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 03:23:04,288 - Train: 16.10% [306300/1902000] [161.0/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 03:24:04,698 - Train: 16.11% [306400/1902000] [161.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 03:25:05,062 - Train: 16.11% [306500/1902000] [161.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:26:05,469 - Train: 16.12% [306600/1902000] [161.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:27:05,987 - Train: 16.13% [306700/1902000] [161.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:28:06,373 - Train: 16.13% [306800/1902000] [161.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 03:29:06,791 - Train: 16.14% [306900/1902000] [161.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:30:07,171 - Train: 16.14% [307000/1902000] [161.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 03:31:07,554 - Train: 16.15% [307100/1902000] [161.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 03:32:07,895 - Train: 16.15% [307200/1902000] [161.5/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:33:08,363 - Train: 16.16% [307300/1902000] [161.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:34:08,754 - Train: 16.16% [307400/1902000] [161.6/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 03:35:09,119 - Train: 16.17% [307500/1902000] [161.7/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 03:36:09,491 - Train: 16.17% [307600/1902000] [161.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:37:09,876 - Train: 16.18% [307700/1902000] [161.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 03:38:10,277 - Train: 16.18% [307800/1902000] [161.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:39:10,685 - Train: 16.19% [307900/1902000] [161.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 03:40:11,205 - Train: 16.19% [308000/1902000] [161.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:41:11,643 - Train: 16.20% [308100/1902000] [162.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:41:26,148 - ==> Total time: 3 days, 2:41:05 Eta: 16 days, 2:19:58 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 03:42:13,998 - Train: 16.20% [308200/1902000] [162.0/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:43:14,389 - Train: 16.21% [308300/1902000] [162.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:44:14,779 - Train: 16.21% [308400/1902000] [162.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 03:45:15,186 - Train: 16.22% [308500/1902000] [162.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:46:15,564 - Train: 16.23% [308600/1902000] [162.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:47:16,060 - Train: 16.23% [308700/1902000] [162.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:48:16,461 - Train: 16.24% [308800/1902000] [162.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 03:49:16,873 - Train: 16.24% [308900/1902000] [162.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 03:50:17,277 - Train: 16.25% [309000/1902000] [162.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 03:51:17,660 - Train: 16.25% [309100/1902000] [162.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 03:52:18,141 - Train: 16.26% [309200/1902000] [162.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:53:18,543 - Train: 16.26% [309300/1902000] [162.6/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 03:54:18,941 - Train: 16.27% [309400/1902000] [162.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 03:55:19,359 - Train: 16.27% [309500/1902000] [162.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 03:56:19,746 - Train: 16.28% [309600/1902000] [162.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:57:20,135 - Train: 16.28% [309700/1902000] [162.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 03:58:20,527 - Train: 16.29% [309800/1902000] [162.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 03:59:21,001 - Train: 16.29% [309900/1902000] [162.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:00:21,403 - Train: 16.30% [310000/1902000] [163.0/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 04:00:37,145 - ==> Total time: 3 days, 3:00:16 Eta: 16 days, 1:08:47 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 04:01:23,388 - Train: 16.30% [310100/1902000] [163.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 04:02:23,804 - Train: 16.31% [310200/1902000] [163.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 04:03:24,162 - Train: 16.31% [310300/1902000] [163.1/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 04:04:24,717 - Train: 16.32% [310400/1902000] [163.2/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 04:05:25,117 - Train: 16.32% [310500/1902000] [163.2/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 04:06:25,480 - Train: 16.33% [310600/1902000] [163.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 04:07:25,858 - Train: 16.34% [310700/1902000] [163.4/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 04:08:26,237 - Train: 16.34% [310800/1902000] [163.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 04:09:26,613 - Train: 16.35% [310900/1902000] [163.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:10:27,001 - Train: 16.35% [311000/1902000] [163.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 04:11:28,120 - Train: 16.36% [311100/1902000] [163.6/1000.0] [batch_t 0.604 (0.611)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 04:12:28,558 - Train: 16.36% [311200/1902000] [163.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 04:13:28,951 - Train: 16.37% [311300/1902000] [163.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 04:14:29,356 - Train: 16.37% [311400/1902000] [163.7/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 04:15:29,779 - Train: 16.38% [311500/1902000] [163.8/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 04:16:30,183 - Train: 16.38% [311600/1902000] [163.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:17:30,660 - Train: 16.39% [311700/1902000] [163.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:18:31,041 - Train: 16.39% [311800/1902000] [163.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 04:19:31,451 - Train: 16.40% [311900/1902000] [164.0/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 04:19:48,367 - ==> Total time: 3 days, 3:19:27 Eta: 15 days, 23:58:14 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 04:20:33,543 - Train: 16.40% [312000/1902000] [164.0/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:21:33,918 - Train: 16.41% [312100/1902000] [164.1/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 04:22:34,320 - Train: 16.41% [312200/1902000] [164.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 04:23:34,844 - Train: 16.42% [312300/1902000] [164.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:24:35,259 - Train: 16.42% [312400/1902000] [164.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 04:25:35,649 - Train: 16.43% [312500/1902000] [164.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:26:36,016 - Train: 16.44% [312600/1902000] [164.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:27:36,427 - Train: 16.44% [312700/1902000] [164.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 04:28:36,805 - Train: 16.45% [312800/1902000] [164.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:29:37,242 - Train: 16.45% [312900/1902000] [164.5/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 04:30:37,625 - Train: 16.46% [313000/1902000] [164.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 04:31:38,090 - Train: 16.46% [313100/1902000] [164.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 04:32:38,484 - Train: 16.47% [313200/1902000] [164.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:33:38,866 - Train: 16.47% [313300/1902000] [164.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:34:39,249 - Train: 16.48% [313400/1902000] [164.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:35:39,611 - Train: 16.48% [313500/1902000] [164.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:36:40,100 - Train: 16.49% [313600/1902000] [164.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 04:37:40,489 - Train: 16.49% [313700/1902000] [164.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:38:40,891 - Train: 16.50% [313800/1902000] [165.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:38:59,012 - ==> Total time: 3 days, 3:38:38 Eta: 15 days, 22:48:16 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 04:39:42,851 - Train: 16.50% [313900/1902000] [165.0/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 04:40:43,253 - Train: 16.51% [314000/1902000] [165.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 04:41:43,677 - Train: 16.51% [314100/1902000] [165.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:42:44,080 - Train: 16.52% [314200/1902000] [165.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:43:44,597 - Train: 16.52% [314300/1902000] [165.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:44:45,027 - Train: 16.53% [314400/1902000] [165.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:45:45,448 - Train: 16.54% [314500/1902000] [165.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:46:45,850 - Train: 16.54% [314600/1902000] [165.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 04:47:46,238 - Train: 16.55% [314700/1902000] [165.5/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 04:48:46,797 - Train: 16.55% [314800/1902000] [165.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 04:49:47,214 - Train: 16.56% [314900/1902000] [165.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 04:50:47,617 - Train: 16.56% [315000/1902000] [165.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 04:51:47,990 - Train: 16.57% [315100/1902000] [165.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 04:52:48,381 - Train: 16.57% [315200/1902000] [165.7/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 04:53:48,788 - Train: 16.58% [315300/1902000] [165.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 04:54:49,191 - Train: 16.58% [315400/1902000] [165.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 04:55:49,588 - Train: 16.59% [315500/1902000] [165.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 04:56:50,078 - Train: 16.59% [315600/1902000] [165.9/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 04:57:50,479 - Train: 16.60% [315700/1902000] [166.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 04:58:09,823 - ==> Total time: 3 days, 3:57:49 Eta: 15 days, 21:38:56 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 04:58:52,420 - Train: 16.60% [315800/1902000] [166.0/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 04:59:52,819 - Train: 16.61% [315900/1902000] [166.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:00:53,243 - Train: 16.61% [316000/1902000] [166.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:01:53,593 - Train: 16.62% [316100/1902000] [166.2/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 05:02:54,084 - Train: 16.62% [316200/1902000] [166.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 05:03:54,455 - Train: 16.63% [316300/1902000] [166.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 05:04:55,398 - Train: 16.64% [316400/1902000] [166.4/1000.0] [batch_t 0.604 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:05:55,785 - Train: 16.64% [316500/1902000] [166.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:06:56,163 - Train: 16.65% [316600/1902000] [166.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 05:07:56,537 - Train: 16.65% [316700/1902000] [166.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 05:08:57,042 - Train: 16.66% [316800/1902000] [166.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 05:09:57,425 - Train: 16.66% [316900/1902000] [166.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 05:10:57,822 - Train: 16.67% [317000/1902000] [166.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:11:58,181 - Train: 16.67% [317100/1902000] [166.7/1000.0] [batch_t 0.605 (0.603)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:12:58,569 - Train: 16.68% [317200/1902000] [166.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 05:13:59,008 - Train: 16.68% [317300/1902000] [166.8/1000.0] [batch_t 0.610 (0.604)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-05 05:14:59,423 - Train: 16.69% [317400/1902000] [166.9/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 05:15:59,885 - Train: 16.69% [317500/1902000] [166.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 05:17:00,300 - Train: 16.70% [317600/1902000] [167.0/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 05:17:20,828 - ==> Total time: 3 days, 4:17:00 Eta: 15 days, 20:30:12 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 05:18:02,499 - Train: 16.70% [317700/1902000] [167.0/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 05:19:02,862 - Train: 16.71% [317800/1902000] [167.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:20:03,329 - Train: 16.71% [317900/1902000] [167.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:21:04,021 - Train: 16.72% [318000/1902000] [167.2/1000.0] [batch_t 0.607 (0.607)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 05:22:04,538 - Train: 16.72% [318100/1902000] [167.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 05:23:05,065 - Train: 16.73% [318200/1902000] [167.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 05:24:05,596 - Train: 16.74% [318300/1902000] [167.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 05:25:06,117 - Train: 16.74% [318400/1902000] [167.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-05 05:26:06,668 - Train: 16.75% [318500/1902000] [167.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:27:07,243 - Train: 16.75% [318600/1902000] [167.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 05:28:07,843 - Train: 16.76% [318700/1902000] [167.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:29:08,571 - Train: 16.76% [318800/1902000] [167.6/1000.0] [batch_t 0.607 (0.607)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 05:30:09,116 - Train: 16.77% [318900/1902000] [167.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 05:31:09,676 - Train: 16.77% [319000/1902000] [167.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-05 05:32:10,220 - Train: 16.78% [319100/1902000] [167.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-05 05:33:10,742 - Train: 16.78% [319200/1902000] [167.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:34:11,364 - Train: 16.79% [319300/1902000] [167.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:35:11,877 - Train: 16.79% [319400/1902000] [167.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 05:36:12,402 - Train: 16.80% [319500/1902000] [168.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:36:34,181 - ==> Total time: 3 days, 4:36:13 Eta: 15 days, 19:22:16 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 05:37:14,608 - Train: 16.80% [319600/1902000] [168.0/1000.0] [batch_t 0.607 (0.611)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-05 05:38:15,146 - Train: 16.81% [319700/1902000] [168.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 05:39:15,664 - Train: 16.81% [319800/1902000] [168.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 05:40:16,188 - Train: 16.82% [319900/1902000] [168.2/1000.0] [batch_t 0.602 (0.605)] [data_t 0.003] [optim_t 0.600] [lr 0.005000] +2024-04-05 05:41:16,864 - Train: 16.82% [320000/1902000] [168.2/1000.0] [batch_t 0.602 (0.607)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 05:42:17,395 - Train: 16.83% [320100/1902000] [168.3/1000.0] [batch_t 0.608 (0.605)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-05 05:43:17,916 - Train: 16.83% [320200/1902000] [168.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:44:18,436 - Train: 16.84% [320300/1902000] [168.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-05 05:45:18,943 - Train: 16.85% [320400/1902000] [168.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 05:46:19,562 - Train: 16.85% [320500/1902000] [168.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:47:20,063 - Train: 16.86% [320600/1902000] [168.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.003] [optim_t 0.601] [lr 0.005000] +2024-04-05 05:48:20,588 - Train: 16.86% [320700/1902000] [168.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:49:21,128 - Train: 16.87% [320800/1902000] [168.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 05:50:21,678 - Train: 16.87% [320900/1902000] [168.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 05:51:22,181 - Train: 16.88% [321000/1902000] [168.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 05:52:22,627 - Train: 16.88% [321100/1902000] [168.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 05:53:23,078 - Train: 16.89% [321200/1902000] [168.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:54:23,606 - Train: 16.89% [321300/1902000] [168.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 05:55:24,061 - Train: 16.90% [321400/1902000] [169.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:55:47,025 - ==> Total time: 3 days, 4:55:26 Eta: 15 days, 18:14:52 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 05:56:25,988 - Train: 16.90% [321500/1902000] [169.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 05:57:26,426 - Train: 16.91% [321600/1902000] [169.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 05:58:26,849 - Train: 16.91% [321700/1902000] [169.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 05:59:27,483 - Train: 16.92% [321800/1902000] [169.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:00:27,930 - Train: 16.92% [321900/1902000] [169.2/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 06:01:28,397 - Train: 16.93% [322000/1902000] [169.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 06:02:28,852 - Train: 16.93% [322100/1902000] [169.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 06:03:29,305 - Train: 16.94% [322200/1902000] [169.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:04:29,754 - Train: 16.95% [322300/1902000] [169.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:05:30,205 - Train: 16.95% [322400/1902000] [169.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:06:30,731 - Train: 16.96% [322500/1902000] [169.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:07:31,186 - Train: 16.96% [322600/1902000] [169.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:08:31,632 - Train: 16.97% [322700/1902000] [169.7/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 06:09:32,078 - Train: 16.97% [322800/1902000] [169.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:10:32,529 - Train: 16.98% [322900/1902000] [169.8/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 06:11:33,060 - Train: 16.98% [323000/1902000] [169.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 06:12:33,481 - Train: 16.99% [323100/1902000] [169.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 06:13:33,910 - Train: 16.99% [323200/1902000] [169.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 06:14:34,366 - Train: 17.00% [323300/1902000] [170.0/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 06:14:58,542 - ==> Total time: 3 days, 5:14:38 Eta: 15 days, 17:07:55 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 06:15:36,334 - Train: 17.00% [323400/1902000] [170.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:16:36,797 - Train: 17.01% [323500/1902000] [170.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 06:17:37,250 - Train: 17.01% [323600/1902000] [170.1/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 06:18:37,675 - Train: 17.02% [323700/1902000] [170.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:19:38,213 - Train: 17.02% [323800/1902000] [170.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:20:38,656 - Train: 17.03% [323900/1902000] [170.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:21:39,097 - Train: 17.03% [324000/1902000] [170.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:22:39,549 - Train: 17.04% [324100/1902000] [170.4/1000.0] [batch_t 0.610 (0.604)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-05 06:23:40,007 - Train: 17.05% [324200/1902000] [170.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:24:40,518 - Train: 17.05% [324300/1902000] [170.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:25:40,965 - Train: 17.06% [324400/1902000] [170.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:26:41,422 - Train: 17.06% [324500/1902000] [170.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:27:41,867 - Train: 17.07% [324600/1902000] [170.7/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 06:28:42,325 - Train: 17.07% [324700/1902000] [170.7/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 06:29:42,793 - Train: 17.08% [324800/1902000] [170.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:30:43,267 - Train: 17.08% [324900/1902000] [170.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:31:43,767 - Train: 17.09% [325000/1902000] [170.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:32:44,208 - Train: 17.09% [325100/1902000] [170.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 06:33:44,691 - Train: 17.10% [325200/1902000] [171.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:34:10,115 - ==> Total time: 3 days, 5:33:49 Eta: 15 days, 16:01:32 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 06:34:47,021 - Train: 17.10% [325300/1902000] [171.0/1000.0] [batch_t 0.603 (0.608)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 06:35:47,479 - Train: 17.11% [325400/1902000] [171.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:36:47,934 - Train: 17.11% [325500/1902000] [171.1/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 06:37:48,475 - Train: 17.12% [325600/1902000] [171.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 06:38:48,904 - Train: 17.12% [325700/1902000] [171.2/1000.0] [batch_t 0.609 (0.604)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 06:39:49,370 - Train: 17.13% [325800/1902000] [171.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:40:49,793 - Train: 17.13% [325900/1902000] [171.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:41:50,255 - Train: 17.14% [326000/1902000] [171.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:42:50,707 - Train: 17.15% [326100/1902000] [171.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 06:43:51,270 - Train: 17.15% [326200/1902000] [171.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 06:44:51,733 - Train: 17.16% [326300/1902000] [171.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:45:52,180 - Train: 17.16% [326400/1902000] [171.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:46:52,651 - Train: 17.17% [326500/1902000] [171.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:47:53,082 - Train: 17.17% [326600/1902000] [171.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 06:48:53,535 - Train: 17.18% [326700/1902000] [171.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:49:54,030 - Train: 17.18% [326800/1902000] [171.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:50:54,565 - Train: 17.19% [326900/1902000] [171.9/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 06:51:55,038 - Train: 17.19% [327000/1902000] [171.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:52:55,503 - Train: 17.20% [327100/1902000] [172.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 06:53:22,106 - ==> Total time: 3 days, 5:53:01 Eta: 15 days, 14:55:44 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 06:53:57,536 - Train: 17.20% [327200/1902000] [172.0/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:54:58,035 - Train: 17.21% [327300/1902000] [172.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 06:55:58,661 - Train: 17.21% [327400/1902000] [172.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:56:59,142 - Train: 17.22% [327500/1902000] [172.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:57:59,624 - Train: 17.22% [327600/1902000] [172.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 06:59:00,088 - Train: 17.23% [327700/1902000] [172.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:00:00,553 - Train: 17.23% [327800/1902000] [172.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:01:01,037 - Train: 17.24% [327900/1902000] [172.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:02:01,551 - Train: 17.25% [328000/1902000] [172.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:03:02,059 - Train: 17.25% [328100/1902000] [172.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:04:02,590 - Train: 17.26% [328200/1902000] [172.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:05:03,056 - Train: 17.26% [328300/1902000] [172.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.003] [optim_t 0.601] [lr 0.005000] +2024-04-05 07:06:03,500 - Train: 17.27% [328400/1902000] [172.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:07:03,958 - Train: 17.27% [328500/1902000] [172.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:08:04,430 - Train: 17.28% [328600/1902000] [172.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:09:04,980 - Train: 17.28% [328700/1902000] [172.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:10:05,471 - Train: 17.29% [328800/1902000] [172.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:11:06,001 - Train: 17.29% [328900/1902000] [172.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:12:06,451 - Train: 17.30% [329000/1902000] [173.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:12:34,271 - ==> Total time: 3 days, 6:12:13 Eta: 15 days, 13:50:29 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 07:13:08,536 - Train: 17.30% [329100/1902000] [173.0/1000.0] [batch_t 0.606 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:14:09,066 - Train: 17.31% [329200/1902000] [173.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:15:09,524 - Train: 17.31% [329300/1902000] [173.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:16:10,122 - Train: 17.32% [329400/1902000] [173.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:17:10,608 - Train: 17.32% [329500/1902000] [173.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:18:11,105 - Train: 17.33% [329600/1902000] [173.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:19:11,619 - Train: 17.33% [329700/1902000] [173.3/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 07:20:12,098 - Train: 17.34% [329800/1902000] [173.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 07:21:12,691 - Train: 17.34% [329900/1902000] [173.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:22:13,157 - Train: 17.35% [330000/1902000] [173.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:23:13,603 - Train: 17.36% [330100/1902000] [173.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:24:14,067 - Train: 17.36% [330200/1902000] [173.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:25:14,514 - Train: 17.37% [330300/1902000] [173.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:26:14,982 - Train: 17.37% [330400/1902000] [173.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:27:15,439 - Train: 17.38% [330500/1902000] [173.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:28:15,929 - Train: 17.38% [330600/1902000] [173.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:29:16,469 - Train: 17.39% [330700/1902000] [173.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:30:16,940 - Train: 17.39% [330800/1902000] [173.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:31:17,425 - Train: 17.40% [330900/1902000] [174.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:31:46,445 - ==> Total time: 3 days, 6:31:25 Eta: 15 days, 12:45:46 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 07:32:19,898 - Train: 17.40% [331000/1902000] [174.0/1000.0] [batch_t 0.606 (0.616)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:33:20,357 - Train: 17.41% [331100/1902000] [174.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 07:34:20,979 - Train: 17.41% [331200/1902000] [174.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:35:21,418 - Train: 17.42% [331300/1902000] [174.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:36:21,886 - Train: 17.42% [331400/1902000] [174.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:37:22,348 - Train: 17.43% [331500/1902000] [174.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:38:22,812 - Train: 17.43% [331600/1902000] [174.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:39:23,257 - Train: 17.44% [331700/1902000] [174.4/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:40:23,699 - Train: 17.44% [331800/1902000] [174.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:41:24,238 - Train: 17.45% [331900/1902000] [174.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:42:24,676 - Train: 17.46% [332000/1902000] [174.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 07:43:25,139 - Train: 17.46% [332100/1902000] [174.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:44:25,552 - Train: 17.47% [332200/1902000] [174.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:45:25,980 - Train: 17.47% [332300/1902000] [174.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:46:26,439 - Train: 17.48% [332400/1902000] [174.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 07:47:27,003 - Train: 17.48% [332500/1902000] [174.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:48:27,449 - Train: 17.49% [332600/1902000] [174.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:49:27,889 - Train: 17.49% [332700/1902000] [174.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:50:28,328 - Train: 17.50% [332800/1902000] [175.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:50:58,558 - ==> Total time: 3 days, 6:50:38 Eta: 15 days, 11:41:33 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 07:51:30,493 - Train: 17.50% [332900/1902000] [175.0/1000.0] [batch_t 0.604 (0.609)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:52:30,951 - Train: 17.51% [333000/1902000] [175.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:53:31,531 - Train: 17.51% [333100/1902000] [175.1/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 07:54:31,973 - Train: 17.52% [333200/1902000] [175.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:55:32,413 - Train: 17.52% [333300/1902000] [175.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:56:32,904 - Train: 17.53% [333400/1902000] [175.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 07:57:33,350 - Train: 17.53% [333500/1902000] [175.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 07:58:33,825 - Train: 17.54% [333600/1902000] [175.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 07:59:34,413 - Train: 17.54% [333700/1902000] [175.4/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 08:00:34,931 - Train: 17.55% [333800/1902000] [175.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:01:35,406 - Train: 17.56% [333900/1902000] [175.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 08:02:35,919 - Train: 17.56% [334000/1902000] [175.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:03:36,466 - Train: 17.57% [334100/1902000] [175.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:04:36,923 - Train: 17.57% [334200/1902000] [175.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:05:37,468 - Train: 17.58% [334300/1902000] [175.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:06:38,108 - Train: 17.58% [334400/1902000] [175.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 08:07:38,601 - Train: 17.59% [334500/1902000] [175.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 08:08:39,099 - Train: 17.59% [334600/1902000] [175.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:09:39,581 - Train: 17.60% [334700/1902000] [176.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 08:10:11,035 - ==> Total time: 3 days, 7:09:50 Eta: 15 days, 10:37:53 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 08:10:41,851 - Train: 17.60% [334800/1902000] [176.0/1000.0] [batch_t 0.603 (0.608)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 08:11:42,421 - Train: 17.61% [334900/1902000] [176.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 08:12:42,834 - Train: 17.61% [335000/1902000] [176.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:13:43,283 - Train: 17.62% [335100/1902000] [176.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:14:43,678 - Train: 17.62% [335200/1902000] [176.2/1000.0] [batch_t 0.607 (0.604)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 08:15:44,092 - Train: 17.63% [335300/1902000] [176.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 08:16:44,496 - Train: 17.63% [335400/1902000] [176.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:17:44,930 - Train: 17.64% [335500/1902000] [176.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 08:18:45,342 - Train: 17.64% [335600/1902000] [176.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:19:45,904 - Train: 17.65% [335700/1902000] [176.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 08:20:46,319 - Train: 17.66% [335800/1902000] [176.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:21:46,729 - Train: 17.66% [335900/1902000] [176.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 08:22:47,167 - Train: 17.67% [336000/1902000] [176.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 08:23:47,592 - Train: 17.67% [336100/1902000] [176.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 08:24:48,178 - Train: 17.68% [336200/1902000] [176.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:25:48,603 - Train: 17.68% [336300/1902000] [176.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 08:26:49,023 - Train: 17.69% [336400/1902000] [176.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 08:27:49,424 - Train: 17.69% [336500/1902000] [176.9/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 08:28:49,849 - Train: 17.70% [336600/1902000] [177.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:29:22,484 - ==> Total time: 3 days, 7:29:02 Eta: 15 days, 9:34:39 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 08:29:51,950 - Train: 17.70% [336700/1902000] [177.0/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:30:52,348 - Train: 17.71% [336800/1902000] [177.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:31:52,760 - Train: 17.71% [336900/1902000] [177.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:32:53,139 - Train: 17.72% [337000/1902000] [177.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:33:53,541 - Train: 17.72% [337100/1902000] [177.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 08:34:53,927 - Train: 17.73% [337200/1902000] [177.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 08:35:54,372 - Train: 17.73% [337300/1902000] [177.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 08:36:54,759 - Train: 17.74% [337400/1902000] [177.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:37:55,258 - Train: 17.74% [337500/1902000] [177.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:38:55,699 - Train: 17.75% [337600/1902000] [177.5/1000.0] [batch_t 0.600 (0.604)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-05 08:39:56,093 - Train: 17.75% [337700/1902000] [177.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:40:56,525 - Train: 17.76% [337800/1902000] [177.6/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 08:41:56,944 - Train: 17.77% [337900/1902000] [177.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:42:57,350 - Train: 17.77% [338000/1902000] [177.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 08:43:57,808 - Train: 17.78% [338100/1902000] [177.8/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 08:44:58,197 - Train: 17.78% [338200/1902000] [177.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 08:45:58,622 - Train: 17.79% [338300/1902000] [177.9/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 08:46:59,036 - Train: 17.79% [338400/1902000] [177.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:47:59,490 - Train: 17.80% [338500/1902000] [178.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 08:48:33,311 - ==> Total time: 3 days, 7:48:12 Eta: 15 days, 8:31:51 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 08:49:01,672 - Train: 17.80% [338600/1902000] [178.0/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:50:02,056 - Train: 17.81% [338700/1902000] [178.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 08:51:02,446 - Train: 17.81% [338800/1902000] [178.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 08:52:02,799 - Train: 17.82% [338900/1902000] [178.2/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 08:53:03,219 - Train: 17.82% [339000/1902000] [178.2/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 08:54:03,753 - Train: 17.83% [339100/1902000] [178.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:55:04,174 - Train: 17.83% [339200/1902000] [178.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 08:56:04,591 - Train: 17.84% [339300/1902000] [178.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:57:04,973 - Train: 17.84% [339400/1902000] [178.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 08:58:05,382 - Train: 17.85% [339500/1902000] [178.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 08:59:05,794 - Train: 17.85% [339600/1902000] [178.5/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 09:00:06,224 - Train: 17.86% [339700/1902000] [178.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:01:06,635 - Train: 17.87% [339800/1902000] [178.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:02:07,152 - Train: 17.87% [339900/1902000] [178.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:03:07,546 - Train: 17.88% [340000/1902000] [178.8/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 09:04:07,927 - Train: 17.88% [340100/1902000] [178.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:05:08,323 - Train: 17.89% [340200/1902000] [178.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:06:08,754 - Train: 17.89% [340300/1902000] [178.9/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 09:07:09,282 - Train: 17.90% [340400/1902000] [179.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 09:07:44,321 - ==> Total time: 3 days, 8:07:23 Eta: 15 days, 7:29:34 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 09:08:11,417 - Train: 17.90% [340500/1902000] [179.0/1000.0] [batch_t 0.603 (0.611)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 09:09:11,798 - Train: 17.91% [340600/1902000] [179.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:10:12,185 - Train: 17.91% [340700/1902000] [179.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 09:11:12,598 - Train: 17.92% [340800/1902000] [179.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:12:13,523 - Train: 17.92% [340900/1902000] [179.2/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:13:14,096 - Train: 17.93% [341000/1902000] [179.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 09:14:14,510 - Train: 17.93% [341100/1902000] [179.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 09:15:14,943 - Train: 17.94% [341200/1902000] [179.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:16:15,402 - Train: 17.94% [341300/1902000] [179.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:17:15,833 - Train: 17.95% [341400/1902000] [179.5/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 09:18:16,254 - Train: 17.95% [341500/1902000] [179.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:19:16,696 - Train: 17.96% [341600/1902000] [179.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:20:17,121 - Train: 17.97% [341700/1902000] [179.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:21:17,797 - Train: 17.97% [341800/1902000] [179.7/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:22:18,187 - Train: 17.98% [341900/1902000] [179.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:23:18,641 - Train: 17.98% [342000/1902000] [179.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:24:19,059 - Train: 17.99% [342100/1902000] [179.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 09:25:19,470 - Train: 17.99% [342200/1902000] [179.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 09:26:20,043 - Train: 18.00% [342300/1902000] [180.0/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 09:26:56,280 - ==> Total time: 3 days, 8:26:35 Eta: 15 days, 6:27:49 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 09:27:21,986 - Train: 18.00% [342400/1902000] [180.0/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:28:22,359 - Train: 18.01% [342500/1902000] [180.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:29:22,727 - Train: 18.01% [342600/1902000] [180.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:30:23,126 - Train: 18.02% [342700/1902000] [180.2/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 09:31:23,524 - Train: 18.02% [342800/1902000] [180.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 09:32:23,937 - Train: 18.03% [342900/1902000] [180.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:33:24,488 - Train: 18.03% [343000/1902000] [180.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 09:34:24,905 - Train: 18.04% [343100/1902000] [180.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 09:35:25,307 - Train: 18.04% [343200/1902000] [180.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 09:36:25,693 - Train: 18.05% [343300/1902000] [180.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 09:37:26,085 - Train: 18.05% [343400/1902000] [180.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 09:38:26,547 - Train: 18.06% [343500/1902000] [180.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 09:39:26,947 - Train: 18.07% [343600/1902000] [180.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:40:27,324 - Train: 18.07% [343700/1902000] [180.7/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 09:41:27,770 - Train: 18.08% [343800/1902000] [180.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:42:28,189 - Train: 18.08% [343900/1902000] [180.8/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 09:43:28,605 - Train: 18.09% [344000/1902000] [180.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 09:44:28,986 - Train: 18.09% [344100/1902000] [180.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:45:29,378 - Train: 18.10% [344200/1902000] [181.0/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 09:46:06,881 - ==> Total time: 3 days, 8:45:46 Eta: 15 days, 5:26:27 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 09:46:31,392 - Train: 18.10% [344300/1902000] [181.0/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:47:31,810 - Train: 18.11% [344400/1902000] [181.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:48:32,211 - Train: 18.11% [344500/1902000] [181.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 09:49:32,612 - Train: 18.12% [344600/1902000] [181.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:50:33,003 - Train: 18.12% [344700/1902000] [181.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:51:33,552 - Train: 18.13% [344800/1902000] [181.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:52:33,908 - Train: 18.13% [344900/1902000] [181.3/1000.0] [batch_t 0.603 (0.603)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 09:53:34,292 - Train: 18.14% [345000/1902000] [181.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 09:54:35,572 - Train: 18.14% [345100/1902000] [181.4/1000.0] [batch_t 0.605 (0.613)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:55:38,726 - Train: 18.15% [345200/1902000] [181.5/1000.0] [batch_t 0.605 (0.631)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:56:42,259 - Train: 18.15% [345300/1902000] [181.5/1000.0] [batch_t 0.605 (0.635)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 09:57:42,660 - Train: 18.16% [345400/1902000] [181.6/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 09:58:43,053 - Train: 18.17% [345500/1902000] [181.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 10:00:32,703 - Train: 18.17% [345600/1902000] [181.7/1000.0] [batch_t 0.603 (1.096)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 10:01:33,086 - Train: 18.18% [345700/1902000] [181.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 10:02:33,862 - Train: 18.18% [345800/1902000] [181.8/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:03:34,247 - Train: 18.19% [345900/1902000] [181.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 10:04:35,028 - Train: 18.19% [346000/1902000] [181.9/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:05:35,440 - Train: 18.20% [346100/1902000] [182.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 10:06:14,121 - ==> Total time: 3 days, 9:05:53 Eta: 15 days, 4:29:47 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 10:06:37,566 - Train: 18.20% [346200/1902000] [182.0/1000.0] [batch_t 0.603 (0.610)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 10:07:41,161 - Train: 18.21% [346300/1902000] [182.1/1000.0] [batch_t 0.604 (0.636)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 10:08:41,588 - Train: 18.21% [346400/1902000] [182.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:09:41,966 - Train: 18.22% [346500/1902000] [182.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:10:51,954 - Train: 18.22% [346600/1902000] [182.2/1000.0] [batch_t 0.605 (0.700)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:11:53,999 - Train: 18.23% [346700/1902000] [182.3/1000.0] [batch_t 0.605 (0.620)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:12:54,378 - Train: 18.23% [346800/1902000] [182.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:13:54,939 - Train: 18.24% [346900/1902000] [182.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 10:14:55,350 - Train: 18.24% [347000/1902000] [182.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 10:15:55,764 - Train: 18.25% [347100/1902000] [182.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 10:16:56,168 - Train: 18.25% [347200/1902000] [182.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 10:17:56,583 - Train: 18.26% [347300/1902000] [182.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 10:18:56,992 - Train: 18.26% [347400/1902000] [182.6/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 10:19:57,387 - Train: 18.27% [347500/1902000] [182.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 10:20:57,858 - Train: 18.28% [347600/1902000] [182.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:21:58,284 - Train: 18.28% [347700/1902000] [182.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:22:58,666 - Train: 18.29% [347800/1902000] [182.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 10:23:59,049 - Train: 18.29% [347900/1902000] [182.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 10:24:59,431 - Train: 18.30% [348000/1902000] [183.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:25:39,294 - ==> Total time: 3 days, 9:25:18 Eta: 15 days, 3:30:23 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 10:26:01,845 - Train: 18.30% [348100/1902000] [183.0/1000.0] [batch_t 0.604 (0.619)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 10:27:02,234 - Train: 18.31% [348200/1902000] [183.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 10:28:02,598 - Train: 18.31% [348300/1902000] [183.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 10:29:03,014 - Train: 18.32% [348400/1902000] [183.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:30:03,419 - Train: 18.32% [348500/1902000] [183.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:31:03,788 - Train: 18.33% [348600/1902000] [183.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 10:32:04,175 - Train: 18.33% [348700/1902000] [183.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 10:33:04,581 - Train: 18.34% [348800/1902000] [183.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:34:05,064 - Train: 18.34% [348900/1902000] [183.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:35:05,478 - Train: 18.35% [349000/1902000] [183.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 10:36:05,893 - Train: 18.35% [349100/1902000] [183.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 10:37:06,261 - Train: 18.36% [349200/1902000] [183.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:38:06,634 - Train: 18.36% [349300/1902000] [183.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 10:39:07,097 - Train: 18.37% [349400/1902000] [183.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:40:07,527 - Train: 18.38% [349500/1902000] [183.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:41:07,951 - Train: 18.38% [349600/1902000] [183.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:42:08,314 - Train: 18.39% [349700/1902000] [183.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:43:08,713 - Train: 18.39% [349800/1902000] [183.9/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 10:44:12,661 - Train: 18.40% [349900/1902000] [184.0/1000.0] [batch_t 0.603 (0.639)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 10:44:53,714 - ==> Total time: 3 days, 9:44:33 Eta: 15 days, 2:30:37 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 10:45:15,069 - Train: 18.40% [350000/1902000] [184.0/1000.0] [batch_t 0.607 (0.615)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 10:46:15,553 - Train: 18.41% [350100/1902000] [184.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:47:15,948 - Train: 18.41% [350200/1902000] [184.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:48:16,336 - Train: 18.42% [350300/1902000] [184.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:49:16,717 - Train: 18.42% [350400/1902000] [184.2/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 10:50:17,082 - Train: 18.43% [350500/1902000] [184.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 10:51:17,551 - Train: 18.43% [350600/1902000] [184.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:52:17,932 - Train: 18.44% [350700/1902000] [184.4/1000.0] [batch_t 0.610 (0.604)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-05 10:53:18,377 - Train: 18.44% [350800/1902000] [184.4/1000.0] [batch_t 0.610 (0.604)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-05 10:54:18,891 - Train: 18.45% [350900/1902000] [184.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 10:55:19,285 - Train: 18.45% [351000/1902000] [184.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 10:56:19,845 - Train: 18.46% [351100/1902000] [184.6/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 10:57:20,380 - Train: 18.46% [351200/1902000] [184.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 10:58:21,219 - Train: 18.47% [351300/1902000] [184.7/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 10:59:21,644 - Train: 18.48% [351400/1902000] [184.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:00:22,055 - Train: 18.48% [351500/1902000] [184.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-05 11:01:22,446 - Train: 18.49% [351600/1902000] [184.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 11:02:22,863 - Train: 18.49% [351700/1902000] [184.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 11:03:23,345 - Train: 18.50% [351800/1902000] [185.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 11:04:05,666 - ==> Total time: 3 days, 10:03:45 Eta: 15 days, 1:31:07 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 11:04:25,352 - Train: 18.50% [351900/1902000] [185.0/1000.0] [batch_t 0.606 (0.610)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 11:05:25,770 - Train: 18.51% [352000/1902000] [185.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 11:06:26,160 - Train: 18.51% [352100/1902000] [185.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 11:07:26,535 - Train: 18.52% [352200/1902000] [185.2/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 11:08:26,906 - Train: 18.52% [352300/1902000] [185.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 11:09:27,470 - Train: 18.53% [352400/1902000] [185.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:10:27,859 - Train: 18.53% [352500/1902000] [185.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:11:28,276 - Train: 18.54% [352600/1902000] [185.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 11:12:28,687 - Train: 18.54% [352700/1902000] [185.4/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 11:13:29,104 - Train: 18.55% [352800/1902000] [185.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:14:29,497 - Train: 18.55% [352900/1902000] [185.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:15:29,911 - Train: 18.56% [353000/1902000] [185.6/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 11:16:31,181 - Train: 18.56% [353100/1902000] [185.6/1000.0] [batch_t 0.603 (0.613)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 11:17:32,913 - Train: 18.57% [353200/1902000] [185.7/1000.0] [batch_t 0.602 (0.617)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 11:18:33,371 - Train: 18.58% [353300/1902000] [185.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:19:33,790 - Train: 18.58% [353400/1902000] [185.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 11:20:34,202 - Train: 18.59% [353500/1902000] [185.9/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 11:21:34,685 - Train: 18.59% [353600/1902000] [185.9/1000.0] [batch_t 0.694 (0.605)] [data_t 0.002] [optim_t 0.692] [lr 0.005000] +2024-04-05 11:22:35,145 - Train: 18.60% [353700/1902000] [186.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 11:23:18,676 - ==> Total time: 3 days, 10:22:58 Eta: 15 days, 0:32:08 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 11:23:37,520 - Train: 18.60% [353800/1902000] [186.0/1000.0] [batch_t 0.604 (0.611)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 11:24:37,939 - Train: 18.61% [353900/1902000] [186.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 11:25:38,388 - Train: 18.61% [354000/1902000] [186.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 11:26:38,795 - Train: 18.62% [354100/1902000] [186.2/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 11:27:39,252 - Train: 18.62% [354200/1902000] [186.2/1000.0] [batch_t 0.611 (0.604)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-05 11:28:39,830 - Train: 18.63% [354300/1902000] [186.3/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 11:29:40,237 - Train: 18.63% [354400/1902000] [186.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 11:30:40,622 - Train: 18.64% [354500/1902000] [186.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 11:31:41,012 - Train: 18.64% [354600/1902000] [186.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 11:32:41,453 - Train: 18.65% [354700/1902000] [186.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:33:41,847 - Train: 18.65% [354800/1902000] [186.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 11:34:42,343 - Train: 18.66% [354900/1902000] [186.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 11:35:42,759 - Train: 18.66% [355000/1902000] [186.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 11:36:43,161 - Train: 18.67% [355100/1902000] [186.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:37:43,539 - Train: 18.68% [355200/1902000] [186.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 11:38:43,926 - Train: 18.68% [355300/1902000] [186.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:39:44,294 - Train: 18.69% [355400/1902000] [186.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:40:44,686 - Train: 18.69% [355500/1902000] [186.9/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 11:41:45,248 - Train: 18.70% [355600/1902000] [187.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:42:29,965 - ==> Total time: 3 days, 10:42:09 Eta: 14 days, 23:33:26 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 11:42:47,433 - Train: 18.70% [355700/1902000] [187.0/1000.0] [batch_t 0.604 (0.619)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:43:47,836 - Train: 18.71% [355800/1902000] [187.1/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 11:44:48,228 - Train: 18.71% [355900/1902000] [187.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:45:48,639 - Train: 18.72% [356000/1902000] [187.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 11:46:49,205 - Train: 18.72% [356100/1902000] [187.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 11:47:49,612 - Train: 18.73% [356200/1902000] [187.3/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 11:48:50,011 - Train: 18.73% [356300/1902000] [187.3/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 11:49:50,410 - Train: 18.74% [356400/1902000] [187.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:50:50,808 - Train: 18.74% [356500/1902000] [187.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 11:51:51,178 - Train: 18.75% [356600/1902000] [187.5/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 11:52:51,556 - Train: 18.75% [356700/1902000] [187.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 11:53:51,919 - Train: 18.76% [356800/1902000] [187.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:54:52,372 - Train: 18.76% [356900/1902000] [187.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:55:52,745 - Train: 18.77% [357000/1902000] [187.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:56:53,117 - Train: 18.77% [357100/1902000] [187.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 11:57:53,531 - Train: 18.78% [357200/1902000] [187.8/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:58:53,924 - Train: 18.79% [357300/1902000] [187.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 11:59:54,344 - Train: 18.79% [357400/1902000] [187.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:00:54,881 - Train: 18.80% [357500/1902000] [188.0/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 12:01:40,782 - ==> Total time: 3 days, 11:01:20 Eta: 14 days, 22:35:08 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 12:01:57,083 - Train: 18.80% [357600/1902000] [188.0/1000.0] [batch_t 0.604 (0.617)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 12:02:57,474 - Train: 18.81% [357700/1902000] [188.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:03:57,822 - Train: 18.81% [357800/1902000] [188.1/1000.0] [batch_t 0.608 (0.603)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 12:04:58,208 - Train: 18.82% [357900/1902000] [188.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 12:05:58,730 - Train: 18.82% [358000/1902000] [188.2/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 12:06:59,149 - Train: 18.83% [358100/1902000] [188.3/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 12:07:59,786 - Train: 18.83% [358200/1902000] [188.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:09:00,197 - Train: 18.84% [358300/1902000] [188.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:10:00,910 - Train: 18.84% [358400/1902000] [188.4/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:11:01,290 - Train: 18.85% [358500/1902000] [188.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 12:12:01,724 - Train: 18.85% [358600/1902000] [188.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:13:02,119 - Train: 18.86% [358700/1902000] [188.6/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 12:14:02,602 - Train: 18.86% [358800/1902000] [188.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 12:15:03,011 - Train: 18.87% [358900/1902000] [188.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:16:03,405 - Train: 18.87% [359000/1902000] [188.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:17:03,846 - Train: 18.88% [359100/1902000] [188.8/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 12:18:04,666 - Train: 18.89% [359200/1902000] [188.9/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 12:19:05,362 - Train: 18.89% [359300/1902000] [188.9/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:20:05,777 - Train: 18.90% [359400/1902000] [189.0/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 12:20:52,941 - ==> Total time: 3 days, 11:20:32 Eta: 14 days, 21:37:20 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 12:21:07,767 - Train: 18.90% [359500/1902000] [189.0/1000.0] [batch_t 0.605 (0.613)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:22:08,225 - Train: 18.91% [359600/1902000] [189.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:23:08,665 - Train: 18.91% [359700/1902000] [189.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 12:24:09,100 - Train: 18.92% [359800/1902000] [189.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:25:09,680 - Train: 18.92% [359900/1902000] [189.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 12:26:10,128 - Train: 18.93% [360000/1902000] [189.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 12:27:10,574 - Train: 18.93% [360100/1902000] [189.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:28:11,001 - Train: 18.94% [360200/1902000] [189.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:29:11,417 - Train: 18.94% [360300/1902000] [189.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:30:12,011 - Train: 18.95% [360400/1902000] [189.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:31:12,457 - Train: 18.95% [360500/1902000] [189.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:32:12,940 - Train: 18.96% [360600/1902000] [189.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 12:33:13,381 - Train: 18.96% [360700/1902000] [189.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:34:14,288 - Train: 18.97% [360800/1902000] [189.7/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 12:35:14,704 - Train: 18.97% [360900/1902000] [189.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 12:36:15,140 - Train: 18.98% [361000/1902000] [189.8/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 12:37:15,548 - Train: 18.99% [361100/1902000] [189.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 12:38:16,031 - Train: 18.99% [361200/1902000] [189.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:39:16,503 - Train: 19.00% [361300/1902000] [190.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 12:40:04,836 - ==> Total time: 3 days, 11:39:44 Eta: 14 days, 20:39:56 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 12:40:19,829 - Train: 19.00% [361400/1902000] [190.0/1000.0] [batch_t 0.604 (0.614)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 12:41:20,226 - Train: 19.01% [361500/1902000] [190.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 12:42:20,641 - Train: 19.01% [361600/1902000] [190.1/1000.0] [batch_t 0.609 (0.604)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 12:43:22,538 - Train: 19.02% [361700/1902000] [190.2/1000.0] [batch_t 0.606 (0.619)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 12:44:22,968 - Train: 19.02% [361800/1902000] [190.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 12:45:23,416 - Train: 19.03% [361900/1902000] [190.3/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 12:46:23,858 - Train: 19.03% [362000/1902000] [190.3/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:47:24,303 - Train: 19.04% [362100/1902000] [190.4/1000.0] [batch_t 0.602 (0.604)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 12:48:24,740 - Train: 19.04% [362200/1902000] [190.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:49:25,156 - Train: 19.05% [362300/1902000] [190.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:50:25,719 - Train: 19.05% [362400/1902000] [190.5/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 12:51:26,117 - Train: 19.06% [362500/1902000] [190.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 12:52:26,520 - Train: 19.06% [362600/1902000] [190.6/1000.0] [batch_t 0.609 (0.604)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 12:53:26,964 - Train: 19.07% [362700/1902000] [190.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:54:27,390 - Train: 19.07% [362800/1902000] [190.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:55:27,789 - Train: 19.08% [362900/1902000] [190.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 12:56:28,309 - Train: 19.09% [363000/1902000] [190.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 12:57:28,762 - Train: 19.09% [363100/1902000] [190.9/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 12:58:29,227 - Train: 19.10% [363200/1902000] [191.0/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 12:59:18,849 - ==> Total time: 3 days, 11:58:58 Eta: 14 days, 19:43:05 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 12:59:31,583 - Train: 19.10% [363300/1902000] [191.0/1000.0] [batch_t 0.605 (0.616)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:00:32,020 - Train: 19.11% [363400/1902000] [191.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:01:34,203 - Train: 19.11% [363500/1902000] [191.1/1000.0] [batch_t 0.606 (0.622)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 13:02:34,767 - Train: 19.12% [363600/1902000] [191.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 13:03:35,221 - Train: 19.12% [363700/1902000] [191.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 13:04:35,666 - Train: 19.13% [363800/1902000] [191.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:05:36,094 - Train: 19.13% [363900/1902000] [191.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:06:36,813 - Train: 19.14% [364000/1902000] [191.4/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 13:07:37,235 - Train: 19.14% [364100/1902000] [191.4/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 13:08:37,850 - Train: 19.15% [364200/1902000] [191.5/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-05 13:09:38,263 - Train: 19.15% [364300/1902000] [191.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 13:10:38,696 - Train: 19.16% [364400/1902000] [191.6/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 13:11:39,206 - Train: 19.16% [364500/1902000] [191.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 13:12:39,658 - Train: 19.17% [364600/1902000] [191.7/1000.0] [batch_t 0.606 (0.604)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:13:40,116 - Train: 19.17% [364700/1902000] [191.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:14:40,565 - Train: 19.18% [364800/1902000] [191.8/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 13:15:41,148 - Train: 19.19% [364900/1902000] [191.9/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 13:16:41,583 - Train: 19.19% [365000/1902000] [191.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:17:42,015 - Train: 19.20% [365100/1902000] [192.0/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 13:18:32,776 - ==> Total time: 3 days, 12:18:12 Eta: 14 days, 18:46:36 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 13:18:44,207 - Train: 19.20% [365200/1902000] [192.0/1000.0] [batch_t 0.607 (0.623)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 13:19:44,631 - Train: 19.21% [365300/1902000] [192.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:20:45,110 - Train: 19.21% [365400/1902000] [192.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:21:45,671 - Train: 19.22% [365500/1902000] [192.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 13:22:46,082 - Train: 19.22% [365600/1902000] [192.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 13:23:46,512 - Train: 19.23% [365700/1902000] [192.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:24:46,931 - Train: 19.23% [365800/1902000] [192.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:25:47,627 - Train: 19.24% [365900/1902000] [192.4/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:26:48,223 - Train: 19.24% [366000/1902000] [192.4/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 13:27:48,618 - Train: 19.25% [366100/1902000] [192.5/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:28:49,021 - Train: 19.25% [366200/1902000] [192.5/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 13:29:49,404 - Train: 19.26% [366300/1902000] [192.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 13:30:52,419 - Train: 19.26% [366400/1902000] [192.6/1000.0] [batch_t 0.604 (0.630)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 13:32:01,444 - Train: 19.27% [366500/1902000] [192.7/1000.0] [batch_t 0.605 (0.690)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:33:01,845 - Train: 19.27% [366600/1902000] [192.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 13:34:02,218 - Train: 19.28% [366700/1902000] [192.8/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 13:35:02,784 - Train: 19.28% [366800/1902000] [192.8/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 13:36:08,487 - Train: 19.29% [366900/1902000] [192.9/1000.0] [batch_t 0.604 (0.657)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 13:37:08,917 - Train: 19.30% [367000/1902000] [193.0/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 13:38:00,879 - ==> Total time: 3 days, 12:37:40 Eta: 14 days, 17:51:30 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 13:38:14,075 - Train: 19.30% [367100/1902000] [193.0/1000.0] [batch_t 0.606 (0.842)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 13:39:14,456 - Train: 19.31% [367200/1902000] [193.1/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 13:40:17,111 - Train: 19.31% [367300/1902000] [193.1/1000.0] [batch_t 0.605 (0.626)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 13:41:18,819 - Train: 19.32% [367400/1902000] [193.2/1000.0] [batch_t 0.603 (0.617)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 13:42:19,373 - Train: 19.32% [367500/1902000] [193.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 13:43:20,224 - Train: 19.33% [367600/1902000] [193.3/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 13:44:20,771 - Train: 19.33% [367700/1902000] [193.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:45:21,181 - Train: 19.34% [367800/1902000] [193.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:46:21,591 - Train: 19.34% [367900/1902000] [193.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:47:21,976 - Train: 19.35% [368000/1902000] [193.5/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 13:48:22,457 - Train: 19.35% [368100/1902000] [193.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:49:22,828 - Train: 19.36% [368200/1902000] [193.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:50:23,249 - Train: 19.36% [368300/1902000] [193.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:51:23,623 - Train: 19.37% [368400/1902000] [193.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:52:23,974 - Train: 19.37% [368500/1902000] [193.7/1000.0] [batch_t 0.604 (0.603)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 13:53:24,427 - Train: 19.38% [368600/1902000] [193.8/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 13:54:24,819 - Train: 19.38% [368700/1902000] [193.8/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 13:55:25,191 - Train: 19.39% [368800/1902000] [193.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 13:56:25,584 - Train: 19.40% [368900/1902000] [194.0/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:57:18,711 - ==> Total time: 3 days, 12:56:58 Eta: 14 days, 16:56:04 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 13:57:27,770 - Train: 19.40% [369000/1902000] [194.0/1000.0] [batch_t 0.603 (0.628)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 13:58:28,151 - Train: 19.41% [369100/1902000] [194.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 13:59:28,536 - Train: 19.41% [369200/1902000] [194.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 14:00:29,051 - Train: 19.42% [369300/1902000] [194.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 14:01:29,464 - Train: 19.42% [369400/1902000] [194.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 14:02:29,935 - Train: 19.43% [369500/1902000] [194.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 14:03:31,135 - Train: 19.43% [369600/1902000] [194.3/1000.0] [batch_t 0.601 (0.612)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 14:04:31,548 - Train: 19.44% [369700/1902000] [194.4/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 14:05:32,017 - Train: 19.44% [369800/1902000] [194.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 14:06:32,404 - Train: 19.45% [369900/1902000] [194.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 14:07:32,778 - Train: 19.45% [370000/1902000] [194.5/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 14:08:33,152 - Train: 19.46% [370100/1902000] [194.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 14:09:33,524 - Train: 19.46% [370200/1902000] [194.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 14:10:33,900 - Train: 19.47% [370300/1902000] [194.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 14:11:34,529 - Train: 19.47% [370400/1902000] [194.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 14:12:35,128 - Train: 19.48% [370500/1902000] [194.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 14:13:35,576 - Train: 19.48% [370600/1902000] [194.8/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 14:14:35,974 - Train: 19.49% [370700/1902000] [194.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 14:15:37,217 - Train: 19.50% [370800/1902000] [195.0/1000.0] [batch_t 0.605 (0.612)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 14:16:31,746 - ==> Total time: 3 days, 13:16:11 Eta: 14 days, 16:00:40 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 14:16:39,601 - Train: 19.50% [370900/1902000] [195.0/1000.0] [batch_t 0.605 (0.630)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 14:17:39,996 - Train: 19.51% [371000/1902000] [195.1/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 14:18:40,468 - Train: 19.51% [371100/1902000] [195.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-05 14:19:41,162 - Train: 19.52% [371200/1902000] [195.2/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 14:20:41,682 - Train: 19.52% [371300/1902000] [195.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:21:42,200 - Train: 19.53% [371400/1902000] [195.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:22:42,712 - Train: 19.53% [371500/1902000] [195.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.003] [optim_t 0.601] [lr 0.005000] +2024-04-05 14:23:43,266 - Train: 19.54% [371600/1902000] [195.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-05 14:24:43,812 - Train: 19.54% [371700/1902000] [195.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-05 14:25:44,581 - Train: 19.55% [371800/1902000] [195.5/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 14:26:45,184 - Train: 19.55% [371900/1902000] [195.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-05 14:27:45,731 - Train: 19.56% [372000/1902000] [195.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:28:46,285 - Train: 19.56% [372100/1902000] [195.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:29:46,822 - Train: 19.57% [372200/1902000] [195.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:30:47,312 - Train: 19.57% [372300/1902000] [195.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-05 14:31:47,856 - Train: 19.58% [372400/1902000] [195.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 14:32:48,520 - Train: 19.58% [372500/1902000] [195.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-05 14:33:49,066 - Train: 19.59% [372600/1902000] [195.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:34:49,649 - Train: 19.60% [372700/1902000] [196.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-05 14:35:45,327 - ==> Total time: 3 days, 13:35:24 Eta: 14 days, 15:05:40 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 14:35:51,902 - Train: 19.60% [372800/1902000] [196.0/1000.0] [batch_t 0.606 (0.648)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-05 14:36:52,425 - Train: 19.61% [372900/1902000] [196.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-05 14:37:52,981 - Train: 19.61% [373000/1902000] [196.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:38:53,665 - Train: 19.62% [373100/1902000] [196.2/1000.0] [batch_t 0.607 (0.607)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:39:54,208 - Train: 19.62% [373200/1902000] [196.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-05 14:40:54,708 - Train: 19.63% [373300/1902000] [196.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:41:55,235 - Train: 19.63% [373400/1902000] [196.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 14:42:55,769 - Train: 19.64% [373500/1902000] [196.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-05 14:43:56,478 - Train: 19.64% [373600/1902000] [196.4/1000.0] [batch_t 0.607 (0.607)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-05 14:44:57,058 - Train: 19.65% [373700/1902000] [196.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:45:57,593 - Train: 19.65% [373800/1902000] [196.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-05 14:46:58,128 - Train: 19.66% [373900/1902000] [196.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:47:58,744 - Train: 19.66% [374000/1902000] [196.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-05 14:48:59,272 - Train: 19.67% [374100/1902000] [196.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:49:59,847 - Train: 19.67% [374200/1902000] [196.7/1000.0] [batch_t 0.603 (0.606)] [data_t 0.003] [optim_t 0.601] [lr 0.005000] +2024-04-05 14:51:00,356 - Train: 19.68% [374300/1902000] [196.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 14:52:00,945 - Train: 19.68% [374400/1902000] [196.8/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 14:53:01,410 - Train: 19.69% [374500/1902000] [196.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:54:01,882 - Train: 19.70% [374600/1902000] [197.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 14:54:58,728 - ==> Total time: 3 days, 13:54:38 Eta: 14 days, 14:11:02 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 14:55:03,977 - Train: 19.70% [374700/1902000] [197.0/1000.0] [batch_t 0.606 (0.635)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 14:56:04,420 - Train: 19.71% [374800/1902000] [197.1/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 14:57:05,054 - Train: 19.71% [374900/1902000] [197.1/1000.0] [batch_t 0.609 (0.606)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-05 14:58:05,506 - Train: 19.72% [375000/1902000] [197.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 14:59:05,926 - Train: 19.72% [375100/1902000] [197.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:00:06,350 - Train: 19.73% [375200/1902000] [197.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:01:06,808 - Train: 19.73% [375300/1902000] [197.3/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 15:02:07,377 - Train: 19.74% [375400/1902000] [197.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 15:03:07,818 - Train: 19.74% [375500/1902000] [197.4/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 15:04:08,233 - Train: 19.75% [375600/1902000] [197.5/1000.0] [batch_t 0.606 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:05:08,732 - Train: 19.75% [375700/1902000] [197.5/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-05 15:06:09,170 - Train: 19.76% [375800/1902000] [197.6/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 15:07:09,606 - Train: 19.76% [375900/1902000] [197.6/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:08:10,042 - Train: 19.77% [376000/1902000] [197.7/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 15:09:10,670 - Train: 19.77% [376100/1902000] [197.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:10:11,117 - Train: 19.78% [376200/1902000] [197.8/1000.0] [batch_t 0.608 (0.604)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 15:11:11,589 - Train: 19.78% [376300/1902000] [197.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:12:12,060 - Train: 19.79% [376400/1902000] [197.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-05 15:13:12,521 - Train: 19.79% [376500/1902000] [197.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 15:14:10,530 - ==> Total time: 3 days, 14:13:50 Eta: 14 days, 13:16:38 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 15:14:14,603 - Train: 19.80% [376600/1902000] [198.0/1000.0] [batch_t 0.606 (0.660)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 15:15:15,032 - Train: 19.81% [376700/1902000] [198.1/1000.0] [batch_t 0.603 (0.604)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 15:16:15,642 - Train: 19.81% [376800/1902000] [198.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 15:17:16,093 - Train: 19.82% [376900/1902000] [198.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 15:18:16,525 - Train: 19.82% [377000/1902000] [198.2/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:19:17,443 - Train: 19.83% [377100/1902000] [198.3/1000.0] [batch_t 0.609 (0.609)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 15:20:17,922 - Train: 19.83% [377200/1902000] [198.3/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 15:21:18,469 - Train: 19.84% [377300/1902000] [198.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 15:22:18,894 - Train: 19.84% [377400/1902000] [198.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:23:19,361 - Train: 19.85% [377500/1902000] [198.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 15:24:19,844 - Train: 19.85% [377600/1902000] [198.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 15:25:20,394 - Train: 19.86% [377700/1902000] [198.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:26:20,888 - Train: 19.86% [377800/1902000] [198.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 15:27:21,345 - Train: 19.87% [377900/1902000] [198.7/1000.0] [batch_t 0.607 (0.604)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 15:28:21,864 - Train: 19.87% [378000/1902000] [198.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 15:29:22,488 - Train: 19.88% [378100/1902000] [198.8/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 15:30:23,002 - Train: 19.88% [378200/1902000] [198.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.003] [optim_t 0.601] [lr 0.005000] +2024-04-05 15:31:23,487 - Train: 19.89% [378300/1902000] [198.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:32:23,899 - Train: 19.89% [378400/1902000] [198.9/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 15:33:23,146 - ==> Total time: 3 days, 14:33:02 Eta: 14 days, 12:22:39 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 15:33:26,353 - Train: 19.90% [378500/1902000] [199.0/1000.0] [batch_t 0.605 (0.768)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:34:26,919 - Train: 19.91% [378600/1902000] [199.1/1000.0] [batch_t 0.601 (0.606)] [data_t 0.002] [optim_t 0.598] [lr 0.005000] +2024-04-05 15:35:27,378 - Train: 19.91% [378700/1902000] [199.1/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:36:27,866 - Train: 19.92% [378800/1902000] [199.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:37:28,323 - Train: 19.92% [378900/1902000] [199.2/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 15:38:28,757 - Train: 19.93% [379000/1902000] [199.3/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:39:29,289 - Train: 19.93% [379100/1902000] [199.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 15:40:29,736 - Train: 19.94% [379200/1902000] [199.4/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 15:41:30,176 - Train: 19.94% [379300/1902000] [199.4/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:42:30,924 - Train: 19.95% [379400/1902000] [199.5/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:43:31,537 - Train: 19.95% [379500/1902000] [199.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:44:31,975 - Train: 19.96% [379600/1902000] [199.6/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 15:45:32,709 - Train: 19.96% [379700/1902000] [199.6/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:46:33,134 - Train: 19.97% [379800/1902000] [199.7/1000.0] [batch_t 0.605 (0.604)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:47:33,626 - Train: 19.97% [379900/1902000] [199.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 15:48:34,103 - Train: 19.98% [380000/1902000] [199.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 15:49:34,547 - Train: 19.98% [380100/1902000] [199.8/1000.0] [batch_t 0.610 (0.604)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 15:50:35,008 - Train: 19.99% [380200/1902000] [199.9/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 15:51:35,433 - Train: 19.99% [380300/1902000] [199.9/1000.0] [batch_t 0.604 (0.604)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 15:52:35,932 - Train: 20.00% [380400/1902000] [200.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 15:52:47,518 - Test: 16.13% [50/310] [batch_t 0.207 (0.224)] +2024-04-05 15:52:58,143 - Test: 32.26% [100/310] [batch_t 0.213 (0.218)] +2024-04-05 15:53:11,513 - Test: 48.39% [150/310] [batch_t 0.213 (0.234)] +2024-04-05 15:53:22,112 - Test: 64.52% [200/310] [batch_t 0.212 (0.229)] +2024-04-05 15:53:32,732 - Test: 80.65% [250/310] [batch_t 0.217 (0.226)] +2024-04-05 15:53:43,531 - Test: 96.77% [300/310] [batch_t 0.213 (0.224)] +2024-04-05 15:53:46,161 - Test: 100.00% [310/310] [batch_t 0.730 (0.225)] +2024-04-05 16:24:29,617 - ==> Metric Time for coco : 0.003 (mAUROC_sp_max) 0.001 (mAP_sp_max) 0.001 (mF1_max_sp_max) 380.849 (mAUROC_px) 278.376 (mAP_px) 34.264 (mF1_max_px) 1074.919 (mAUPRO_px) 11.407 (mF1_px_0.2_0.8_0.1) 11.095 (mAcc_px_0.2_0.8_0.1) 11.091 (mIoU_px_0.2_0.8_0.1) 32.188 (mIoU_max_px) +2024-04-05 16:24:30,063 - +| Name | mAUROC_sp_max | mAUROC_sp_max (Max) | mAP_sp_max | mAP_sp_max (Max) | mF1_max_sp_max | mF1_max_sp_max (Max) | mAUROC_px | mAUROC_px (Max) | mAP_px | mAP_px (Max) | mF1_max_px | mF1_max_px (Max) | mAUPRO_px | mAUPRO_px (Max) | mF1_px_0.2_0.8_0.1 | mF1_px_0.2_0.8_0.1 (Max) | mAcc_px_0.2_0.8_0.1 | mAcc_px_0.2_0.8_0.1 (Max) | mIoU_px_0.2_0.8_0.1 | mIoU_px_0.2_0.8_0.1 (Max) | mIoU_max_px | mIoU_max_px (Max) | +|:------:|:---------------:|:---------------------:|:------------:|:------------------:|:----------------:|:----------------------:|:-----------:|:------------------:|:--------:|:------------------:|:------------:|:------------------:|:-----------:|:------------------:|:--------------------:|:--------------------------:|:---------------------:|:---------------------------:|:---------------------:|:---------------------------:|:-------------:|:-------------------:| +| coco | 75.271 | 75.271 (200 epoch) | 89.783 | 89.783 (200 epoch) | 85.222 | 85.222 (200 epoch) | 75.629 | 75.629 (200 epoch) | 38.587 | 38.587 (200 epoch) | 41.670 | 41.670 (200 epoch) | 46.245 | 46.245 (200 epoch) | 22.444 | 22.444 (200 epoch) | 40.626 | 40.626 (200 epoch) | 13.514 | 13.514 (200 epoch) | 26.319 | 26.319 (200 epoch) | +2024-04-05 16:24:30,450 - ==> Total time: 3 days, 15:24:09 Eta: 14 days, 13:36:39 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 16:25:36,388 - Train: 20.01% [380500/1902000] [200.1/1000.0] [batch_t 0.606 (0.636)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:26:36,999 - Train: 20.01% [380600/1902000] [200.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 16:27:37,635 - Train: 20.02% [380700/1902000] [200.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:28:38,258 - Train: 20.02% [380800/1902000] [200.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 16:29:38,859 - Train: 20.03% [380900/1902000] [200.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 16:30:39,499 - Train: 20.03% [381000/1902000] [200.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-05 16:31:40,241 - Train: 20.04% [381100/1902000] [200.4/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:32:40,827 - Train: 20.04% [381200/1902000] [200.4/1000.0] [batch_t 0.608 (0.606)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-05 16:33:41,416 - Train: 20.05% [381300/1902000] [200.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:34:42,141 - Train: 20.05% [381400/1902000] [200.5/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 16:35:42,745 - Train: 20.06% [381500/1902000] [200.6/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 16:36:43,439 - Train: 20.06% [381600/1902000] [200.6/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 16:37:44,132 - Train: 20.07% [381700/1902000] [200.7/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 16:38:44,729 - Train: 20.07% [381800/1902000] [200.7/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 16:39:45,341 - Train: 20.08% [381900/1902000] [200.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 16:40:45,958 - Train: 20.08% [382000/1902000] [200.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 16:41:46,628 - Train: 20.09% [382100/1902000] [200.9/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 16:42:47,276 - Train: 20.09% [382200/1902000] [200.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:43:47,924 - Train: 20.10% [382300/1902000] [201.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 16:43:49,143 - ==> Total time: 3 days, 15:43:28 Eta: 14 days, 12:42:58 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 16:44:50,878 - Train: 20.11% [382400/1902000] [201.1/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 16:45:51,515 - Train: 20.11% [382500/1902000] [201.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:46:52,143 - Train: 20.12% [382600/1902000] [201.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:47:52,767 - Train: 20.12% [382700/1902000] [201.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 16:48:53,370 - Train: 20.13% [382800/1902000] [201.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:49:53,996 - Train: 20.13% [382900/1902000] [201.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 16:50:54,619 - Train: 20.14% [383000/1902000] [201.4/1000.0] [batch_t 0.615 (0.606)] [data_t 0.003] [optim_t 0.612] [lr 0.005000] +2024-04-05 16:51:55,318 - Train: 20.14% [383100/1902000] [201.4/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 16:52:55,901 - Train: 20.15% [383200/1902000] [201.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:53:56,524 - Train: 20.15% [383300/1902000] [201.5/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 16:54:57,125 - Train: 20.16% [383400/1902000] [201.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 16:55:57,756 - Train: 20.16% [383500/1902000] [201.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 16:56:58,373 - Train: 20.17% [383600/1902000] [201.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 16:57:59,010 - Train: 20.17% [383700/1902000] [201.7/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 16:58:59,684 - Train: 20.18% [383800/1902000] [201.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:00:00,307 - Train: 20.18% [383900/1902000] [201.8/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 17:01:00,951 - Train: 20.19% [384000/1902000] [201.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:02:01,557 - Train: 20.19% [384100/1902000] [201.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:03:02,153 - Train: 20.20% [384200/1902000] [202.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 17:03:04,583 - ==> Total time: 3 days, 16:02:44 Eta: 14 days, 11:49:25 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 17:04:06,341 - Train: 20.21% [384300/1902000] [202.1/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:05:06,977 - Train: 20.21% [384400/1902000] [202.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 17:06:07,788 - Train: 20.22% [384500/1902000] [202.2/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 17:07:08,381 - Train: 20.22% [384600/1902000] [202.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:08:08,998 - Train: 20.23% [384700/1902000] [202.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:09:09,676 - Train: 20.23% [384800/1902000] [202.3/1000.0] [batch_t 0.612 (0.607)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-05 17:10:10,319 - Train: 20.24% [384900/1902000] [202.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:11:11,007 - Train: 20.24% [385000/1902000] [202.4/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:12:11,706 - Train: 20.25% [385100/1902000] [202.5/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 17:13:12,360 - Train: 20.25% [385200/1902000] [202.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:14:12,993 - Train: 20.26% [385300/1902000] [202.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:15:13,629 - Train: 20.26% [385400/1902000] [202.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 17:16:14,593 - Train: 20.27% [385500/1902000] [202.7/1000.0] [batch_t 0.607 (0.610)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:17:15,235 - Train: 20.27% [385600/1902000] [202.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:18:16,324 - Train: 20.28% [385700/1902000] [202.8/1000.0] [batch_t 0.607 (0.611)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 17:19:17,073 - Train: 20.28% [385800/1902000] [202.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:20:17,686 - Train: 20.29% [385900/1902000] [202.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:21:18,314 - Train: 20.29% [386000/1902000] [202.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:22:18,945 - Train: 20.30% [386100/1902000] [203.0/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 17:22:22,589 - ==> Total time: 3 days, 16:22:02 Eta: 14 days, 10:56:21 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 17:23:21,633 - Train: 20.30% [386200/1902000] [203.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:24:22,211 - Train: 20.31% [386300/1902000] [203.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 17:25:22,956 - Train: 20.32% [386400/1902000] [203.2/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:26:23,570 - Train: 20.32% [386500/1902000] [203.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:27:24,180 - Train: 20.33% [386600/1902000] [203.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:28:24,778 - Train: 20.33% [386700/1902000] [203.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:29:25,392 - Train: 20.34% [386800/1902000] [203.4/1000.0] [batch_t 0.611 (0.606)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-05 17:30:26,124 - Train: 20.34% [386900/1902000] [203.4/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 17:31:26,743 - Train: 20.35% [387000/1902000] [203.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:32:27,352 - Train: 20.35% [387100/1902000] [203.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:33:28,099 - Train: 20.36% [387200/1902000] [203.6/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:34:28,708 - Train: 20.36% [387300/1902000] [203.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:35:29,351 - Train: 20.37% [387400/1902000] [203.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 17:36:30,008 - Train: 20.37% [387500/1902000] [203.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-05 17:37:30,645 - Train: 20.38% [387600/1902000] [203.8/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 17:38:31,489 - Train: 20.38% [387700/1902000] [203.8/1000.0] [batch_t 0.612 (0.608)] [data_t 0.002] [optim_t 0.610] [lr 0.005000] +2024-04-05 17:39:32,185 - Train: 20.39% [387800/1902000] [203.9/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:40:32,862 - Train: 20.39% [387900/1902000] [203.9/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 17:41:33,515 - Train: 20.40% [388000/1902000] [204.0/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 17:41:38,382 - ==> Total time: 3 days, 16:41:17 Eta: 14 days, 10:03:29 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 17:42:36,948 - Train: 20.40% [388100/1902000] [204.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:43:37,658 - Train: 20.41% [388200/1902000] [204.1/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 17:44:38,284 - Train: 20.42% [388300/1902000] [204.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 17:45:38,908 - Train: 20.42% [388400/1902000] [204.2/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 17:46:39,523 - Train: 20.43% [388500/1902000] [204.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:47:40,175 - Train: 20.43% [388600/1902000] [204.3/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 17:48:40,858 - Train: 20.44% [388700/1902000] [204.4/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 17:49:41,455 - Train: 20.44% [388800/1902000] [204.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:50:42,095 - Train: 20.45% [388900/1902000] [204.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:51:42,675 - Train: 20.45% [389000/1902000] [204.5/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 17:52:43,241 - Train: 20.46% [389100/1902000] [204.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:53:43,840 - Train: 20.46% [389200/1902000] [204.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:54:44,452 - Train: 20.47% [389300/1902000] [204.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 17:55:45,087 - Train: 20.47% [389400/1902000] [204.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:56:45,787 - Train: 20.48% [389500/1902000] [204.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:57:47,242 - Train: 20.48% [389600/1902000] [204.8/1000.0] [batch_t 0.606 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 17:58:47,875 - Train: 20.49% [389700/1902000] [204.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 17:59:48,507 - Train: 20.49% [389800/1902000] [204.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-05 18:00:49,144 - Train: 20.50% [389900/1902000] [205.0/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 18:00:55,204 - ==> Total time: 3 days, 17:00:34 Eta: 14 days, 9:11:01 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 18:01:51,808 - Train: 20.50% [390000/1902000] [205.0/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:02:52,414 - Train: 20.51% [390100/1902000] [205.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:03:53,025 - Train: 20.52% [390200/1902000] [205.2/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-05 18:04:53,649 - Train: 20.52% [390300/1902000] [205.2/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 18:05:54,238 - Train: 20.53% [390400/1902000] [205.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 18:06:54,946 - Train: 20.53% [390500/1902000] [205.3/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 18:07:55,544 - Train: 20.54% [390600/1902000] [205.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:08:56,144 - Train: 20.54% [390700/1902000] [205.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 18:09:56,754 - Train: 20.55% [390800/1902000] [205.5/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 18:10:57,583 - Train: 20.55% [390900/1902000] [205.5/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:11:58,217 - Train: 20.56% [391000/1902000] [205.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 18:12:58,807 - Train: 20.56% [391100/1902000] [205.6/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 18:13:59,447 - Train: 20.57% [391200/1902000] [205.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:15:00,137 - Train: 20.57% [391300/1902000] [205.7/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 18:16:00,740 - Train: 20.58% [391400/1902000] [205.8/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 18:17:01,360 - Train: 20.58% [391500/1902000] [205.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:18:02,017 - Train: 20.59% [391600/1902000] [205.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:19:02,635 - Train: 20.59% [391700/1902000] [205.9/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 18:20:03,325 - Train: 20.60% [391800/1902000] [206.0/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 18:20:10,602 - ==> Total time: 3 days, 17:19:50 Eta: 14 days, 8:18:46 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 18:21:06,693 - Train: 20.60% [391900/1902000] [206.0/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:22:15,730 - Train: 20.61% [392000/1902000] [206.1/1000.0] [batch_t 0.606 (0.690)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:23:25,235 - Train: 20.62% [392100/1902000] [206.2/1000.0] [batch_t 0.605 (0.695)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 18:24:33,954 - Train: 20.62% [392200/1902000] [206.2/1000.0] [batch_t 0.605 (0.687)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 18:25:49,414 - Train: 20.63% [392300/1902000] [206.3/1000.0] [batch_t 0.607 (0.755)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 18:27:11,027 - Train: 20.63% [392400/1902000] [206.3/1000.0] [batch_t 0.606 (0.816)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:28:18,059 - Train: 20.64% [392500/1902000] [206.4/1000.0] [batch_t 0.979 (0.670)] [data_t 0.380] [optim_t 0.599] [lr 0.005000] +2024-04-05 18:29:29,171 - Train: 20.64% [392600/1902000] [206.4/1000.0] [batch_t 0.606 (0.711)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:30:29,781 - Train: 20.65% [392700/1902000] [206.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 18:31:30,444 - Train: 20.65% [392800/1902000] [206.5/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:32:31,104 - Train: 20.66% [392900/1902000] [206.6/1000.0] [batch_t 0.613 (0.607)] [data_t 0.002] [optim_t 0.610] [lr 0.005000] +2024-04-05 18:33:31,720 - Train: 20.66% [393000/1902000] [206.6/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 18:34:32,459 - Train: 20.67% [393100/1902000] [206.7/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 18:35:33,070 - Train: 20.67% [393200/1902000] [206.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 18:36:33,691 - Train: 20.68% [393300/1902000] [206.8/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 18:37:34,312 - Train: 20.68% [393400/1902000] [206.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 18:38:34,910 - Train: 20.69% [393500/1902000] [206.9/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 18:39:35,642 - Train: 20.69% [393600/1902000] [206.9/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 18:40:36,245 - Train: 20.70% [393700/1902000] [207.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 18:40:44,734 - ==> Total time: 3 days, 17:40:24 Eta: 14 days, 7:31:53 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 18:41:39,037 - Train: 20.70% [393800/1902000] [207.0/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 18:42:39,639 - Train: 20.71% [393900/1902000] [207.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:43:41,097 - Train: 20.72% [394000/1902000] [207.2/1000.0] [batch_t 0.607 (0.614)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 18:44:42,634 - Train: 20.72% [394100/1902000] [207.2/1000.0] [batch_t 0.607 (0.615)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 18:45:43,595 - Train: 20.73% [394200/1902000] [207.3/1000.0] [batch_t 0.604 (0.610)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 18:46:44,359 - Train: 20.73% [394300/1902000] [207.3/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 18:47:44,993 - Train: 20.74% [394400/1902000] [207.4/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 18:48:45,622 - Train: 20.74% [394500/1902000] [207.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 18:49:46,320 - Train: 20.75% [394600/1902000] [207.5/1000.0] [batch_t 0.606 (0.607)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:50:46,924 - Train: 20.75% [394700/1902000] [207.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 18:51:47,526 - Train: 20.76% [394800/1902000] [207.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 18:52:48,099 - Train: 20.76% [394900/1902000] [207.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-05 18:53:48,803 - Train: 20.77% [395000/1902000] [207.7/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 18:54:49,412 - Train: 20.77% [395100/1902000] [207.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 18:55:52,249 - Train: 20.78% [395200/1902000] [207.8/1000.0] [batch_t 0.606 (0.628)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 18:56:52,878 - Train: 20.78% [395300/1902000] [207.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:57:53,472 - Train: 20.79% [395400/1902000] [207.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-05 18:58:54,063 - Train: 20.79% [395500/1902000] [207.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 18:59:54,768 - Train: 20.80% [395600/1902000] [208.0/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 19:00:04,464 - ==> Total time: 3 days, 17:59:43 Eta: 14 days, 6:40:31 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 19:00:59,332 - Train: 20.80% [395700/1902000] [208.0/1000.0] [batch_t 0.606 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 19:01:59,946 - Train: 20.81% [395800/1902000] [208.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 19:03:01,895 - Train: 20.81% [395900/1902000] [208.1/1000.0] [batch_t 0.608 (0.619)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 19:04:21,421 - Train: 20.82% [396000/1902000] [208.2/1000.0] [batch_t 1.016 (0.795)] [data_t 0.411] [optim_t 0.606] [lr 0.005000] +2024-04-05 19:05:27,904 - Train: 20.83% [396100/1902000] [208.3/1000.0] [batch_t 0.675 (0.665)] [data_t 0.072] [optim_t 0.603] [lr 0.005000] +2024-04-05 19:06:37,629 - Train: 20.83% [396200/1902000] [208.3/1000.0] [batch_t 0.612 (0.697)] [data_t 0.002] [optim_t 0.610] [lr 0.005000] +2024-04-05 19:07:45,701 - Train: 20.84% [396300/1902000] [208.4/1000.0] [batch_t 0.605 (0.681)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 19:08:46,477 - Train: 20.84% [396400/1902000] [208.4/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 19:09:51,299 - Train: 20.85% [396500/1902000] [208.5/1000.0] [batch_t 0.611 (0.648)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-05 19:11:16,153 - Train: 20.85% [396600/1902000] [208.5/1000.0] [batch_t 5.175 (0.848)] [data_t 4.575] [optim_t 0.600] [lr 0.005000] +2024-04-05 19:13:07,279 - Train: 20.86% [396700/1902000] [208.6/1000.0] [batch_t 4.130 (1.111)] [data_t 3.527] [optim_t 0.603] [lr 0.005000] +2024-04-05 19:14:41,188 - Train: 20.86% [396800/1902000] [208.6/1000.0] [batch_t 0.607 (0.939)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-05 19:16:05,191 - Train: 20.87% [396900/1902000] [208.7/1000.0] [batch_t 0.607 (0.840)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 19:17:05,819 - Train: 20.87% [397000/1902000] [208.7/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 19:18:06,458 - Train: 20.88% [397100/1902000] [208.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 19:19:07,263 - Train: 20.88% [397200/1902000] [208.8/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 19:20:10,327 - Train: 20.89% [397300/1902000] [208.9/1000.0] [batch_t 0.607 (0.631)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 19:21:10,952 - Train: 20.89% [397400/1902000] [208.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 19:22:12,392 - Train: 20.90% [397500/1902000] [209.0/1000.0] [batch_t 0.604 (0.614)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 19:22:23,305 - ==> Total time: 3 days, 18:22:02 Eta: 14 days, 6:00:45 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 19:23:19,645 - Train: 20.90% [397600/1902000] [209.0/1000.0] [batch_t 0.605 (0.655)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 19:24:20,233 - Train: 20.91% [397700/1902000] [209.1/1000.0] [batch_t 0.612 (0.606)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-05 19:25:41,140 - Train: 20.91% [397800/1902000] [209.1/1000.0] [batch_t 2.852 (0.809)] [data_t 2.248] [optim_t 0.604] [lr 0.005000] +2024-04-05 19:27:13,897 - Train: 20.92% [397900/1902000] [209.2/1000.0] [batch_t 1.152 (0.927)] [data_t 0.548] [optim_t 0.604] [lr 0.005000] +2024-04-05 19:28:58,012 - Train: 20.93% [398000/1902000] [209.3/1000.0] [batch_t 5.953 (1.041)] [data_t 5.347] [optim_t 0.606] [lr 0.005000] +2024-04-05 19:31:21,747 - Train: 20.93% [398100/1902000] [209.3/1000.0] [batch_t 4.094 (1.437)] [data_t 3.487] [optim_t 0.606] [lr 0.005000] +2024-04-05 19:33:16,026 - Train: 20.94% [398200/1902000] [209.4/1000.0] [batch_t 0.605 (1.143)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 19:35:21,185 - Train: 20.94% [398300/1902000] [209.4/1000.0] [batch_t 1.703 (1.252)] [data_t 1.099] [optim_t 0.604] [lr 0.005000] +2024-04-05 19:37:33,155 - Train: 20.95% [398400/1902000] [209.5/1000.0] [batch_t 1.018 (1.320)] [data_t 0.414] [optim_t 0.603] [lr 0.005000] +2024-04-05 19:39:07,564 - Train: 20.95% [398500/1902000] [209.5/1000.0] [batch_t 0.605 (0.944)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 19:40:50,772 - Train: 20.96% [398600/1902000] [209.6/1000.0] [batch_t 0.608 (1.032)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 19:41:51,433 - Train: 20.96% [398700/1902000] [209.6/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 19:42:52,280 - Train: 20.97% [398800/1902000] [209.7/1000.0] [batch_t 0.610 (0.608)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 19:43:52,933 - Train: 20.97% [398900/1902000] [209.7/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 19:44:53,606 - Train: 20.98% [399000/1902000] [209.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 19:45:54,257 - Train: 20.98% [399100/1902000] [209.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 19:46:54,951 - Train: 20.99% [399200/1902000] [209.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 19:47:55,645 - Train: 20.99% [399300/1902000] [209.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 19:48:56,446 - Train: 21.00% [399400/1902000] [210.0/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 19:49:08,578 - ==> Total time: 3 days, 18:48:48 Eta: 14 days, 5:37:52 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 19:49:59,400 - Train: 21.00% [399500/1902000] [210.0/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 19:51:00,047 - Train: 21.01% [399600/1902000] [210.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 19:52:00,680 - Train: 21.01% [399700/1902000] [210.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 19:53:01,351 - Train: 21.02% [399800/1902000] [210.2/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 19:54:02,004 - Train: 21.03% [399900/1902000] [210.3/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 19:55:02,603 - Train: 21.03% [400000/1902000] [210.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 19:56:03,387 - Train: 21.04% [400100/1902000] [210.4/1000.0] [batch_t 0.608 (0.608)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-05 19:57:04,009 - Train: 21.04% [400200/1902000] [210.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 19:58:04,614 - Train: 21.05% [400300/1902000] [210.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 19:59:05,313 - Train: 21.05% [400400/1902000] [210.5/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 20:00:05,968 - Train: 21.06% [400500/1902000] [210.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 20:01:06,716 - Train: 21.06% [400600/1902000] [210.6/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 20:02:07,364 - Train: 21.07% [400700/1902000] [210.7/1000.0] [batch_t 0.608 (0.606)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-05 20:03:08,008 - Train: 21.07% [400800/1902000] [210.7/1000.0] [batch_t 0.611 (0.606)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-05 20:04:09,505 - Train: 21.08% [400900/1902000] [210.8/1000.0] [batch_t 0.605 (0.615)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 20:05:10,276 - Train: 21.08% [401000/1902000] [210.8/1000.0] [batch_t 0.614 (0.608)] [data_t 0.003] [optim_t 0.611] [lr 0.005000] +2024-04-05 20:06:13,305 - Train: 21.09% [401100/1902000] [210.9/1000.0] [batch_t 0.605 (0.630)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 20:07:13,933 - Train: 21.09% [401200/1902000] [210.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 20:08:14,734 - Train: 21.10% [401300/1902000] [211.0/1000.0] [batch_t 0.606 (0.608)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-05 20:08:28,067 - ==> Total time: 3 days, 19:08:07 Eta: 14 days, 4:47:09 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 20:09:19,174 - Train: 21.10% [401400/1902000] [211.0/1000.0] [batch_t 0.605 (0.613)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 20:10:19,792 - Train: 21.11% [401500/1902000] [211.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 20:11:20,415 - Train: 21.11% [401600/1902000] [211.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 20:12:21,027 - Train: 21.12% [401700/1902000] [211.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 20:13:21,803 - Train: 21.13% [401800/1902000] [211.3/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 20:14:22,456 - Train: 21.13% [401900/1902000] [211.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 20:15:23,085 - Train: 21.14% [402000/1902000] [211.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-05 20:16:23,703 - Train: 21.14% [402100/1902000] [211.4/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 20:17:24,365 - Train: 21.15% [402200/1902000] [211.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 20:18:25,139 - Train: 21.15% [402300/1902000] [211.5/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 20:19:26,492 - Train: 21.16% [402400/1902000] [211.6/1000.0] [batch_t 0.603 (0.613)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 20:20:28,033 - Train: 21.16% [402500/1902000] [211.6/1000.0] [batch_t 0.607 (0.615)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 20:21:28,656 - Train: 21.17% [402600/1902000] [211.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 20:22:29,303 - Train: 21.17% [402700/1902000] [211.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 20:23:29,933 - Train: 21.18% [402800/1902000] [211.8/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 20:24:30,581 - Train: 21.18% [402900/1902000] [211.8/1000.0] [batch_t 0.608 (0.606)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-05 20:25:31,201 - Train: 21.19% [403000/1902000] [211.9/1000.0] [batch_t 0.601 (0.606)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-05 20:26:31,898 - Train: 21.19% [403100/1902000] [211.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 20:27:32,495 - Train: 21.20% [403200/1902000] [212.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 20:27:47,060 - ==> Total time: 3 days, 19:27:26 Eta: 14 days, 3:56:43 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 20:28:35,495 - Train: 21.20% [403300/1902000] [212.0/1000.0] [batch_t 0.607 (0.610)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 20:29:36,157 - Train: 21.21% [403400/1902000] [212.1/1000.0] [batch_t 0.612 (0.607)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-05 20:30:36,799 - Train: 21.21% [403500/1902000] [212.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 20:31:39,270 - Train: 21.22% [403600/1902000] [212.2/1000.0] [batch_t 0.606 (0.625)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 20:32:39,928 - Train: 21.23% [403700/1902000] [212.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 20:33:40,519 - Train: 21.23% [403800/1902000] [212.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 20:34:41,104 - Train: 21.24% [403900/1902000] [212.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 20:35:41,781 - Train: 21.24% [404000/1902000] [212.4/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 20:36:42,401 - Train: 21.25% [404100/1902000] [212.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 20:37:43,020 - Train: 21.25% [404200/1902000] [212.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 20:38:43,751 - Train: 21.26% [404300/1902000] [212.6/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 20:39:45,476 - Train: 21.26% [404400/1902000] [212.6/1000.0] [batch_t 0.606 (0.617)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 20:40:46,112 - Train: 21.27% [404500/1902000] [212.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 20:41:46,783 - Train: 21.27% [404600/1902000] [212.7/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 20:42:47,426 - Train: 21.28% [404700/1902000] [212.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 20:43:48,072 - Train: 21.28% [404800/1902000] [212.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 20:44:48,735 - Train: 21.29% [404900/1902000] [212.9/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 20:45:49,479 - Train: 21.29% [405000/1902000] [212.9/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 20:46:50,110 - Train: 21.30% [405100/1902000] [213.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 20:47:05,864 - ==> Total time: 3 days, 19:46:45 Eta: 14 days, 3:06:33 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 20:47:53,749 - Train: 21.30% [405200/1902000] [213.0/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 20:48:54,354 - Train: 21.31% [405300/1902000] [213.1/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 20:49:54,952 - Train: 21.31% [405400/1902000] [213.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 20:50:55,564 - Train: 21.32% [405500/1902000] [213.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 20:51:56,174 - Train: 21.32% [405600/1902000] [213.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 20:52:56,925 - Train: 21.33% [405700/1902000] [213.3/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 20:53:57,539 - Train: 21.34% [405800/1902000] [213.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 20:54:58,191 - Train: 21.34% [405900/1902000] [213.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 20:55:58,804 - Train: 21.35% [406000/1902000] [213.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 20:56:59,449 - Train: 21.35% [406100/1902000] [213.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 20:58:00,194 - Train: 21.36% [406200/1902000] [213.6/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 20:59:00,809 - Train: 21.36% [406300/1902000] [213.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:00:05,876 - Train: 21.37% [406400/1902000] [213.7/1000.0] [batch_t 0.611 (0.651)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-05 21:01:06,512 - Train: 21.37% [406500/1902000] [213.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:02:07,538 - Train: 21.38% [406600/1902000] [213.8/1000.0] [batch_t 0.606 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:03:08,169 - Train: 21.38% [406700/1902000] [213.8/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 21:04:09,354 - Train: 21.39% [406800/1902000] [213.9/1000.0] [batch_t 0.605 (0.612)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:05:10,082 - Train: 21.39% [406900/1902000] [213.9/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 21:06:10,713 - Train: 21.40% [407000/1902000] [214.0/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 21:06:27,706 - ==> Total time: 3 days, 20:06:07 Eta: 14 days, 2:16:52 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 21:07:13,798 - Train: 21.40% [407100/1902000] [214.0/1000.0] [batch_t 0.610 (0.610)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-05 21:08:14,470 - Train: 21.41% [407200/1902000] [214.1/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:09:15,452 - Train: 21.41% [407300/1902000] [214.1/1000.0] [batch_t 0.608 (0.610)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 21:10:16,071 - Train: 21.42% [407400/1902000] [214.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:11:18,122 - Train: 21.42% [407500/1902000] [214.2/1000.0] [batch_t 0.604 (0.620)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 21:12:18,844 - Train: 21.43% [407600/1902000] [214.3/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 21:13:19,476 - Train: 21.44% [407700/1902000] [214.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:14:20,098 - Train: 21.44% [407800/1902000] [214.4/1000.0] [batch_t 0.612 (0.606)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-05 21:15:20,757 - Train: 21.45% [407900/1902000] [214.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 21:16:21,409 - Train: 21.45% [408000/1902000] [214.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:17:22,210 - Train: 21.46% [408100/1902000] [214.6/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:18:23,242 - Train: 21.46% [408200/1902000] [214.6/1000.0] [batch_t 0.606 (0.610)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 21:19:24,193 - Train: 21.47% [408300/1902000] [214.7/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:20:24,844 - Train: 21.47% [408400/1902000] [214.7/1000.0] [batch_t 0.613 (0.606)] [data_t 0.002] [optim_t 0.611] [lr 0.005000] +2024-04-05 21:21:25,487 - Train: 21.48% [408500/1902000] [214.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 21:22:26,135 - Train: 21.48% [408600/1902000] [214.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:23:26,780 - Train: 21.49% [408700/1902000] [214.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 21:24:27,590 - Train: 21.49% [408800/1902000] [214.9/1000.0] [batch_t 0.744 (0.608)] [data_t 0.002] [optim_t 0.742] [lr 0.005000] +2024-04-05 21:25:28,762 - Train: 21.50% [408900/1902000] [215.0/1000.0] [batch_t 0.605 (0.612)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:25:46,952 - ==> Total time: 3 days, 20:25:26 Eta: 14 days, 1:27:18 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 21:26:31,679 - Train: 21.50% [409000/1902000] [215.0/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 21:27:32,321 - Train: 21.51% [409100/1902000] [215.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 21:28:32,940 - Train: 21.51% [409200/1902000] [215.1/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 21:29:33,560 - Train: 21.52% [409300/1902000] [215.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:30:34,192 - Train: 21.52% [409400/1902000] [215.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:31:34,947 - Train: 21.53% [409500/1902000] [215.3/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 21:32:36,803 - Train: 21.54% [409600/1902000] [215.4/1000.0] [batch_t 0.604 (0.618)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 21:33:37,420 - Train: 21.54% [409700/1902000] [215.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:34:38,032 - Train: 21.55% [409800/1902000] [215.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:35:38,653 - Train: 21.55% [409900/1902000] [215.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:36:39,337 - Train: 21.56% [410000/1902000] [215.6/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 21:37:39,948 - Train: 21.56% [410100/1902000] [215.6/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 21:38:41,325 - Train: 21.57% [410200/1902000] [215.7/1000.0] [batch_t 0.605 (0.614)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:39:41,964 - Train: 21.57% [410300/1902000] [215.7/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 21:40:42,634 - Train: 21.58% [410400/1902000] [215.8/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:41:43,321 - Train: 21.58% [410500/1902000] [215.8/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 21:42:43,975 - Train: 21.59% [410600/1902000] [215.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:43:45,805 - Train: 21.59% [410700/1902000] [215.9/1000.0] [batch_t 0.604 (0.618)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:44:46,567 - Train: 21.60% [410800/1902000] [216.0/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 21:45:05,985 - ==> Total time: 3 days, 20:44:45 Eta: 14 days, 0:38:00 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 21:45:49,670 - Train: 21.60% [410900/1902000] [216.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 21:46:50,307 - Train: 21.61% [411000/1902000] [216.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:47:50,920 - Train: 21.61% [411100/1902000] [216.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:48:51,551 - Train: 21.62% [411200/1902000] [216.2/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 21:49:52,353 - Train: 21.62% [411300/1902000] [216.2/1000.0] [batch_t 0.602 (0.608)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 21:50:52,980 - Train: 21.63% [411400/1902000] [216.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:51:53,566 - Train: 21.64% [411500/1902000] [216.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:52:54,195 - Train: 21.64% [411600/1902000] [216.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:53:54,806 - Train: 21.65% [411700/1902000] [216.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 21:54:55,468 - Train: 21.65% [411800/1902000] [216.5/1000.0] [batch_t 0.608 (0.607)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-05 21:55:56,094 - Train: 21.66% [411900/1902000] [216.6/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 21:56:56,781 - Train: 21.66% [412000/1902000] [216.6/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 21:57:57,388 - Train: 21.67% [412100/1902000] [216.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 21:58:58,006 - Train: 21.67% [412200/1902000] [216.7/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 21:59:58,619 - Train: 21.68% [412300/1902000] [216.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 22:00:59,463 - Train: 21.68% [412400/1902000] [216.8/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 22:02:00,074 - Train: 21.69% [412500/1902000] [216.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 22:03:00,811 - Train: 21.69% [412600/1902000] [216.9/1000.0] [batch_t 0.609 (0.607)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 22:04:02,097 - Train: 21.70% [412700/1902000] [217.0/1000.0] [batch_t 0.607 (0.613)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 22:04:23,796 - ==> Total time: 3 days, 21:04:03 Eta: 13 days, 23:48:55 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 22:05:06,267 - Train: 21.70% [412800/1902000] [217.0/1000.0] [batch_t 0.608 (0.611)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-05 22:06:06,899 - Train: 21.71% [412900/1902000] [217.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 22:07:07,534 - Train: 21.71% [413000/1902000] [217.1/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 22:08:08,696 - Train: 21.72% [413100/1902000] [217.2/1000.0] [batch_t 0.606 (0.612)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 22:09:09,328 - Train: 21.72% [413200/1902000] [217.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 22:10:10,122 - Train: 21.73% [413300/1902000] [217.3/1000.0] [batch_t 0.608 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 22:11:11,764 - Train: 21.74% [413400/1902000] [217.4/1000.0] [batch_t 0.605 (0.616)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 22:12:12,418 - Train: 21.74% [413500/1902000] [217.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 22:13:13,032 - Train: 21.75% [413600/1902000] [217.5/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-05 22:14:13,665 - Train: 21.75% [413700/1902000] [217.5/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-05 22:15:14,435 - Train: 21.76% [413800/1902000] [217.6/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 22:16:15,058 - Train: 21.76% [413900/1902000] [217.6/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 22:17:15,807 - Train: 21.77% [414000/1902000] [217.7/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 22:18:16,452 - Train: 21.77% [414100/1902000] [217.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 22:19:17,045 - Train: 21.78% [414200/1902000] [217.8/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 22:20:17,670 - Train: 21.78% [414300/1902000] [217.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 22:21:18,289 - Train: 21.79% [414400/1902000] [217.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 22:22:18,995 - Train: 21.79% [414500/1902000] [217.9/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 22:23:19,907 - Train: 21.80% [414600/1902000] [218.0/1000.0] [batch_t 0.607 (0.609)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 22:23:41,732 - ==> Total time: 3 days, 21:23:21 Eta: 13 days, 23:00:06 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 22:24:22,561 - Train: 21.80% [414700/1902000] [218.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 22:25:23,195 - Train: 21.81% [414800/1902000] [218.1/1000.0] [batch_t 0.609 (0.606)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-05 22:26:23,796 - Train: 21.81% [414900/1902000] [218.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 22:27:24,412 - Train: 21.82% [415000/1902000] [218.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 22:28:25,063 - Train: 21.82% [415100/1902000] [218.2/1000.0] [batch_t 0.612 (0.606)] [data_t 0.002] [optim_t 0.610] [lr 0.005000] +2024-04-05 22:29:25,863 - Train: 21.83% [415200/1902000] [218.3/1000.0] [batch_t 0.606 (0.608)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-05 22:30:26,518 - Train: 21.83% [415300/1902000] [218.3/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 22:31:27,091 - Train: 21.84% [415400/1902000] [218.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 22:32:27,709 - Train: 21.85% [415500/1902000] [218.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 22:33:28,266 - Train: 21.85% [415600/1902000] [218.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 22:34:29,036 - Train: 21.86% [415700/1902000] [218.6/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 22:35:29,638 - Train: 21.86% [415800/1902000] [218.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 22:36:30,270 - Train: 21.87% [415900/1902000] [218.7/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 22:37:30,867 - Train: 21.87% [416000/1902000] [218.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 22:38:31,463 - Train: 21.88% [416100/1902000] [218.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 22:39:32,560 - Train: 21.88% [416200/1902000] [218.8/1000.0] [batch_t 0.606 (0.611)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 22:40:33,488 - Train: 21.89% [416300/1902000] [218.9/1000.0] [batch_t 0.607 (0.609)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 22:41:34,113 - Train: 21.89% [416400/1902000] [218.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 22:42:35,188 - Train: 21.90% [416500/1902000] [219.0/1000.0] [batch_t 0.605 (0.611)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 22:42:58,219 - ==> Total time: 3 days, 21:42:37 Eta: 13 days, 22:11:28 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 22:43:38,145 - Train: 21.90% [416600/1902000] [219.0/1000.0] [batch_t 0.607 (0.610)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 22:44:38,790 - Train: 21.91% [416700/1902000] [219.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 22:45:39,398 - Train: 21.91% [416800/1902000] [219.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 22:46:40,002 - Train: 21.92% [416900/1902000] [219.2/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 22:47:40,641 - Train: 21.92% [417000/1902000] [219.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 22:48:41,429 - Train: 21.93% [417100/1902000] [219.3/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 22:49:42,128 - Train: 21.93% [417200/1902000] [219.3/1000.0] [batch_t 0.608 (0.607)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-05 22:50:42,791 - Train: 21.94% [417300/1902000] [219.4/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 22:51:43,415 - Train: 21.95% [417400/1902000] [219.5/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 22:52:44,023 - Train: 21.95% [417500/1902000] [219.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 22:53:44,705 - Train: 21.96% [417600/1902000] [219.6/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 22:54:46,325 - Train: 21.96% [417700/1902000] [219.6/1000.0] [batch_t 0.605 (0.616)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 22:55:46,964 - Train: 21.97% [417800/1902000] [219.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 22:56:47,609 - Train: 21.97% [417900/1902000] [219.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 22:57:48,262 - Train: 21.98% [418000/1902000] [219.8/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 22:58:48,983 - Train: 21.98% [418100/1902000] [219.8/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 22:59:49,615 - Train: 21.99% [418200/1902000] [219.9/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 23:00:50,246 - Train: 21.99% [418300/1902000] [219.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:01:50,998 - Train: 22.00% [418400/1902000] [220.0/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 23:02:15,222 - ==> Total time: 3 days, 22:01:54 Eta: 13 days, 21:23:08 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 23:02:54,034 - Train: 22.00% [418500/1902000] [220.0/1000.0] [batch_t 0.607 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:03:54,667 - Train: 22.01% [418600/1902000] [220.1/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 23:04:55,285 - Train: 22.01% [418700/1902000] [220.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:05:55,916 - Train: 22.02% [418800/1902000] [220.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 23:06:56,675 - Train: 22.02% [418900/1902000] [220.2/1000.0] [batch_t 0.606 (0.607)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-05 23:07:57,301 - Train: 22.03% [419000/1902000] [220.3/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 23:08:57,927 - Train: 22.03% [419100/1902000] [220.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:09:58,573 - Train: 22.04% [419200/1902000] [220.4/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 23:10:59,175 - Train: 22.05% [419300/1902000] [220.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 23:12:00,368 - Train: 22.05% [419400/1902000] [220.5/1000.0] [batch_t 0.604 (0.612)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 23:13:01,077 - Train: 22.06% [419500/1902000] [220.6/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 23:14:01,661 - Train: 22.06% [419600/1902000] [220.6/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 23:15:02,313 - Train: 22.07% [419700/1902000] [220.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 23:16:02,908 - Train: 22.07% [419800/1902000] [220.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:17:03,492 - Train: 22.08% [419900/1902000] [220.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 23:18:04,054 - Train: 22.08% [420000/1902000] [220.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:19:04,623 - Train: 22.09% [420100/1902000] [220.9/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 23:20:05,295 - Train: 22.09% [420200/1902000] [220.9/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 23:21:06,934 - Train: 22.10% [420300/1902000] [221.0/1000.0] [batch_t 0.604 (0.616)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 23:21:32,371 - ==> Total time: 3 days, 22:21:11 Eta: 13 days, 20:35:04 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 23:22:09,973 - Train: 22.10% [420400/1902000] [221.0/1000.0] [batch_t 0.604 (0.614)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 23:23:10,907 - Train: 22.11% [420500/1902000] [221.1/1000.0] [batch_t 0.607 (0.609)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 23:24:11,507 - Train: 22.11% [420600/1902000] [221.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 23:25:12,087 - Train: 22.12% [420700/1902000] [221.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 23:26:12,610 - Train: 22.12% [420800/1902000] [221.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 23:27:13,305 - Train: 22.13% [420900/1902000] [221.3/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 23:28:13,863 - Train: 22.13% [421000/1902000] [221.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-05 23:29:14,426 - Train: 22.14% [421100/1902000] [221.4/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 23:30:15,006 - Train: 22.15% [421200/1902000] [221.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 23:31:15,585 - Train: 22.15% [421300/1902000] [221.5/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 23:32:16,254 - Train: 22.16% [421400/1902000] [221.6/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 23:33:16,833 - Train: 22.16% [421500/1902000] [221.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 23:34:17,822 - Train: 22.17% [421600/1902000] [221.7/1000.0] [batch_t 0.609 (0.610)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-05 23:35:18,413 - Train: 22.17% [421700/1902000] [221.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 23:36:18,959 - Train: 22.18% [421800/1902000] [221.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 23:37:19,516 - Train: 22.18% [421900/1902000] [221.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 23:38:20,178 - Train: 22.19% [422000/1902000] [221.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:39:20,748 - Train: 22.19% [422100/1902000] [221.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 23:40:22,187 - Train: 22.20% [422200/1902000] [222.0/1000.0] [batch_t 0.604 (0.614)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 23:40:48,848 - ==> Total time: 3 days, 22:40:28 Eta: 13 days, 19:47:14 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-05 23:41:25,463 - Train: 22.20% [422300/1902000] [222.0/1000.0] [batch_t 0.607 (0.618)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 23:42:26,049 - Train: 22.21% [422400/1902000] [222.1/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-05 23:43:26,597 - Train: 22.21% [422500/1902000] [222.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 23:44:27,313 - Train: 22.22% [422600/1902000] [222.2/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 23:45:27,902 - Train: 22.22% [422700/1902000] [222.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 23:46:29,003 - Train: 22.23% [422800/1902000] [222.3/1000.0] [batch_t 0.605 (0.611)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 23:47:29,924 - Train: 22.23% [422900/1902000] [222.3/1000.0] [batch_t 0.610 (0.609)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-05 23:48:30,454 - Train: 22.24% [423000/1902000] [222.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 23:49:31,022 - Train: 22.25% [423100/1902000] [222.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:50:31,575 - Train: 22.25% [423200/1902000] [222.5/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-05 23:51:32,188 - Train: 22.26% [423300/1902000] [222.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 23:52:33,037 - Train: 22.26% [423400/1902000] [222.6/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-05 23:53:33,603 - Train: 22.27% [423500/1902000] [222.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-05 23:54:34,173 - Train: 22.27% [423600/1902000] [222.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:55:34,747 - Train: 22.28% [423700/1902000] [222.8/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-05 23:56:35,780 - Train: 22.28% [423800/1902000] [222.8/1000.0] [batch_t 0.608 (0.610)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-05 23:57:36,382 - Train: 22.29% [423900/1902000] [222.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:58:37,630 - Train: 22.29% [424000/1902000] [222.9/1000.0] [batch_t 0.606 (0.612)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-05 23:59:39,061 - Train: 22.30% [424100/1902000] [223.0/1000.0] [batch_t 0.610 (0.614)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-06 00:00:06,929 - ==> Total time: 3 days, 22:59:46 Eta: 13 days, 18:59:45 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 00:00:41,779 - Train: 22.30% [424200/1902000] [223.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 00:01:42,396 - Train: 22.31% [424300/1902000] [223.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 00:02:42,974 - Train: 22.31% [424400/1902000] [223.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 00:03:43,716 - Train: 22.32% [424500/1902000] [223.2/1000.0] [batch_t 0.609 (0.607)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 00:04:44,316 - Train: 22.32% [424600/1902000] [223.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:05:44,872 - Train: 22.33% [424700/1902000] [223.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 00:06:45,410 - Train: 22.33% [424800/1902000] [223.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 00:07:46,493 - Train: 22.34% [424900/1902000] [223.4/1000.0] [batch_t 0.605 (0.611)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:08:47,080 - Train: 22.34% [425000/1902000] [223.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 00:09:47,653 - Train: 22.35% [425100/1902000] [223.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:10:48,218 - Train: 22.36% [425200/1902000] [223.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 00:11:50,234 - Train: 22.36% [425300/1902000] [223.6/1000.0] [batch_t 0.607 (0.620)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 00:12:50,820 - Train: 22.37% [425400/1902000] [223.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 00:13:51,416 - Train: 22.37% [425500/1902000] [223.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:14:52,020 - Train: 22.38% [425600/1902000] [223.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 00:15:52,665 - Train: 22.38% [425700/1902000] [223.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 00:16:53,527 - Train: 22.39% [425800/1902000] [223.9/1000.0] [batch_t 0.603 (0.609)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 00:17:54,161 - Train: 22.39% [425900/1902000] [223.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 00:18:54,732 - Train: 22.40% [426000/1902000] [224.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 00:19:24,509 - ==> Total time: 3 days, 23:19:04 Eta: 13 days, 18:12:28 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 00:19:58,315 - Train: 22.40% [426100/1902000] [224.0/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 00:20:58,906 - Train: 22.41% [426200/1902000] [224.1/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 00:21:59,482 - Train: 22.41% [426300/1902000] [224.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 00:23:00,023 - Train: 22.42% [426400/1902000] [224.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 00:24:00,764 - Train: 22.42% [426500/1902000] [224.2/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:25:01,383 - Train: 22.43% [426600/1902000] [224.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 00:26:01,932 - Train: 22.43% [426700/1902000] [224.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 00:27:02,485 - Train: 22.44% [426800/1902000] [224.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 00:28:03,039 - Train: 22.44% [426900/1902000] [224.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 00:29:03,754 - Train: 22.45% [427000/1902000] [224.5/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 00:30:04,296 - Train: 22.46% [427100/1902000] [224.6/1000.0] [batch_t 0.611 (0.605)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-06 00:31:04,882 - Train: 22.46% [427200/1902000] [224.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:32:05,454 - Train: 22.47% [427300/1902000] [224.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 00:33:06,042 - Train: 22.47% [427400/1902000] [224.7/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-06 00:34:06,585 - Train: 22.48% [427500/1902000] [224.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 00:35:07,145 - Train: 22.48% [427600/1902000] [224.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 00:36:07,706 - Train: 22.49% [427700/1902000] [224.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:37:08,602 - Train: 22.49% [427800/1902000] [224.9/1000.0] [batch_t 0.607 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:38:09,129 - Train: 22.50% [427900/1902000] [225.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 00:38:39,397 - ==> Total time: 3 days, 23:38:18 Eta: 13 days, 17:25:18 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 00:39:12,026 - Train: 22.50% [428000/1902000] [225.0/1000.0] [batch_t 0.608 (0.608)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 00:40:12,593 - Train: 22.51% [428100/1902000] [225.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 00:41:13,140 - Train: 22.51% [428200/1902000] [225.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 00:42:13,834 - Train: 22.52% [428300/1902000] [225.2/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 00:43:14,380 - Train: 22.52% [428400/1902000] [225.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 00:44:14,963 - Train: 22.53% [428500/1902000] [225.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 00:45:15,541 - Train: 22.53% [428600/1902000] [225.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 00:46:16,198 - Train: 22.54% [428700/1902000] [225.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 00:47:16,775 - Train: 22.54% [428800/1902000] [225.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 00:48:18,011 - Train: 22.55% [428900/1902000] [225.5/1000.0] [batch_t 0.607 (0.612)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 00:49:18,758 - Train: 22.56% [429000/1902000] [225.6/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 00:50:19,303 - Train: 22.56% [429100/1902000] [225.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 00:51:19,912 - Train: 22.57% [429200/1902000] [225.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:52:21,066 - Train: 22.57% [429300/1902000] [225.7/1000.0] [batch_t 0.605 (0.611)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:53:21,633 - Train: 22.58% [429400/1902000] [225.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:54:22,198 - Train: 22.58% [429500/1902000] [225.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:55:22,796 - Train: 22.59% [429600/1902000] [225.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:56:23,508 - Train: 22.59% [429700/1902000] [225.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:57:24,135 - Train: 22.60% [429800/1902000] [226.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 00:57:55,645 - ==> Total time: 3 days, 23:57:35 Eta: 13 days, 16:38:27 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 00:58:27,070 - Train: 22.60% [429900/1902000] [226.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 00:59:27,624 - Train: 22.61% [430000/1902000] [226.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 01:00:28,194 - Train: 22.61% [430100/1902000] [226.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 01:01:28,730 - Train: 22.62% [430200/1902000] [226.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 01:02:29,458 - Train: 22.62% [430300/1902000] [226.2/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 01:03:29,992 - Train: 22.63% [430400/1902000] [226.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 01:04:31,449 - Train: 22.63% [430500/1902000] [226.3/1000.0] [batch_t 0.606 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 01:05:31,971 - Train: 22.64% [430600/1902000] [226.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 01:06:32,560 - Train: 22.64% [430700/1902000] [226.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 01:07:33,501 - Train: 22.65% [430800/1902000] [226.5/1000.0] [batch_t 0.604 (0.609)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 01:08:34,155 - Train: 22.66% [430900/1902000] [226.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 01:09:34,745 - Train: 22.66% [431000/1902000] [226.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 01:10:35,786 - Train: 22.67% [431100/1902000] [226.7/1000.0] [batch_t 0.607 (0.610)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 01:11:36,475 - Train: 22.67% [431200/1902000] [226.7/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 01:12:37,430 - Train: 22.68% [431300/1902000] [226.8/1000.0] [batch_t 0.608 (0.609)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 01:13:38,144 - Train: 22.68% [431400/1902000] [226.8/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 01:14:38,907 - Train: 22.69% [431500/1902000] [226.9/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 01:15:39,635 - Train: 22.69% [431600/1902000] [226.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 01:16:40,644 - Train: 22.70% [431700/1902000] [227.0/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 01:17:13,353 - ==> Total time: 4 days, 0:16:52 Eta: 13 days, 15:51:56 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 01:17:43,507 - Train: 22.70% [431800/1902000] [227.0/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 01:18:44,078 - Train: 22.71% [431900/1902000] [227.1/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-06 01:19:44,646 - Train: 22.71% [432000/1902000] [227.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 01:20:45,356 - Train: 22.72% [432100/1902000] [227.2/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 01:21:45,943 - Train: 22.72% [432200/1902000] [227.2/1000.0] [batch_t 0.615 (0.606)] [data_t 0.002] [optim_t 0.613] [lr 0.005000] +2024-04-06 01:22:46,557 - Train: 22.73% [432300/1902000] [227.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 01:23:47,325 - Train: 22.73% [432400/1902000] [227.3/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 01:24:47,862 - Train: 22.74% [432500/1902000] [227.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 01:25:48,472 - Train: 22.74% [432600/1902000] [227.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 01:26:49,183 - Train: 22.75% [432700/1902000] [227.5/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 01:27:49,790 - Train: 22.75% [432800/1902000] [227.5/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 01:28:50,766 - Train: 22.76% [432900/1902000] [227.6/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 01:29:51,940 - Train: 22.77% [433000/1902000] [227.7/1000.0] [batch_t 0.605 (0.612)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 01:30:53,396 - Train: 22.77% [433100/1902000] [227.7/1000.0] [batch_t 0.606 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 01:31:53,982 - Train: 22.78% [433200/1902000] [227.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 01:32:54,544 - Train: 22.78% [433300/1902000] [227.8/1000.0] [batch_t 0.601 (0.606)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 01:33:55,260 - Train: 22.79% [433400/1902000] [227.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 01:34:55,818 - Train: 22.79% [433500/1902000] [227.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 01:35:56,380 - Train: 22.80% [433600/1902000] [228.0/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 01:36:30,299 - ==> Total time: 4 days, 0:36:09 Eta: 13 days, 15:05:36 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 01:36:59,396 - Train: 22.80% [433700/1902000] [228.0/1000.0] [batch_t 0.608 (0.612)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 01:37:59,963 - Train: 22.81% [433800/1902000] [228.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 01:39:00,485 - Train: 22.81% [433900/1902000] [228.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 01:40:01,087 - Train: 22.82% [434000/1902000] [228.2/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 01:41:01,819 - Train: 22.82% [434100/1902000] [228.2/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 01:42:02,656 - Train: 22.83% [434200/1902000] [228.3/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 01:43:03,228 - Train: 22.83% [434300/1902000] [228.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 01:44:03,781 - Train: 22.84% [434400/1902000] [228.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 01:45:04,336 - Train: 22.84% [434500/1902000] [228.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 01:46:04,880 - Train: 22.85% [434600/1902000] [228.5/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 01:47:05,457 - Train: 22.85% [434700/1902000] [228.5/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 01:48:06,083 - Train: 22.86% [434800/1902000] [228.6/1000.0] [batch_t 0.679 (0.606)] [data_t 0.002] [optim_t 0.677] [lr 0.005000] +2024-04-06 01:49:06,836 - Train: 22.87% [434900/1902000] [228.7/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 01:50:07,364 - Train: 22.87% [435000/1902000] [228.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 01:51:07,951 - Train: 22.88% [435100/1902000] [228.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 01:52:08,492 - Train: 22.88% [435200/1902000] [228.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 01:53:09,053 - Train: 22.89% [435300/1902000] [228.9/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 01:54:09,700 - Train: 22.89% [435400/1902000] [228.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 01:55:10,261 - Train: 22.90% [435500/1902000] [229.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 01:55:45,393 - ==> Total time: 4 days, 0:55:24 Eta: 13 days, 14:19:24 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 01:56:13,003 - Train: 22.90% [435600/1902000] [229.0/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 01:57:13,567 - Train: 22.91% [435700/1902000] [229.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 01:58:14,129 - Train: 22.91% [435800/1902000] [229.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 01:59:14,822 - Train: 22.92% [435900/1902000] [229.2/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 02:00:15,401 - Train: 22.92% [436000/1902000] [229.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 02:01:15,947 - Train: 22.93% [436100/1902000] [229.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 02:02:16,467 - Train: 22.93% [436200/1902000] [229.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 02:03:16,993 - Train: 22.94% [436300/1902000] [229.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:04:17,509 - Train: 22.94% [436400/1902000] [229.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 02:05:18,043 - Train: 22.95% [436500/1902000] [229.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:06:18,985 - Train: 22.95% [436600/1902000] [229.5/1000.0] [batch_t 0.602 (0.609)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 02:07:19,602 - Train: 22.96% [436700/1902000] [229.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 02:08:20,137 - Train: 22.97% [436800/1902000] [229.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:09:20,684 - Train: 22.97% [436900/1902000] [229.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 02:10:21,260 - Train: 22.98% [437000/1902000] [229.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 02:11:21,875 - Train: 22.98% [437100/1902000] [229.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 02:12:22,599 - Train: 22.99% [437200/1902000] [229.9/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:13:23,834 - Train: 22.99% [437300/1902000] [229.9/1000.0] [batch_t 0.607 (0.612)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 02:14:24,457 - Train: 23.00% [437400/1902000] [230.0/1000.0] [batch_t 0.612 (0.606)] [data_t 0.002] [optim_t 0.610] [lr 0.005000] +2024-04-06 02:15:00,832 - ==> Total time: 4 days, 1:14:40 Eta: 13 days, 13:33:28 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 02:15:30,945 - Train: 23.00% [437500/1902000] [230.0/1000.0] [batch_t 0.612 (0.663)] [data_t 0.002] [optim_t 0.610] [lr 0.005000] +2024-04-06 02:16:31,509 - Train: 23.01% [437600/1902000] [230.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 02:17:32,094 - Train: 23.01% [437700/1902000] [230.1/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 02:18:32,664 - Train: 23.02% [437800/1902000] [230.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:19:33,369 - Train: 23.02% [437900/1902000] [230.2/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:20:34,002 - Train: 23.03% [438000/1902000] [230.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 02:21:34,553 - Train: 23.03% [438100/1902000] [230.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 02:22:35,118 - Train: 23.04% [438200/1902000] [230.4/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 02:23:35,698 - Train: 23.04% [438300/1902000] [230.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:24:36,293 - Train: 23.05% [438400/1902000] [230.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 02:25:36,954 - Train: 23.05% [438500/1902000] [230.5/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 02:26:37,530 - Train: 23.06% [438600/1902000] [230.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 02:27:38,110 - Train: 23.07% [438700/1902000] [230.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 02:28:38,750 - Train: 23.07% [438800/1902000] [230.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 02:29:39,358 - Train: 23.08% [438900/1902000] [230.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:30:39,925 - Train: 23.08% [439000/1902000] [230.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:31:40,492 - Train: 23.09% [439100/1902000] [230.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:32:41,152 - Train: 23.09% [439200/1902000] [230.9/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:33:41,717 - Train: 23.10% [439300/1902000] [231.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 02:34:19,282 - ==> Total time: 4 days, 1:33:58 Eta: 13 days, 12:47:55 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 02:34:44,767 - Train: 23.10% [439400/1902000] [231.0/1000.0] [batch_t 0.617 (0.607)] [data_t 0.002] [optim_t 0.615] [lr 0.005000] +2024-04-06 02:35:45,338 - Train: 23.11% [439500/1902000] [231.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 02:36:45,892 - Train: 23.11% [439600/1902000] [231.1/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 02:37:47,200 - Train: 23.12% [439700/1902000] [231.2/1000.0] [batch_t 0.607 (0.613)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 02:38:47,802 - Train: 23.12% [439800/1902000] [231.2/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 02:39:48,626 - Train: 23.13% [439900/1902000] [231.3/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:40:49,184 - Train: 23.13% [440000/1902000] [231.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 02:41:50,188 - Train: 23.14% [440100/1902000] [231.4/1000.0] [batch_t 0.606 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 02:42:50,723 - Train: 23.14% [440200/1902000] [231.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:43:51,270 - Train: 23.15% [440300/1902000] [231.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 02:44:51,914 - Train: 23.15% [440400/1902000] [231.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-06 02:45:52,674 - Train: 23.16% [440500/1902000] [231.6/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:46:53,401 - Train: 23.17% [440600/1902000] [231.7/1000.0] [batch_t 0.607 (0.607)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-06 02:47:54,134 - Train: 23.17% [440700/1902000] [231.7/1000.0] [batch_t 0.612 (0.607)] [data_t 0.003] [optim_t 0.609] [lr 0.005000] +2024-04-06 02:48:54,927 - Train: 23.18% [440800/1902000] [231.8/1000.0] [batch_t 0.610 (0.608)] [data_t 0.003] [optim_t 0.608] [lr 0.005000] +2024-04-06 02:49:55,656 - Train: 23.18% [440900/1902000] [231.8/1000.0] [batch_t 0.609 (0.607)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-06 02:50:56,449 - Train: 23.19% [441000/1902000] [231.9/1000.0] [batch_t 0.608 (0.608)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-06 02:51:57,149 - Train: 23.19% [441100/1902000] [231.9/1000.0] [batch_t 0.603 (0.607)] [data_t 0.003] [optim_t 0.601] [lr 0.005000] +2024-04-06 02:52:57,865 - Train: 23.20% [441200/1902000] [232.0/1000.0] [batch_t 0.609 (0.607)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-06 02:53:36,727 - ==> Total time: 4 days, 1:53:16 Eta: 13 days, 12:02:33 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 02:54:00,817 - Train: 23.20% [441300/1902000] [232.0/1000.0] [batch_t 0.607 (0.608)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-06 02:55:01,518 - Train: 23.21% [441400/1902000] [232.1/1000.0] [batch_t 0.607 (0.607)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-06 02:56:02,247 - Train: 23.21% [441500/1902000] [232.1/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 02:57:03,106 - Train: 23.22% [441600/1902000] [232.2/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 02:58:03,816 - Train: 23.22% [441700/1902000] [232.2/1000.0] [batch_t 0.606 (0.607)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-06 02:59:04,516 - Train: 23.23% [441800/1902000] [232.3/1000.0] [batch_t 0.608 (0.607)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-06 03:00:05,244 - Train: 23.23% [441900/1902000] [232.3/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:01:06,028 - Train: 23.24% [442000/1902000] [232.4/1000.0] [batch_t 0.604 (0.608)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-06 03:02:06,744 - Train: 23.24% [442100/1902000] [232.4/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 03:03:07,457 - Train: 23.25% [442200/1902000] [232.5/1000.0] [batch_t 0.607 (0.607)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-06 03:04:08,248 - Train: 23.25% [442300/1902000] [232.5/1000.0] [batch_t 0.608 (0.608)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-06 03:05:08,929 - Train: 23.26% [442400/1902000] [232.6/1000.0] [batch_t 0.609 (0.607)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-06 03:06:09,644 - Train: 23.26% [442500/1902000] [232.6/1000.0] [batch_t 0.607 (0.607)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-06 03:07:10,375 - Train: 23.27% [442600/1902000] [232.7/1000.0] [batch_t 0.608 (0.607)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-06 03:08:11,091 - Train: 23.28% [442700/1902000] [232.8/1000.0] [batch_t 0.609 (0.607)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-06 03:09:11,760 - Train: 23.28% [442800/1902000] [232.8/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 03:10:12,638 - Train: 23.29% [442900/1902000] [232.9/1000.0] [batch_t 0.606 (0.609)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:11:13,350 - Train: 23.29% [443000/1902000] [232.9/1000.0] [batch_t 0.609 (0.607)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-06 03:12:14,073 - Train: 23.30% [443100/1902000] [233.0/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 03:12:54,137 - ==> Total time: 4 days, 2:12:33 Eta: 13 days, 11:17:24 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 03:13:17,041 - Train: 23.30% [443200/1902000] [233.0/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:14:17,720 - Train: 23.31% [443300/1902000] [233.1/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 03:15:18,414 - Train: 23.31% [443400/1902000] [233.1/1000.0] [batch_t 0.608 (0.607)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-06 03:16:19,190 - Train: 23.32% [443500/1902000] [233.2/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:17:19,807 - Train: 23.32% [443600/1902000] [233.2/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 03:18:20,403 - Train: 23.33% [443700/1902000] [233.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:19:21,019 - Train: 23.33% [443800/1902000] [233.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 03:20:21,625 - Train: 23.34% [443900/1902000] [233.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:21:23,189 - Train: 23.34% [444000/1902000] [233.4/1000.0] [batch_t 0.607 (0.616)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 03:22:23,844 - Train: 23.35% [444100/1902000] [233.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 03:23:24,440 - Train: 23.35% [444200/1902000] [233.5/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 03:24:25,116 - Train: 23.36% [444300/1902000] [233.6/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:25:25,705 - Train: 23.36% [444400/1902000] [233.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 03:26:26,299 - Train: 23.37% [444500/1902000] [233.7/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 03:27:26,929 - Train: 23.38% [444600/1902000] [233.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:28:27,542 - Train: 23.38% [444700/1902000] [233.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:29:28,430 - Train: 23.39% [444800/1902000] [233.9/1000.0] [batch_t 0.607 (0.609)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 03:30:29,054 - Train: 23.39% [444900/1902000] [233.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:31:29,696 - Train: 23.40% [445000/1902000] [234.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:32:10,943 - ==> Total time: 4 days, 2:31:50 Eta: 13 days, 10:32:26 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 03:32:32,409 - Train: 23.40% [445100/1902000] [234.0/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:33:33,050 - Train: 23.41% [445200/1902000] [234.1/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 03:34:33,650 - Train: 23.41% [445300/1902000] [234.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:35:34,272 - Train: 23.42% [445400/1902000] [234.2/1000.0] [batch_t 0.610 (0.606)] [data_t 0.003] [optim_t 0.607] [lr 0.005000] +2024-04-06 03:36:34,999 - Train: 23.42% [445500/1902000] [234.2/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 03:37:35,645 - Train: 23.43% [445600/1902000] [234.3/1000.0] [batch_t 0.612 (0.606)] [data_t 0.003] [optim_t 0.610] [lr 0.005000] +2024-04-06 03:38:36,400 - Train: 23.43% [445700/1902000] [234.3/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 03:39:37,070 - Train: 23.44% [445800/1902000] [234.4/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:40:37,698 - Train: 23.44% [445900/1902000] [234.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 03:41:38,547 - Train: 23.45% [446000/1902000] [234.5/1000.0] [batch_t 0.608 (0.608)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 03:42:39,843 - Train: 23.45% [446100/1902000] [234.5/1000.0] [batch_t 0.604 (0.613)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 03:43:41,458 - Train: 23.46% [446200/1902000] [234.6/1000.0] [batch_t 0.606 (0.616)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:44:42,046 - Train: 23.46% [446300/1902000] [234.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 03:45:43,018 - Train: 23.47% [446400/1902000] [234.7/1000.0] [batch_t 0.607 (0.610)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 03:46:44,944 - Train: 23.48% [446500/1902000] [234.8/1000.0] [batch_t 0.607 (0.619)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 03:47:45,604 - Train: 23.48% [446600/1902000] [234.8/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 03:48:46,269 - Train: 23.49% [446700/1902000] [234.9/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 03:49:47,042 - Train: 23.49% [446800/1902000] [234.9/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 03:50:48,477 - Train: 23.50% [446900/1902000] [235.0/1000.0] [batch_t 0.607 (0.614)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 03:51:30,920 - ==> Total time: 4 days, 2:51:10 Eta: 13 days, 9:47:51 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 03:51:51,634 - Train: 23.50% [447000/1902000] [235.0/1000.0] [batch_t 0.607 (0.609)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 03:52:52,780 - Train: 23.51% [447100/1902000] [235.1/1000.0] [batch_t 0.606 (0.611)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:53:53,872 - Train: 23.51% [447200/1902000] [235.1/1000.0] [batch_t 0.608 (0.611)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-06 03:54:55,171 - Train: 23.52% [447300/1902000] [235.2/1000.0] [batch_t 0.605 (0.613)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 03:55:55,773 - Train: 23.52% [447400/1902000] [235.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:56:56,376 - Train: 23.53% [447500/1902000] [235.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 03:57:56,974 - Train: 23.53% [447600/1902000] [235.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 03:58:57,599 - Train: 23.54% [447700/1902000] [235.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 03:59:58,484 - Train: 23.54% [447800/1902000] [235.4/1000.0] [batch_t 0.608 (0.609)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 04:00:59,134 - Train: 23.55% [447900/1902000] [235.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:01:59,917 - Train: 23.55% [448000/1902000] [235.5/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:03:00,659 - Train: 23.56% [448100/1902000] [235.6/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 04:04:02,149 - Train: 23.56% [448200/1902000] [235.6/1000.0] [batch_t 0.607 (0.615)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 04:05:02,835 - Train: 23.57% [448300/1902000] [235.7/1000.0] [batch_t 0.606 (0.607)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:06:03,456 - Train: 23.58% [448400/1902000] [235.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:07:04,116 - Train: 23.58% [448500/1902000] [235.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:08:04,916 - Train: 23.59% [448600/1902000] [235.9/1000.0] [batch_t 0.607 (0.608)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:09:05,571 - Train: 23.59% [448700/1902000] [235.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:10:06,248 - Train: 23.60% [448800/1902000] [236.0/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 04:10:49,931 - ==> Total time: 4 days, 3:10:29 Eta: 13 days, 9:03:27 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 04:11:09,003 - Train: 23.60% [448900/1902000] [236.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:12:10,180 - Train: 23.61% [449000/1902000] [236.1/1000.0] [batch_t 0.608 (0.612)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 04:13:10,830 - Train: 23.61% [449100/1902000] [236.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:14:12,208 - Train: 23.62% [449200/1902000] [236.2/1000.0] [batch_t 0.606 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:15:12,838 - Train: 23.62% [449300/1902000] [236.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 04:16:13,500 - Train: 23.63% [449400/1902000] [236.3/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:17:14,124 - Train: 23.63% [449500/1902000] [236.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 04:18:14,789 - Train: 23.64% [449600/1902000] [236.4/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 04:19:15,442 - Train: 23.64% [449700/1902000] [236.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:20:16,732 - Train: 23.65% [449800/1902000] [236.5/1000.0] [batch_t 0.604 (0.613)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 04:21:17,479 - Train: 23.65% [449900/1902000] [236.5/1000.0] [batch_t 0.610 (0.607)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-06 04:22:19,413 - Train: 23.66% [450000/1902000] [236.6/1000.0] [batch_t 0.605 (0.619)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:23:20,022 - Train: 23.66% [450100/1902000] [236.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:24:20,658 - Train: 23.67% [450200/1902000] [236.7/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 04:25:21,296 - Train: 23.68% [450300/1902000] [236.8/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 04:26:21,956 - Train: 23.68% [450400/1902000] [236.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:27:22,708 - Train: 23.69% [450500/1902000] [236.9/1000.0] [batch_t 0.608 (0.607)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-06 04:28:23,341 - Train: 23.69% [450600/1902000] [236.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 04:29:23,983 - Train: 23.70% [450700/1902000] [237.0/1000.0] [batch_t 0.610 (0.606)] [data_t 0.003] [optim_t 0.607] [lr 0.005000] +2024-04-06 04:30:08,908 - ==> Total time: 4 days, 3:29:48 Eta: 13 days, 8:19:15 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 04:30:26,846 - Train: 23.70% [450800/1902000] [237.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 04:31:27,469 - Train: 23.71% [450900/1902000] [237.1/1000.0] [batch_t 0.608 (0.606)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-06 04:32:28,109 - Train: 23.71% [451000/1902000] [237.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:33:28,759 - Train: 23.72% [451100/1902000] [237.2/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 04:34:29,543 - Train: 23.72% [451200/1902000] [237.2/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 04:35:30,188 - Train: 23.73% [451300/1902000] [237.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 04:36:30,815 - Train: 23.73% [451400/1902000] [237.3/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 04:37:31,407 - Train: 23.74% [451500/1902000] [237.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:38:32,014 - Train: 23.74% [451600/1902000] [237.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:39:32,707 - Train: 23.75% [451700/1902000] [237.5/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 04:40:33,318 - Train: 23.75% [451800/1902000] [237.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:41:33,943 - Train: 23.76% [451900/1902000] [237.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:42:34,580 - Train: 23.76% [452000/1902000] [237.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:43:35,223 - Train: 23.77% [452100/1902000] [237.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:44:35,829 - Train: 23.77% [452200/1902000] [237.7/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 04:45:36,440 - Train: 23.78% [452300/1902000] [237.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 04:46:37,130 - Train: 23.79% [452400/1902000] [237.9/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:47:37,734 - Train: 23.79% [452500/1902000] [237.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:48:38,364 - Train: 23.80% [452600/1902000] [238.0/1000.0] [batch_t 0.614 (0.606)] [data_t 0.002] [optim_t 0.612] [lr 0.005000] +2024-04-06 04:49:24,434 - ==> Total time: 4 days, 3:49:03 Eta: 13 days, 7:35:04 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 04:49:41,339 - Train: 23.80% [452700/1902000] [238.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:50:41,949 - Train: 23.81% [452800/1902000] [238.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 04:51:42,534 - Train: 23.81% [452900/1902000] [238.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 04:52:43,227 - Train: 23.82% [453000/1902000] [238.2/1000.0] [batch_t 0.606 (0.607)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:53:43,830 - Train: 23.82% [453100/1902000] [238.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 04:54:44,448 - Train: 23.83% [453200/1902000] [238.3/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 04:55:45,104 - Train: 23.83% [453300/1902000] [238.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 04:56:45,760 - Train: 23.84% [453400/1902000] [238.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 04:57:46,368 - Train: 23.84% [453500/1902000] [238.4/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 04:58:46,990 - Train: 23.85% [453600/1902000] [238.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 04:59:47,711 - Train: 23.85% [453700/1902000] [238.5/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 05:00:48,392 - Train: 23.86% [453800/1902000] [238.6/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 05:01:49,042 - Train: 23.86% [453900/1902000] [238.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:02:49,669 - Train: 23.87% [454000/1902000] [238.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:03:50,298 - Train: 23.87% [454100/1902000] [238.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 05:04:51,024 - Train: 23.88% [454200/1902000] [238.8/1000.0] [batch_t 0.608 (0.607)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-06 05:05:51,672 - Train: 23.89% [454300/1902000] [238.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:06:52,303 - Train: 23.89% [454400/1902000] [238.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 05:07:52,935 - Train: 23.90% [454500/1902000] [239.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 05:08:40,239 - ==> Total time: 4 days, 4:08:19 Eta: 13 days, 6:51:07 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 05:08:55,576 - Train: 23.90% [454600/1902000] [239.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:09:56,230 - Train: 23.91% [454700/1902000] [239.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 05:10:57,032 - Train: 23.91% [454800/1902000] [239.1/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:11:57,642 - Train: 23.92% [454900/1902000] [239.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 05:12:58,233 - Train: 23.92% [455000/1902000] [239.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 05:13:58,779 - Train: 23.93% [455100/1902000] [239.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 05:14:59,627 - Train: 23.93% [455200/1902000] [239.3/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:16:00,171 - Train: 23.94% [455300/1902000] [239.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:17:00,707 - Train: 23.94% [455400/1902000] [239.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 05:18:01,265 - Train: 23.95% [455500/1902000] [239.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 05:19:01,902 - Train: 23.95% [455600/1902000] [239.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 05:20:02,833 - Train: 23.96% [455700/1902000] [239.6/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:21:03,412 - Train: 23.96% [455800/1902000] [239.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 05:22:03,987 - Train: 23.97% [455900/1902000] [239.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 05:23:05,602 - Train: 23.97% [456000/1902000] [239.7/1000.0] [batch_t 0.607 (0.616)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 05:24:06,383 - Train: 23.98% [456100/1902000] [239.8/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:25:06,940 - Train: 23.99% [456200/1902000] [239.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 05:26:07,514 - Train: 23.99% [456300/1902000] [239.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:27:08,055 - Train: 24.00% [456400/1902000] [240.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:27:56,547 - ==> Total time: 4 days, 4:27:36 Eta: 13 days, 6:07:24 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 05:28:10,733 - Train: 24.00% [456500/1902000] [240.0/1000.0] [batch_t 0.612 (0.606)] [data_t 0.002] [optim_t 0.610] [lr 0.005000] +2024-04-06 05:29:11,320 - Train: 24.01% [456600/1902000] [240.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 05:30:12,012 - Train: 24.01% [456700/1902000] [240.1/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 05:31:12,943 - Train: 24.02% [456800/1902000] [240.2/1000.0] [batch_t 0.607 (0.609)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 05:32:13,475 - Train: 24.02% [456900/1902000] [240.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:33:14,039 - Train: 24.03% [457000/1902000] [240.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 05:34:15,572 - Train: 24.03% [457100/1902000] [240.3/1000.0] [batch_t 0.609 (0.615)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 05:35:16,627 - Train: 24.04% [457200/1902000] [240.4/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 05:36:17,192 - Train: 24.04% [457300/1902000] [240.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:37:17,777 - Train: 24.05% [457400/1902000] [240.5/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-06 05:38:18,436 - Train: 24.05% [457500/1902000] [240.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 05:39:18,976 - Train: 24.06% [457600/1902000] [240.6/1000.0] [batch_t 0.599 (0.605)] [data_t 0.002] [optim_t 0.597] [lr 0.005000] +2024-04-06 05:40:19,531 - Train: 24.06% [457700/1902000] [240.6/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 05:41:20,078 - Train: 24.07% [457800/1902000] [240.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:42:20,644 - Train: 24.07% [457900/1902000] [240.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:43:21,579 - Train: 24.08% [458000/1902000] [240.8/1000.0] [batch_t 0.608 (0.609)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 05:44:22,172 - Train: 24.09% [458100/1902000] [240.9/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 05:45:22,733 - Train: 24.09% [458200/1902000] [240.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 05:46:23,564 - Train: 24.10% [458300/1902000] [241.0/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 05:47:13,562 - ==> Total time: 4 days, 4:46:53 Eta: 13 days, 5:23:55 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 05:47:26,890 - Train: 24.10% [458400/1902000] [241.0/1000.0] [batch_t 0.605 (0.617)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 05:48:27,434 - Train: 24.11% [458500/1902000] [241.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 05:49:28,920 - Train: 24.11% [458600/1902000] [241.1/1000.0] [batch_t 0.606 (0.615)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 05:50:29,633 - Train: 24.12% [458700/1902000] [241.2/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 05:51:30,237 - Train: 24.12% [458800/1902000] [241.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:52:30,791 - Train: 24.13% [458900/1902000] [241.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 05:53:31,335 - Train: 24.13% [459000/1902000] [241.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:54:31,884 - Train: 24.14% [459100/1902000] [241.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 05:55:32,574 - Train: 24.14% [459200/1902000] [241.4/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 05:56:33,590 - Train: 24.15% [459300/1902000] [241.5/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 05:57:34,139 - Train: 24.15% [459400/1902000] [241.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 05:58:35,732 - Train: 24.16% [459500/1902000] [241.6/1000.0] [batch_t 0.606 (0.616)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 05:59:36,300 - Train: 24.16% [459600/1902000] [241.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 06:00:36,900 - Train: 24.17% [459700/1902000] [241.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 06:01:37,461 - Train: 24.17% [459800/1902000] [241.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 06:02:38,096 - Train: 24.18% [459900/1902000] [241.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:03:38,746 - Train: 24.19% [460000/1902000] [241.9/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 06:04:39,286 - Train: 24.19% [460100/1902000] [241.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 06:05:39,836 - Train: 24.20% [460200/1902000] [242.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:06:30,694 - ==> Total time: 4 days, 5:06:10 Eta: 13 days, 4:40:38 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 06:06:42,631 - Train: 24.20% [460300/1902000] [242.0/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 06:07:43,200 - Train: 24.21% [460400/1902000] [242.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 06:08:43,794 - Train: 24.21% [460500/1902000] [242.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 06:09:44,399 - Train: 24.22% [460600/1902000] [242.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:10:45,093 - Train: 24.22% [460700/1902000] [242.2/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 06:11:45,765 - Train: 24.23% [460800/1902000] [242.3/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 06:12:46,293 - Train: 24.23% [460900/1902000] [242.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 06:13:46,822 - Train: 24.24% [461000/1902000] [242.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 06:14:47,382 - Train: 24.24% [461100/1902000] [242.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:15:48,035 - Train: 24.25% [461200/1902000] [242.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:16:48,564 - Train: 24.25% [461300/1902000] [242.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 06:17:49,167 - Train: 24.26% [461400/1902000] [242.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:18:49,694 - Train: 24.26% [461500/1902000] [242.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:19:50,379 - Train: 24.27% [461600/1902000] [242.7/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 06:20:50,920 - Train: 24.27% [461700/1902000] [242.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:21:51,454 - Train: 24.28% [461800/1902000] [242.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:22:51,992 - Train: 24.28% [461900/1902000] [242.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 06:23:52,595 - Train: 24.29% [462000/1902000] [242.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:24:53,209 - Train: 24.30% [462100/1902000] [243.0/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 06:25:45,279 - ==> Total time: 4 days, 5:25:24 Eta: 13 days, 3:57:26 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 06:25:56,155 - Train: 24.30% [462200/1902000] [243.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 06:26:56,698 - Train: 24.31% [462300/1902000] [243.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:27:57,309 - Train: 24.31% [462400/1902000] [243.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 06:28:57,833 - Train: 24.32% [462500/1902000] [243.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 06:29:58,484 - Train: 24.32% [462600/1902000] [243.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:30:59,026 - Train: 24.33% [462700/1902000] [243.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 06:31:59,581 - Train: 24.33% [462800/1902000] [243.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 06:33:00,102 - Train: 24.34% [462900/1902000] [243.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:34:00,692 - Train: 24.34% [463000/1902000] [243.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 06:35:01,318 - Train: 24.35% [463100/1902000] [243.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:36:01,898 - Train: 24.35% [463200/1902000] [243.5/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 06:37:02,468 - Train: 24.36% [463300/1902000] [243.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:38:03,052 - Train: 24.36% [463400/1902000] [243.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 06:39:03,631 - Train: 24.37% [463500/1902000] [243.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:40:04,193 - Train: 24.37% [463600/1902000] [243.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:41:04,731 - Train: 24.38% [463700/1902000] [243.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 06:42:05,351 - Train: 24.38% [463800/1902000] [243.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 06:43:06,899 - Train: 24.39% [463900/1902000] [243.9/1000.0] [batch_t 0.604 (0.615)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 06:44:07,487 - Train: 24.40% [464000/1902000] [244.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 06:45:01,198 - ==> Total time: 4 days, 5:44:40 Eta: 13 days, 3:14:29 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 06:45:13,088 - Train: 24.40% [464100/1902000] [244.0/1000.0] [batch_t 0.603 (0.660)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 06:46:13,659 - Train: 24.41% [464200/1902000] [244.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:47:14,198 - Train: 24.41% [464300/1902000] [244.1/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-06 06:48:14,735 - Train: 24.42% [464400/1902000] [244.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 06:49:15,458 - Train: 24.42% [464500/1902000] [244.2/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 06:50:16,382 - Train: 24.43% [464600/1902000] [244.3/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 06:51:16,951 - Train: 24.43% [464700/1902000] [244.3/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 06:52:17,543 - Train: 24.44% [464800/1902000] [244.4/1000.0] [batch_t 0.613 (0.606)] [data_t 0.002] [optim_t 0.611] [lr 0.005000] +2024-04-06 06:53:18,773 - Train: 24.44% [464900/1902000] [244.4/1000.0] [batch_t 0.608 (0.612)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 06:54:19,358 - Train: 24.45% [465000/1902000] [244.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:55:20,124 - Train: 24.45% [465100/1902000] [244.5/1000.0] [batch_t 0.603 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 06:56:20,701 - Train: 24.46% [465200/1902000] [244.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 06:57:22,032 - Train: 24.46% [465300/1902000] [244.6/1000.0] [batch_t 0.605 (0.613)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 06:58:22,597 - Train: 24.47% [465400/1902000] [244.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 06:59:23,190 - Train: 24.47% [465500/1902000] [244.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 07:00:23,933 - Train: 24.48% [465600/1902000] [244.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 07:01:24,489 - Train: 24.48% [465700/1902000] [244.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 07:02:26,215 - Train: 24.49% [465800/1902000] [244.9/1000.0] [batch_t 0.607 (0.617)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 07:03:26,807 - Train: 24.50% [465900/1902000] [245.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 07:04:21,322 - ==> Total time: 4 days, 6:04:00 Eta: 13 days, 2:31:57 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 07:04:29,555 - Train: 24.50% [466000/1902000] [245.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:05:30,090 - Train: 24.51% [466100/1902000] [245.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:06:30,808 - Train: 24.51% [466200/1902000] [245.1/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 07:07:31,555 - Train: 24.52% [466300/1902000] [245.2/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 07:08:32,139 - Train: 24.52% [466400/1902000] [245.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 07:09:32,770 - Train: 24.53% [466500/1902000] [245.3/1000.0] [batch_t 0.611 (0.606)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-06 07:10:33,370 - Train: 24.53% [466600/1902000] [245.3/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-06 07:11:34,495 - Train: 24.54% [466700/1902000] [245.4/1000.0] [batch_t 0.607 (0.611)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 07:12:35,549 - Train: 24.54% [466800/1902000] [245.4/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 07:13:36,143 - Train: 24.55% [466900/1902000] [245.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 07:14:36,711 - Train: 24.55% [467000/1902000] [245.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:15:37,435 - Train: 24.56% [467100/1902000] [245.6/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 07:16:38,413 - Train: 24.56% [467200/1902000] [245.6/1000.0] [batch_t 0.603 (0.610)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 07:17:38,979 - Train: 24.57% [467300/1902000] [245.7/1000.0] [batch_t 0.613 (0.606)] [data_t 0.002] [optim_t 0.611] [lr 0.005000] +2024-04-06 07:18:39,536 - Train: 24.57% [467400/1902000] [245.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 07:19:40,114 - Train: 24.58% [467500/1902000] [245.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 07:20:41,215 - Train: 24.58% [467600/1902000] [245.8/1000.0] [batch_t 0.607 (0.611)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 07:21:41,850 - Train: 24.59% [467700/1902000] [245.9/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 07:22:42,733 - Train: 24.60% [467800/1902000] [246.0/1000.0] [batch_t 0.604 (0.609)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 07:23:38,447 - ==> Total time: 4 days, 6:23:17 Eta: 13 days, 1:49:27 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 07:23:45,902 - Train: 24.60% [467900/1902000] [246.0/1000.0] [batch_t 0.607 (0.619)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 07:24:46,435 - Train: 24.61% [468000/1902000] [246.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 07:25:46,992 - Train: 24.61% [468100/1902000] [246.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 07:26:47,663 - Train: 24.62% [468200/1902000] [246.2/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:27:48,199 - Train: 24.62% [468300/1902000] [246.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:28:48,733 - Train: 24.63% [468400/1902000] [246.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 07:29:49,306 - Train: 24.63% [468500/1902000] [246.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 07:30:49,875 - Train: 24.64% [468600/1902000] [246.4/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 07:31:50,441 - Train: 24.64% [468700/1902000] [246.4/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 07:32:51,009 - Train: 24.65% [468800/1902000] [246.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:33:51,665 - Train: 24.65% [468900/1902000] [246.5/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 07:34:52,222 - Train: 24.66% [469000/1902000] [246.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:35:52,806 - Train: 24.66% [469100/1902000] [246.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 07:36:53,365 - Train: 24.67% [469200/1902000] [246.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:37:53,945 - Train: 24.67% [469300/1902000] [246.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:38:54,770 - Train: 24.68% [469400/1902000] [246.8/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 07:39:55,423 - Train: 24.68% [469500/1902000] [246.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:40:55,995 - Train: 24.69% [469600/1902000] [246.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:41:56,583 - Train: 24.70% [469700/1902000] [247.0/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 07:42:53,525 - ==> Total time: 4 days, 6:42:33 Eta: 13 days, 1:07:02 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 07:42:59,660 - Train: 24.70% [469800/1902000] [247.0/1000.0] [batch_t 0.606 (0.621)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:44:00,257 - Train: 24.71% [469900/1902000] [247.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 07:45:00,812 - Train: 24.71% [470000/1902000] [247.1/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 07:46:01,512 - Train: 24.72% [470100/1902000] [247.2/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:47:02,641 - Train: 24.72% [470200/1902000] [247.2/1000.0] [batch_t 0.604 (0.611)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 07:48:03,197 - Train: 24.73% [470300/1902000] [247.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 07:49:03,741 - Train: 24.73% [470400/1902000] [247.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 07:50:04,289 - Train: 24.74% [470500/1902000] [247.4/1000.0] [batch_t 0.611 (0.605)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-06 07:51:04,894 - Train: 24.74% [470600/1902000] [247.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 07:52:05,446 - Train: 24.75% [470700/1902000] [247.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 07:53:06,077 - Train: 24.75% [470800/1902000] [247.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:54:06,653 - Train: 24.76% [470900/1902000] [247.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:55:07,237 - Train: 24.76% [471000/1902000] [247.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:56:07,868 - Train: 24.77% [471100/1902000] [247.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 07:57:08,444 - Train: 24.77% [471200/1902000] [247.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 07:58:09,203 - Train: 24.78% [471300/1902000] [247.8/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 07:59:09,875 - Train: 24.78% [471400/1902000] [247.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 08:00:10,438 - Train: 24.79% [471500/1902000] [247.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 08:01:11,033 - Train: 24.79% [471600/1902000] [247.9/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 08:02:09,226 - ==> Total time: 4 days, 7:01:48 Eta: 13 days, 0:24:51 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 08:02:13,845 - Train: 24.80% [471700/1902000] [248.0/1000.0] [batch_t 0.603 (0.688)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 08:03:14,416 - Train: 24.81% [471800/1902000] [248.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 08:04:15,417 - Train: 24.81% [471900/1902000] [248.1/1000.0] [batch_t 0.603 (0.610)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 08:05:15,962 - Train: 24.82% [472000/1902000] [248.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 08:06:16,632 - Train: 24.82% [472100/1902000] [248.2/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 08:07:17,178 - Train: 24.83% [472200/1902000] [248.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:08:17,730 - Train: 24.83% [472300/1902000] [248.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 08:09:18,286 - Train: 24.84% [472400/1902000] [248.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 08:10:18,835 - Train: 24.84% [472500/1902000] [248.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:11:19,442 - Train: 24.85% [472600/1902000] [248.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:12:20,019 - Train: 24.85% [472700/1902000] [248.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:13:20,562 - Train: 24.86% [472800/1902000] [248.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:14:21,131 - Train: 24.86% [472900/1902000] [248.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:15:21,699 - Train: 24.87% [473000/1902000] [248.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:16:22,248 - Train: 24.87% [473100/1902000] [248.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 08:17:22,816 - Train: 24.88% [473200/1902000] [248.8/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 08:18:23,432 - Train: 24.88% [473300/1902000] [248.8/1000.0] [batch_t 0.675 (0.606)] [data_t 0.002] [optim_t 0.674] [lr 0.005000] +2024-04-06 08:19:23,971 - Train: 24.89% [473400/1902000] [248.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:20:24,502 - Train: 24.89% [473500/1902000] [248.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:21:23,839 - ==> Total time: 4 days, 7:21:03 Eta: 12 days, 23:42:46 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 08:21:27,913 - Train: 24.90% [473600/1902000] [249.0/1000.0] [batch_t 0.607 (0.801)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 08:22:28,504 - Train: 24.91% [473700/1902000] [249.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 08:23:29,204 - Train: 24.91% [473800/1902000] [249.1/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 08:24:29,736 - Train: 24.92% [473900/1902000] [249.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 08:25:30,269 - Train: 24.92% [474000/1902000] [249.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:26:30,815 - Train: 24.93% [474100/1902000] [249.3/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 08:27:31,355 - Train: 24.93% [474200/1902000] [249.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:28:31,899 - Train: 24.94% [474300/1902000] [249.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:29:32,475 - Train: 24.94% [474400/1902000] [249.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 08:30:33,062 - Train: 24.95% [474500/1902000] [249.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 08:31:33,701 - Train: 24.95% [474600/1902000] [249.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:32:34,249 - Train: 24.96% [474700/1902000] [249.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:33:34,819 - Train: 24.96% [474800/1902000] [249.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 08:34:35,355 - Train: 24.97% [474900/1902000] [249.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 08:35:35,910 - Train: 24.97% [475000/1902000] [249.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:36:36,562 - Train: 24.98% [475100/1902000] [249.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:37:37,105 - Train: 24.98% [475200/1902000] [249.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 08:38:38,014 - Train: 24.99% [475300/1902000] [249.9/1000.0] [batch_t 0.604 (0.609)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 08:39:38,609 - Train: 24.99% [475400/1902000] [249.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 08:40:39,213 - Train: 25.00% [475500/1902000] [250.0/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 08:40:39,216 - ==> Total time: 4 days, 7:40:18 Eta: 12 days, 23:00:56 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 08:41:41,879 - Train: 25.01% [475600/1902000] [250.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:42:42,556 - Train: 25.01% [475700/1902000] [250.1/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 08:43:43,125 - Train: 25.02% [475800/1902000] [250.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 08:44:43,684 - Train: 25.02% [475900/1902000] [250.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:45:44,227 - Train: 25.03% [476000/1902000] [250.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:46:44,764 - Train: 25.03% [476100/1902000] [250.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 08:47:45,287 - Train: 25.04% [476200/1902000] [250.4/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 08:48:45,841 - Train: 25.04% [476300/1902000] [250.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 08:49:46,403 - Train: 25.05% [476400/1902000] [250.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:50:47,750 - Train: 25.05% [476500/1902000] [250.5/1000.0] [batch_t 0.605 (0.613)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:51:48,767 - Train: 25.06% [476600/1902000] [250.6/1000.0] [batch_t 0.608 (0.610)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 08:52:49,284 - Train: 25.06% [476700/1902000] [250.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 08:53:49,825 - Train: 25.07% [476800/1902000] [250.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 08:54:50,364 - Train: 25.07% [476900/1902000] [250.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:55:51,055 - Train: 25.08% [477000/1902000] [250.8/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 08:56:51,646 - Train: 25.08% [477100/1902000] [250.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 08:57:52,201 - Train: 25.09% [477200/1902000] [250.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 08:58:52,760 - Train: 25.09% [477300/1902000] [250.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:59:53,346 - Train: 25.10% [477400/1902000] [251.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 08:59:54,563 - ==> Total time: 4 days, 7:59:34 Eta: 12 days, 22:19:16 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 09:00:56,989 - Train: 25.11% [477500/1902000] [251.1/1000.0] [batch_t 0.606 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 09:01:57,570 - Train: 25.11% [477600/1902000] [251.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 09:02:58,325 - Train: 25.12% [477700/1902000] [251.2/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 09:03:58,863 - Train: 25.12% [477800/1902000] [251.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:04:59,409 - Train: 25.13% [477900/1902000] [251.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:05:59,972 - Train: 25.13% [478000/1902000] [251.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:07:00,511 - Train: 25.14% [478100/1902000] [251.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 09:08:02,757 - Train: 25.14% [478200/1902000] [251.4/1000.0] [batch_t 0.606 (0.622)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 09:09:03,295 - Train: 25.15% [478300/1902000] [251.5/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 09:10:03,858 - Train: 25.15% [478400/1902000] [251.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 09:11:04,424 - Train: 25.16% [478500/1902000] [251.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 09:12:04,994 - Train: 25.16% [478600/1902000] [251.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:13:07,170 - Train: 25.17% [478700/1902000] [251.7/1000.0] [batch_t 0.607 (0.622)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 09:14:07,737 - Train: 25.17% [478800/1902000] [251.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 09:15:08,348 - Train: 25.18% [478900/1902000] [251.8/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 09:16:09,082 - Train: 25.18% [479000/1902000] [251.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:17:10,725 - Train: 25.19% [479100/1902000] [251.9/1000.0] [batch_t 0.608 (0.616)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 09:18:11,312 - Train: 25.19% [479200/1902000] [251.9/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 09:19:11,873 - Train: 25.20% [479300/1902000] [252.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:19:14,296 - ==> Total time: 4 days, 8:18:53 Eta: 12 days, 21:37:59 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 09:20:14,461 - Train: 25.21% [479400/1902000] [252.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 09:21:15,166 - Train: 25.21% [479500/1902000] [252.1/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:22:15,725 - Train: 25.22% [479600/1902000] [252.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 09:23:16,316 - Train: 25.22% [479700/1902000] [252.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 09:24:16,883 - Train: 25.23% [479800/1902000] [252.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 09:25:17,440 - Train: 25.23% [479900/1902000] [252.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 09:26:18,018 - Train: 25.24% [480000/1902000] [252.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 09:27:18,584 - Train: 25.24% [480100/1902000] [252.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:28:19,117 - Train: 25.25% [480200/1902000] [252.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:29:19,741 - Train: 25.25% [480300/1902000] [252.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 09:30:20,281 - Train: 25.26% [480400/1902000] [252.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 09:31:20,839 - Train: 25.26% [480500/1902000] [252.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 09:32:21,372 - Train: 25.27% [480600/1902000] [252.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 09:33:21,931 - Train: 25.27% [480700/1902000] [252.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 09:34:22,819 - Train: 25.28% [480800/1902000] [252.8/1000.0] [batch_t 0.607 (0.609)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 09:35:23,355 - Train: 25.28% [480900/1902000] [252.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:36:23,913 - Train: 25.29% [481000/1902000] [252.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:37:24,472 - Train: 25.29% [481100/1902000] [252.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 09:38:25,010 - Train: 25.30% [481200/1902000] [253.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 09:38:28,645 - ==> Total time: 4 days, 8:38:08 Eta: 12 days, 20:56:37 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 09:39:27,597 - Train: 25.30% [481300/1902000] [253.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 09:40:28,282 - Train: 25.31% [481400/1902000] [253.1/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 09:41:28,847 - Train: 25.32% [481500/1902000] [253.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:42:29,372 - Train: 25.32% [481600/1902000] [253.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 09:43:29,952 - Train: 25.33% [481700/1902000] [253.3/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 09:44:30,530 - Train: 25.33% [481800/1902000] [253.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:45:31,078 - Train: 25.34% [481900/1902000] [253.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 09:46:31,687 - Train: 25.34% [482000/1902000] [253.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 09:47:32,350 - Train: 25.35% [482100/1902000] [253.5/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:48:32,950 - Train: 25.35% [482200/1902000] [253.5/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 09:49:33,513 - Train: 25.36% [482300/1902000] [253.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 09:50:34,291 - Train: 25.36% [482400/1902000] [253.6/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 09:51:34,879 - Train: 25.37% [482500/1902000] [253.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:52:35,414 - Train: 25.37% [482600/1902000] [253.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 09:53:36,072 - Train: 25.38% [482700/1902000] [253.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 09:54:36,627 - Train: 25.38% [482800/1902000] [253.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 09:55:37,155 - Train: 25.39% [482900/1902000] [253.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 09:56:37,676 - Train: 25.39% [483000/1902000] [253.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 09:57:38,213 - Train: 25.40% [483100/1902000] [254.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 09:57:43,059 - ==> Total time: 4 days, 8:57:22 Eta: 12 days, 20:15:26 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 09:58:40,849 - Train: 25.40% [483200/1902000] [254.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 09:59:41,529 - Train: 25.41% [483300/1902000] [254.1/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 10:00:42,073 - Train: 25.42% [483400/1902000] [254.2/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 10:01:42,637 - Train: 25.42% [483500/1902000] [254.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 10:02:43,190 - Train: 25.43% [483600/1902000] [254.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 10:03:43,982 - Train: 25.43% [483700/1902000] [254.3/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 10:04:44,559 - Train: 25.44% [483800/1902000] [254.4/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-06 10:05:45,111 - Train: 25.44% [483900/1902000] [254.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:06:45,681 - Train: 25.45% [484000/1902000] [254.5/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 10:07:46,860 - Train: 25.45% [484100/1902000] [254.5/1000.0] [batch_t 0.605 (0.612)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:08:47,398 - Train: 25.46% [484200/1902000] [254.6/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 10:09:47,973 - Train: 25.46% [484300/1902000] [254.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:10:48,509 - Train: 25.47% [484400/1902000] [254.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 10:11:49,055 - Train: 25.47% [484500/1902000] [254.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 10:12:50,351 - Train: 25.48% [484600/1902000] [254.8/1000.0] [batch_t 0.606 (0.613)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:13:50,900 - Train: 25.48% [484700/1902000] [254.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:14:51,446 - Train: 25.49% [484800/1902000] [254.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 10:15:52,016 - Train: 25.49% [484900/1902000] [254.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 10:16:53,020 - Train: 25.50% [485000/1902000] [255.0/1000.0] [batch_t 0.607 (0.610)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 10:16:59,078 - ==> Total time: 4 days, 9:16:38 Eta: 12 days, 19:34:30 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 10:17:56,229 - Train: 25.50% [485100/1902000] [255.0/1000.0] [batch_t 0.607 (0.611)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 10:18:56,941 - Train: 25.51% [485200/1902000] [255.1/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 10:19:57,820 - Train: 25.52% [485300/1902000] [255.2/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 10:20:58,594 - Train: 25.52% [485400/1902000] [255.2/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 10:21:59,523 - Train: 25.53% [485500/1902000] [255.3/1000.0] [batch_t 0.604 (0.609)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 10:23:00,118 - Train: 25.53% [485600/1902000] [255.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 10:24:00,729 - Train: 25.54% [485700/1902000] [255.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 10:25:01,306 - Train: 25.54% [485800/1902000] [255.4/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 10:26:02,034 - Train: 25.55% [485900/1902000] [255.5/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 10:27:02,607 - Train: 25.55% [486000/1902000] [255.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:28:03,153 - Train: 25.56% [486100/1902000] [255.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:29:04,685 - Train: 25.56% [486200/1902000] [255.6/1000.0] [batch_t 0.607 (0.615)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 10:30:05,694 - Train: 25.57% [486300/1902000] [255.7/1000.0] [batch_t 0.607 (0.610)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 10:31:06,251 - Train: 25.57% [486400/1902000] [255.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:32:06,989 - Train: 25.58% [486500/1902000] [255.8/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:33:07,564 - Train: 25.58% [486600/1902000] [255.8/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-06 10:34:08,100 - Train: 25.59% [486700/1902000] [255.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 10:35:08,657 - Train: 25.59% [486800/1902000] [255.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 10:36:09,251 - Train: 25.60% [486900/1902000] [256.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 10:36:16,524 - ==> Total time: 4 days, 9:35:56 Eta: 12 days, 18:53:48 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 10:37:11,939 - Train: 25.60% [487000/1902000] [256.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:38:12,562 - Train: 25.61% [487100/1902000] [256.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 10:39:13,260 - Train: 25.62% [487200/1902000] [256.2/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:40:13,847 - Train: 25.62% [487300/1902000] [256.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:41:14,425 - Train: 25.63% [487400/1902000] [256.3/1000.0] [batch_t 0.611 (0.606)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-06 10:42:15,038 - Train: 25.63% [487500/1902000] [256.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:43:15,587 - Train: 25.64% [487600/1902000] [256.4/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 10:44:16,216 - Train: 25.64% [487700/1902000] [256.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:45:16,786 - Train: 25.65% [487800/1902000] [256.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 10:46:17,337 - Train: 25.65% [487900/1902000] [256.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-06 10:47:17,939 - Train: 25.66% [488000/1902000] [256.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 10:48:18,493 - Train: 25.66% [488100/1902000] [256.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:49:19,053 - Train: 25.67% [488200/1902000] [256.7/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-06 10:50:19,607 - Train: 25.67% [488300/1902000] [256.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 10:51:20,151 - Train: 25.68% [488400/1902000] [256.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 10:52:20,784 - Train: 25.68% [488500/1902000] [256.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 10:53:21,368 - Train: 25.69% [488600/1902000] [256.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:54:21,929 - Train: 25.69% [488700/1902000] [256.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 10:55:22,506 - Train: 25.70% [488800/1902000] [257.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 10:55:30,984 - ==> Total time: 4 days, 9:55:10 Eta: 12 days, 18:13:07 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 10:56:25,414 - Train: 25.70% [488900/1902000] [257.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 10:57:26,105 - Train: 25.71% [489000/1902000] [257.1/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 10:58:26,683 - Train: 25.72% [489100/1902000] [257.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 10:59:27,229 - Train: 25.72% [489200/1902000] [257.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 11:00:27,817 - Train: 25.73% [489300/1902000] [257.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 11:01:28,362 - Train: 25.73% [489400/1902000] [257.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:02:29,051 - Train: 25.74% [489500/1902000] [257.4/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 11:03:29,594 - Train: 25.74% [489600/1902000] [257.4/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 11:04:31,017 - Train: 25.75% [489700/1902000] [257.5/1000.0] [batch_t 0.603 (0.614)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 11:05:31,791 - Train: 25.75% [489800/1902000] [257.5/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:06:32,410 - Train: 25.76% [489900/1902000] [257.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:07:33,123 - Train: 25.76% [490000/1902000] [257.6/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:08:33,669 - Train: 25.77% [490100/1902000] [257.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 11:09:34,409 - Train: 25.77% [490200/1902000] [257.7/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 11:10:35,044 - Train: 25.78% [490300/1902000] [257.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:11:36,910 - Train: 25.78% [490400/1902000] [257.8/1000.0] [batch_t 0.607 (0.619)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 11:12:37,556 - Train: 25.79% [490500/1902000] [257.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 11:13:38,692 - Train: 25.79% [490600/1902000] [257.9/1000.0] [batch_t 0.605 (0.611)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:14:39,358 - Train: 25.80% [490700/1902000] [258.0/1000.0] [batch_t 0.610 (0.607)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-06 11:14:49,047 - ==> Total time: 4 days, 10:14:28 Eta: 12 days, 17:32:47 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 11:15:42,223 - Train: 25.80% [490800/1902000] [258.0/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 11:16:42,805 - Train: 25.81% [490900/1902000] [258.1/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 11:17:43,417 - Train: 25.81% [491000/1902000] [258.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:18:43,980 - Train: 25.82% [491100/1902000] [258.2/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 11:19:44,527 - Train: 25.83% [491200/1902000] [258.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:20:45,077 - Train: 25.83% [491300/1902000] [258.3/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 11:21:45,653 - Train: 25.84% [491400/1902000] [258.4/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 11:22:46,543 - Train: 25.84% [491500/1902000] [258.4/1000.0] [batch_t 0.609 (0.609)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 11:23:47,330 - Train: 25.85% [491600/1902000] [258.5/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 11:24:47,929 - Train: 25.85% [491700/1902000] [258.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 11:25:48,527 - Train: 25.86% [491800/1902000] [258.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:26:49,789 - Train: 25.86% [491900/1902000] [258.6/1000.0] [batch_t 0.608 (0.613)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 11:27:50,363 - Train: 25.87% [492000/1902000] [258.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 11:28:51,022 - Train: 25.87% [492100/1902000] [258.7/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 11:29:51,631 - Train: 25.88% [492200/1902000] [258.8/1000.0] [batch_t 0.608 (0.606)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-06 11:30:52,299 - Train: 25.88% [492300/1902000] [258.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:31:52,993 - Train: 25.89% [492400/1902000] [258.9/1000.0] [batch_t 0.607 (0.607)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-06 11:32:53,675 - Train: 25.89% [492500/1902000] [258.9/1000.0] [batch_t 0.608 (0.607)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-06 11:33:54,365 - Train: 25.90% [492600/1902000] [259.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-06 11:34:05,298 - ==> Total time: 4 days, 10:33:44 Eta: 12 days, 16:52:32 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 11:34:57,975 - Train: 25.90% [492700/1902000] [259.0/1000.0] [batch_t 0.605 (0.608)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-06 11:35:58,837 - Train: 25.91% [492800/1902000] [259.1/1000.0] [batch_t 0.609 (0.609)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-06 11:36:59,582 - Train: 25.91% [492900/1902000] [259.1/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 11:38:00,289 - Train: 25.92% [493000/1902000] [259.2/1000.0] [batch_t 0.606 (0.607)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-06 11:39:01,023 - Train: 25.93% [493100/1902000] [259.3/1000.0] [batch_t 0.608 (0.607)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-06 11:40:01,862 - Train: 25.93% [493200/1902000] [259.3/1000.0] [batch_t 0.608 (0.608)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-06 11:41:02,556 - Train: 25.94% [493300/1902000] [259.4/1000.0] [batch_t 0.607 (0.607)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-06 11:42:03,233 - Train: 25.94% [493400/1902000] [259.4/1000.0] [batch_t 0.608 (0.607)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-06 11:43:03,940 - Train: 25.95% [493500/1902000] [259.5/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:44:04,618 - Train: 25.95% [493600/1902000] [259.5/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 11:45:05,340 - Train: 25.96% [493700/1902000] [259.6/1000.0] [batch_t 0.609 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 11:46:08,115 - Train: 25.96% [493800/1902000] [259.6/1000.0] [batch_t 0.605 (0.628)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:47:15,091 - Train: 25.97% [493900/1902000] [259.7/1000.0] [batch_t 0.605 (0.670)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-06 11:48:30,931 - Train: 25.97% [494000/1902000] [259.7/1000.0] [batch_t 1.265 (0.758)] [data_t 0.660] [optim_t 0.605] [lr 0.005000] +2024-04-06 11:50:15,001 - Train: 25.98% [494100/1902000] [259.8/1000.0] [batch_t 0.608 (1.041)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-06 11:51:17,284 - Train: 25.98% [494200/1902000] [259.8/1000.0] [batch_t 0.607 (0.623)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-06 11:52:20,365 - Train: 25.99% [494300/1902000] [259.9/1000.0] [batch_t 0.607 (0.631)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 11:53:21,083 - Train: 25.99% [494400/1902000] [259.9/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:54:21,848 - Train: 26.00% [494500/1902000] [260.0/1000.0] [batch_t 0.607 (0.608)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-06 11:54:33,987 - ==> Total time: 4 days, 10:54:13 Eta: 12 days, 16:15:52 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 11:55:25,927 - Train: 26.00% [494600/1902000] [260.0/1000.0] [batch_t 0.608 (0.616)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-06 11:56:26,624 - Train: 26.01% [494700/1902000] [260.1/1000.0] [batch_t 0.606 (0.607)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:57:27,312 - Train: 26.01% [494800/1902000] [260.1/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 11:58:28,098 - Train: 26.02% [494900/1902000] [260.2/1000.0] [batch_t 0.606 (0.608)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-06 11:59:28,821 - Train: 26.03% [495000/1902000] [260.3/1000.0] [batch_t 0.609 (0.607)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-06 12:00:29,542 - Train: 26.03% [495100/1902000] [260.3/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 12:01:30,319 - Train: 26.04% [495200/1902000] [260.4/1000.0] [batch_t 0.608 (0.608)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 12:02:30,970 - Train: 26.04% [495300/1902000] [260.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 12:03:31,597 - Train: 26.05% [495400/1902000] [260.5/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 12:04:32,245 - Train: 26.05% [495500/1902000] [260.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:05:32,922 - Train: 26.06% [495600/1902000] [260.6/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 12:06:33,739 - Train: 26.06% [495700/1902000] [260.6/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 12:07:34,409 - Train: 26.07% [495800/1902000] [260.7/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:08:35,040 - Train: 26.07% [495900/1902000] [260.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:09:35,644 - Train: 26.08% [496000/1902000] [260.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-06 12:10:36,294 - Train: 26.08% [496100/1902000] [260.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 12:11:36,916 - Train: 26.09% [496200/1902000] [260.9/1000.0] [batch_t 0.609 (0.606)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-06 12:12:37,542 - Train: 26.09% [496300/1902000] [260.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 12:13:38,277 - Train: 26.10% [496400/1902000] [261.0/1000.0] [batch_t 0.609 (0.607)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 12:13:51,613 - ==> Total time: 4 days, 11:13:31 Eta: 12 days, 15:35:58 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 12:14:41,387 - Train: 26.10% [496500/1902000] [261.0/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 12:15:42,054 - Train: 26.11% [496600/1902000] [261.1/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 12:16:42,680 - Train: 26.11% [496700/1902000] [261.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 12:17:43,355 - Train: 26.12% [496800/1902000] [261.2/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 12:18:44,142 - Train: 26.13% [496900/1902000] [261.3/1000.0] [batch_t 0.610 (0.608)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-06 12:19:44,813 - Train: 26.13% [497000/1902000] [261.3/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 12:20:45,424 - Train: 26.14% [497100/1902000] [261.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 12:21:46,085 - Train: 26.14% [497200/1902000] [261.4/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 12:22:46,730 - Train: 26.15% [497300/1902000] [261.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 12:23:47,349 - Train: 26.15% [497400/1902000] [261.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:24:48,066 - Train: 26.16% [497500/1902000] [261.6/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 12:25:48,686 - Train: 26.16% [497600/1902000] [261.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 12:26:49,327 - Train: 26.17% [497700/1902000] [261.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:27:49,923 - Train: 26.17% [497800/1902000] [261.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 12:28:50,531 - Train: 26.18% [497900/1902000] [261.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:29:51,139 - Train: 26.18% [498000/1902000] [261.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:30:51,769 - Train: 26.19% [498100/1902000] [261.9/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 12:31:52,585 - Train: 26.19% [498200/1902000] [261.9/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:32:53,304 - Train: 26.20% [498300/1902000] [262.0/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 12:33:07,859 - ==> Total time: 4 days, 11:32:47 Eta: 12 days, 14:56:10 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 12:33:57,284 - Train: 26.20% [498400/1902000] [262.0/1000.0] [batch_t 0.606 (0.615)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:34:57,887 - Train: 26.21% [498500/1902000] [262.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 12:35:58,497 - Train: 26.21% [498600/1902000] [262.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 12:36:59,083 - Train: 26.22% [498700/1902000] [262.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 12:37:59,873 - Train: 26.23% [498800/1902000] [262.3/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 12:39:00,496 - Train: 26.23% [498900/1902000] [262.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:40:01,134 - Train: 26.24% [499000/1902000] [262.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 12:41:01,768 - Train: 26.24% [499100/1902000] [262.4/1000.0] [batch_t 0.610 (0.606)] [data_t 0.003] [optim_t 0.607] [lr 0.005000] +2024-04-06 12:42:02,431 - Train: 26.25% [499200/1902000] [262.5/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:43:03,049 - Train: 26.25% [499300/1902000] [262.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:44:03,864 - Train: 26.26% [499400/1902000] [262.6/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 12:45:04,710 - Train: 26.26% [499500/1902000] [262.6/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 12:46:05,356 - Train: 26.27% [499600/1902000] [262.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 12:47:06,021 - Train: 26.27% [499700/1902000] [262.7/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 12:48:06,642 - Train: 26.28% [499800/1902000] [262.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:49:07,277 - Train: 26.28% [499900/1902000] [262.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 12:50:07,917 - Train: 26.29% [500000/1902000] [262.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 12:51:08,552 - Train: 26.29% [500100/1902000] [262.9/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 12:52:09,600 - Train: 26.30% [500200/1902000] [263.0/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 12:52:25,348 - ==> Total time: 4 days, 11:52:04 Eta: 12 days, 14:16:35 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 12:53:12,530 - Train: 26.30% [500300/1902000] [263.0/1000.0] [batch_t 0.608 (0.609)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 12:54:13,122 - Train: 26.31% [500400/1902000] [263.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 12:55:13,750 - Train: 26.31% [500500/1902000] [263.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:56:14,366 - Train: 26.32% [500600/1902000] [263.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 12:57:15,131 - Train: 26.32% [500700/1902000] [263.2/1000.0] [batch_t 0.608 (0.608)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-06 12:58:15,751 - Train: 26.33% [500800/1902000] [263.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 12:59:16,369 - Train: 26.34% [500900/1902000] [263.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 13:00:17,008 - Train: 26.34% [501000/1902000] [263.4/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 13:01:17,664 - Train: 26.35% [501100/1902000] [263.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:02:18,275 - Train: 26.35% [501200/1902000] [263.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 13:03:18,970 - Train: 26.36% [501300/1902000] [263.6/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:04:19,569 - Train: 26.36% [501400/1902000] [263.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 13:05:20,182 - Train: 26.37% [501500/1902000] [263.7/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 13:06:20,770 - Train: 26.37% [501600/1902000] [263.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 13:07:21,355 - Train: 26.38% [501700/1902000] [263.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:08:21,994 - Train: 26.38% [501800/1902000] [263.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 13:09:22,601 - Train: 26.39% [501900/1902000] [263.9/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 13:10:23,502 - Train: 26.39% [502000/1902000] [263.9/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 13:11:24,193 - Train: 26.40% [502100/1902000] [264.0/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 13:11:41,166 - ==> Total time: 4 days, 12:11:20 Eta: 12 days, 13:37:04 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 13:12:27,135 - Train: 26.40% [502200/1902000] [264.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:13:28,016 - Train: 26.41% [502300/1902000] [264.1/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 13:14:28,634 - Train: 26.41% [502400/1902000] [264.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 13:15:29,258 - Train: 26.42% [502500/1902000] [264.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 13:16:30,517 - Train: 26.42% [502600/1902000] [264.2/1000.0] [batch_t 0.601 (0.612)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 13:17:31,136 - Train: 26.43% [502700/1902000] [264.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 13:18:32,274 - Train: 26.44% [502800/1902000] [264.4/1000.0] [batch_t 0.606 (0.611)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 13:19:32,910 - Train: 26.44% [502900/1902000] [264.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:20:33,542 - Train: 26.45% [503000/1902000] [264.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:21:34,184 - Train: 26.45% [503100/1902000] [264.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 13:22:34,906 - Train: 26.46% [503200/1902000] [264.6/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 13:23:35,537 - Train: 26.46% [503300/1902000] [264.6/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 13:24:36,181 - Train: 26.47% [503400/1902000] [264.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:25:36,804 - Train: 26.47% [503500/1902000] [264.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 13:26:37,447 - Train: 26.48% [503600/1902000] [264.8/1000.0] [batch_t 0.608 (0.606)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-06 13:27:38,089 - Train: 26.48% [503700/1902000] [264.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:28:38,718 - Train: 26.49% [503800/1902000] [264.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 13:29:39,434 - Train: 26.49% [503900/1902000] [264.9/1000.0] [batch_t 0.607 (0.607)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-06 13:30:40,058 - Train: 26.50% [504000/1902000] [265.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:30:58,243 - ==> Total time: 4 days, 12:30:37 Eta: 12 days, 12:57:46 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 13:31:43,546 - Train: 26.50% [504100/1902000] [265.0/1000.0] [batch_t 0.607 (0.614)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 13:32:44,154 - Train: 26.51% [504200/1902000] [265.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 13:33:44,789 - Train: 26.51% [504300/1902000] [265.1/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 13:34:45,573 - Train: 26.52% [504400/1902000] [265.2/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 13:35:46,186 - Train: 26.52% [504500/1902000] [265.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 13:36:46,810 - Train: 26.53% [504600/1902000] [265.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 13:37:47,475 - Train: 26.54% [504700/1902000] [265.4/1000.0] [batch_t 0.611 (0.607)] [data_t 0.003] [optim_t 0.609] [lr 0.005000] +2024-04-06 13:38:48,132 - Train: 26.54% [504800/1902000] [265.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 13:39:48,779 - Train: 26.55% [504900/1902000] [265.5/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 13:40:49,427 - Train: 26.55% [505000/1902000] [265.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:41:50,099 - Train: 26.56% [505100/1902000] [265.6/1000.0] [batch_t 0.611 (0.607)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-06 13:42:50,870 - Train: 26.56% [505200/1902000] [265.6/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 13:43:51,481 - Train: 26.57% [505300/1902000] [265.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 13:44:52,097 - Train: 26.57% [505400/1902000] [265.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 13:45:52,731 - Train: 26.58% [505500/1902000] [265.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:46:53,336 - Train: 26.58% [505600/1902000] [265.8/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 13:47:53,984 - Train: 26.59% [505700/1902000] [265.9/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 13:48:54,761 - Train: 26.59% [505800/1902000] [265.9/1000.0] [batch_t 0.612 (0.608)] [data_t 0.002] [optim_t 0.610] [lr 0.005000] +2024-04-06 13:49:55,385 - Train: 26.60% [505900/1902000] [266.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:50:14,785 - ==> Total time: 4 days, 12:49:54 Eta: 12 days, 12:18:36 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 13:50:58,222 - Train: 26.60% [506000/1902000] [266.0/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 13:51:58,903 - Train: 26.61% [506100/1902000] [266.1/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 13:52:59,496 - Train: 26.61% [506200/1902000] [266.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:54:00,278 - Train: 26.62% [506300/1902000] [266.2/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 13:55:00,881 - Train: 26.62% [506400/1902000] [266.2/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 13:56:01,501 - Train: 26.63% [506500/1902000] [266.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 13:57:02,205 - Train: 26.64% [506600/1902000] [266.4/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 13:58:02,865 - Train: 26.64% [506700/1902000] [266.4/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 13:59:03,549 - Train: 26.65% [506800/1902000] [266.5/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:00:04,155 - Train: 26.65% [506900/1902000] [266.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:01:04,829 - Train: 26.66% [507000/1902000] [266.6/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:02:05,537 - Train: 26.66% [507100/1902000] [266.6/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:03:06,174 - Train: 26.67% [507200/1902000] [266.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 14:04:06,829 - Train: 26.67% [507300/1902000] [266.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:05:07,419 - Train: 26.68% [507400/1902000] [266.8/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 14:06:08,041 - Train: 26.68% [507500/1902000] [266.8/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 14:07:08,759 - Train: 26.69% [507600/1902000] [266.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:08:09,399 - Train: 26.69% [507700/1902000] [266.9/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 14:09:10,028 - Train: 26.70% [507800/1902000] [267.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:09:30,646 - ==> Total time: 4 days, 13:09:10 Eta: 12 days, 11:39:33 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 14:10:12,783 - Train: 26.70% [507900/1902000] [267.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:11:13,453 - Train: 26.71% [508000/1902000] [267.1/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:12:14,035 - Train: 26.71% [508100/1902000] [267.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:13:14,740 - Train: 26.72% [508200/1902000] [267.2/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:14:15,286 - Train: 26.72% [508300/1902000] [267.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 14:15:15,867 - Train: 26.73% [508400/1902000] [267.3/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 14:16:16,403 - Train: 26.74% [508500/1902000] [267.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 14:17:17,396 - Train: 26.74% [508600/1902000] [267.4/1000.0] [batch_t 0.601 (0.610)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 14:18:18,650 - Train: 26.75% [508700/1902000] [267.5/1000.0] [batch_t 0.611 (0.612)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-06 14:19:19,222 - Train: 26.75% [508800/1902000] [267.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:20:19,847 - Train: 26.76% [508900/1902000] [267.6/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 14:21:20,503 - Train: 26.76% [509000/1902000] [267.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:22:21,091 - Train: 26.77% [509100/1902000] [267.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:23:21,679 - Train: 26.77% [509200/1902000] [267.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 14:24:22,204 - Train: 26.78% [509300/1902000] [267.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 14:25:23,576 - Train: 26.78% [509400/1902000] [267.8/1000.0] [batch_t 0.606 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:26:24,307 - Train: 26.79% [509500/1902000] [267.9/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 14:27:24,892 - Train: 26.79% [509600/1902000] [267.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 14:28:25,433 - Train: 26.80% [509700/1902000] [268.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 14:28:47,297 - ==> Total time: 4 days, 13:28:26 Eta: 12 days, 11:00:41 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 14:29:28,135 - Train: 26.80% [509800/1902000] [268.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 14:30:29,398 - Train: 26.81% [509900/1902000] [268.1/1000.0] [batch_t 0.607 (0.613)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 14:31:29,958 - Train: 26.81% [510000/1902000] [268.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 14:32:30,672 - Train: 26.82% [510100/1902000] [268.2/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:33:31,237 - Train: 26.82% [510200/1902000] [268.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:34:31,780 - Train: 26.83% [510300/1902000] [268.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:35:32,327 - Train: 26.83% [510400/1902000] [268.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:36:32,876 - Train: 26.84% [510500/1902000] [268.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:37:33,418 - Train: 26.85% [510600/1902000] [268.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:38:33,963 - Train: 26.85% [510700/1902000] [268.5/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 14:39:34,613 - Train: 26.86% [510800/1902000] [268.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:40:35,180 - Train: 26.86% [510900/1902000] [268.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:41:35,732 - Train: 26.87% [511000/1902000] [268.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:42:36,322 - Train: 26.87% [511100/1902000] [268.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:43:36,895 - Train: 26.88% [511200/1902000] [268.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:44:37,538 - Train: 26.88% [511300/1902000] [268.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:45:38,091 - Train: 26.89% [511400/1902000] [268.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:46:38,637 - Train: 26.89% [511500/1902000] [268.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:47:39,199 - Train: 26.90% [511600/1902000] [269.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:48:02,212 - ==> Total time: 4 days, 13:47:41 Eta: 12 days, 10:21:52 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 14:48:44,837 - Train: 26.90% [511700/1902000] [269.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 14:49:45,380 - Train: 26.91% [511800/1902000] [269.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 14:50:46,073 - Train: 26.91% [511900/1902000] [269.1/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:51:46,606 - Train: 26.92% [512000/1902000] [269.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:52:47,154 - Train: 26.92% [512100/1902000] [269.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:53:47,698 - Train: 26.93% [512200/1902000] [269.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 14:54:48,599 - Train: 26.93% [512300/1902000] [269.3/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:55:49,180 - Train: 26.94% [512400/1902000] [269.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:56:50,100 - Train: 26.95% [512500/1902000] [269.5/1000.0] [batch_t 0.605 (0.609)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:57:50,847 - Train: 26.95% [512600/1902000] [269.5/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 14:58:51,609 - Train: 26.96% [512700/1902000] [269.6/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 14:59:52,201 - Train: 26.96% [512800/1902000] [269.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 15:00:52,855 - Train: 26.97% [512900/1902000] [269.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 15:01:53,441 - Train: 26.97% [513000/1902000] [269.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 15:02:54,180 - Train: 26.98% [513100/1902000] [269.8/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 15:03:54,766 - Train: 26.98% [513200/1902000] [269.8/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 15:04:55,324 - Train: 26.99% [513300/1902000] [269.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 15:05:55,897 - Train: 26.99% [513400/1902000] [269.9/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 15:06:56,489 - Train: 27.00% [513500/1902000] [270.0/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 15:07:20,717 - ==> Total time: 4 days, 14:07:00 Eta: 12 days, 9:43:22 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 15:07:59,129 - Train: 27.00% [513600/1902000] [270.0/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 15:08:59,677 - Train: 27.01% [513700/1902000] [270.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 15:10:00,338 - Train: 27.01% [513800/1902000] [270.1/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 15:11:12,521 - Train: 27.02% [513900/1902000] [270.2/1000.0] [batch_t 0.607 (0.722)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 15:12:28,028 - Train: 27.02% [514000/1902000] [270.2/1000.0] [batch_t 0.605 (0.755)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 15:13:37,519 - Train: 27.03% [514100/1902000] [270.3/1000.0] [batch_t 0.607 (0.695)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 15:14:47,491 - Train: 27.03% [514200/1902000] [270.3/1000.0] [batch_t 0.605 (0.700)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 15:16:45,822 - Train: 27.04% [514300/1902000] [270.4/1000.0] [batch_t 0.726 (1.183)] [data_t 0.125] [optim_t 0.601] [lr 0.005000] +2024-04-06 15:18:09,071 - Train: 27.05% [514400/1902000] [270.5/1000.0] [batch_t 0.606 (0.832)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 15:19:18,082 - Train: 27.05% [514500/1902000] [270.5/1000.0] [batch_t 0.604 (0.690)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 15:20:34,210 - Train: 27.06% [514600/1902000] [270.6/1000.0] [batch_t 0.607 (0.761)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 15:21:42,926 - Train: 27.06% [514700/1902000] [270.6/1000.0] [batch_t 0.606 (0.687)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 15:22:48,380 - Train: 27.07% [514800/1902000] [270.7/1000.0] [batch_t 0.606 (0.654)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 15:23:55,910 - Train: 27.07% [514900/1902000] [270.7/1000.0] [batch_t 0.605 (0.675)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 15:25:03,107 - Train: 27.08% [515000/1902000] [270.8/1000.0] [batch_t 0.607 (0.672)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 15:26:04,992 - Train: 27.08% [515100/1902000] [270.8/1000.0] [batch_t 0.605 (0.619)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 15:27:05,671 - Train: 27.09% [515200/1902000] [270.9/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 15:28:11,179 - Train: 27.09% [515300/1902000] [270.9/1000.0] [batch_t 0.607 (0.655)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 15:29:19,514 - Train: 27.10% [515400/1902000] [271.0/1000.0] [batch_t 0.603 (0.683)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 15:29:45,244 - ==> Total time: 4 days, 14:29:24 Eta: 12 days, 9:13:21 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 15:30:24,819 - Train: 27.10% [515500/1902000] [271.0/1000.0] [batch_t 0.604 (0.623)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 15:31:25,393 - Train: 27.11% [515600/1902000] [271.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 15:32:26,000 - Train: 27.11% [515700/1902000] [271.1/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 15:33:26,791 - Train: 27.12% [515800/1902000] [271.2/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 15:34:27,351 - Train: 27.12% [515900/1902000] [271.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 15:35:27,907 - Train: 27.13% [516000/1902000] [271.3/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 15:36:28,486 - Train: 27.13% [516100/1902000] [271.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 15:37:29,019 - Train: 27.14% [516200/1902000] [271.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 15:38:29,585 - Train: 27.15% [516300/1902000] [271.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 15:39:30,126 - Train: 27.15% [516400/1902000] [271.5/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 15:40:30,730 - Train: 27.16% [516500/1902000] [271.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 15:41:33,213 - Train: 27.16% [516600/1902000] [271.6/1000.0] [batch_t 0.604 (0.625)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 15:42:54,369 - Train: 27.17% [516700/1902000] [271.7/1000.0] [batch_t 0.603 (0.811)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 15:43:58,774 - Train: 27.17% [516800/1902000] [271.7/1000.0] [batch_t 0.904 (0.644)] [data_t 0.300] [optim_t 0.604] [lr 0.005000] +2024-04-06 15:45:06,626 - Train: 27.18% [516900/1902000] [271.8/1000.0] [batch_t 0.605 (0.678)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 15:46:07,152 - Train: 27.18% [517000/1902000] [271.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 15:47:07,835 - Train: 27.19% [517100/1902000] [271.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 15:48:08,521 - Train: 27.19% [517200/1902000] [271.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 15:49:09,099 - Train: 27.20% [517300/1902000] [272.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 15:49:36,125 - ==> Total time: 4 days, 14:49:15 Eta: 12 days, 8:36:33 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 15:50:15,132 - Train: 27.20% [517400/1902000] [272.0/1000.0] [batch_t 0.607 (0.628)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 15:51:18,403 - Train: 27.21% [517500/1902000] [272.1/1000.0] [batch_t 0.606 (0.633)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 15:52:20,235 - Train: 27.21% [517600/1902000] [272.1/1000.0] [batch_t 0.604 (0.618)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 15:53:20,956 - Train: 27.22% [517700/1902000] [272.2/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 15:54:21,500 - Train: 27.22% [517800/1902000] [272.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 15:55:29,313 - Train: 27.23% [517900/1902000] [272.3/1000.0] [batch_t 0.605 (0.678)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 15:56:31,369 - Train: 27.23% [518000/1902000] [272.3/1000.0] [batch_t 0.606 (0.620)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 15:57:33,908 - Train: 27.24% [518100/1902000] [272.4/1000.0] [batch_t 0.603 (0.625)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 15:58:40,280 - Train: 27.25% [518200/1902000] [272.5/1000.0] [batch_t 0.605 (0.664)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 16:02:11,539 - Train: 27.25% [518300/1902000] [272.5/1000.0] [batch_t 0.606 (2.112)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:03:12,068 - Train: 27.26% [518400/1902000] [272.6/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 16:04:12,801 - Train: 27.26% [518500/1902000] [272.6/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:05:13,391 - Train: 27.27% [518600/1902000] [272.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:06:13,947 - Train: 27.27% [518700/1902000] [272.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:07:15,019 - Train: 27.28% [518800/1902000] [272.8/1000.0] [batch_t 0.606 (0.611)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:08:15,622 - Train: 27.28% [518900/1902000] [272.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 16:09:16,282 - Train: 27.29% [519000/1902000] [272.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:10:16,906 - Train: 27.29% [519100/1902000] [272.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 16:11:17,501 - Train: 27.30% [519200/1902000] [273.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:11:45,348 - ==> Total time: 4 days, 15:11:24 Eta: 12 days, 8:06:00 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 16:12:20,874 - Train: 27.30% [519300/1902000] [273.0/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:13:21,407 - Train: 27.31% [519400/1902000] [273.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:14:21,957 - Train: 27.31% [519500/1902000] [273.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 16:15:22,671 - Train: 27.32% [519600/1902000] [273.2/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:16:23,227 - Train: 27.32% [519700/1902000] [273.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 16:17:23,810 - Train: 27.33% [519800/1902000] [273.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 16:18:24,543 - Train: 27.33% [519900/1902000] [273.3/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 16:19:25,109 - Train: 27.34% [520000/1902000] [273.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 16:20:25,737 - Train: 27.34% [520100/1902000] [273.4/1000.0] [batch_t 0.699 (0.606)] [data_t 0.002] [optim_t 0.697] [lr 0.005000] +2024-04-06 16:21:26,727 - Train: 27.35% [520200/1902000] [273.5/1000.0] [batch_t 0.605 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 16:22:28,160 - Train: 27.36% [520300/1902000] [273.6/1000.0] [batch_t 0.605 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:23:29,406 - Train: 27.36% [520400/1902000] [273.6/1000.0] [batch_t 0.608 (0.612)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 16:24:29,942 - Train: 27.37% [520500/1902000] [273.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 16:25:30,497 - Train: 27.37% [520600/1902000] [273.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 16:26:31,072 - Train: 27.38% [520700/1902000] [273.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 16:27:31,661 - Train: 27.38% [520800/1902000] [273.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 16:28:32,310 - Train: 27.39% [520900/1902000] [273.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:29:32,878 - Train: 27.39% [521000/1902000] [273.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 16:30:33,458 - Train: 27.40% [521100/1902000] [274.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:31:02,516 - ==> Total time: 4 days, 15:30:42 Eta: 12 days, 7:27:54 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 16:31:38,805 - Train: 27.40% [521200/1902000] [274.0/1000.0] [batch_t 0.604 (0.610)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 16:32:39,329 - Train: 27.41% [521300/1902000] [274.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:33:40,501 - Train: 27.41% [521400/1902000] [274.1/1000.0] [batch_t 0.606 (0.612)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:34:41,092 - Train: 27.42% [521500/1902000] [274.2/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 16:35:41,672 - Train: 27.42% [521600/1902000] [274.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 16:36:42,235 - Train: 27.43% [521700/1902000] [274.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:37:42,818 - Train: 27.43% [521800/1902000] [274.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:38:43,503 - Train: 27.44% [521900/1902000] [274.4/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 16:39:44,045 - Train: 27.44% [522000/1902000] [274.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 16:40:44,629 - Train: 27.45% [522100/1902000] [274.5/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 16:41:45,198 - Train: 27.46% [522200/1902000] [274.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:42:45,744 - Train: 27.46% [522300/1902000] [274.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:43:46,387 - Train: 27.47% [522400/1902000] [274.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 16:44:46,974 - Train: 27.47% [522500/1902000] [274.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:45:47,699 - Train: 27.48% [522600/1902000] [274.8/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:46:48,314 - Train: 27.48% [522700/1902000] [274.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 16:47:48,906 - Train: 27.49% [522800/1902000] [274.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:48:49,510 - Train: 27.49% [522900/1902000] [274.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 16:49:50,058 - Train: 27.50% [523000/1902000] [275.0/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 16:50:20,356 - ==> Total time: 4 days, 15:49:59 Eta: 12 days, 6:49:59 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 16:50:52,772 - Train: 27.50% [523100/1902000] [275.0/1000.0] [batch_t 0.613 (0.606)] [data_t 0.002] [optim_t 0.611] [lr 0.005000] +2024-04-06 16:51:53,425 - Train: 27.51% [523200/1902000] [275.1/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 16:52:53,934 - Train: 27.51% [523300/1902000] [275.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 16:53:54,524 - Train: 27.52% [523400/1902000] [275.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 16:54:55,110 - Train: 27.52% [523500/1902000] [275.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 16:55:55,831 - Train: 27.53% [523600/1902000] [275.3/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 16:56:56,554 - Train: 27.53% [523700/1902000] [275.3/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 16:57:57,126 - Train: 27.54% [523800/1902000] [275.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 16:58:57,708 - Train: 27.54% [523900/1902000] [275.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 16:59:58,281 - Train: 27.55% [524000/1902000] [275.5/1000.0] [batch_t 0.609 (0.606)] [data_t 0.004] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:00:58,860 - Train: 27.56% [524100/1902000] [275.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 17:01:59,429 - Train: 27.56% [524200/1902000] [275.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:03:00,013 - Train: 27.57% [524300/1902000] [275.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:04:00,921 - Train: 27.57% [524400/1902000] [275.7/1000.0] [batch_t 0.604 (0.609)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 17:05:01,642 - Train: 27.58% [524500/1902000] [275.8/1000.0] [batch_t 0.611 (0.607)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-06 17:06:02,203 - Train: 27.58% [524600/1902000] [275.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 17:07:02,783 - Train: 27.59% [524700/1902000] [275.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:08:03,366 - Train: 27.59% [524800/1902000] [275.9/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 17:09:03,914 - Train: 27.60% [524900/1902000] [276.0/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 17:09:35,417 - ==> Total time: 4 days, 16:09:14 Eta: 12 days, 6:12:05 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 17:10:08,069 - Train: 27.60% [525000/1902000] [276.0/1000.0] [batch_t 1.660 (0.629)] [data_t 1.053] [optim_t 0.607] [lr 0.005000] +2024-04-06 17:11:09,471 - Train: 27.61% [525100/1902000] [276.1/1000.0] [batch_t 0.604 (0.614)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 17:12:10,048 - Train: 27.61% [525200/1902000] [276.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 17:13:10,640 - Train: 27.62% [525300/1902000] [276.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 17:14:11,253 - Train: 27.62% [525400/1902000] [276.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:15:11,832 - Train: 27.63% [525500/1902000] [276.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 17:16:12,430 - Train: 27.63% [525600/1902000] [276.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:17:12,986 - Train: 27.64% [525700/1902000] [276.4/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 17:18:13,724 - Train: 27.64% [525800/1902000] [276.4/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 17:19:14,688 - Train: 27.65% [525900/1902000] [276.5/1000.0] [batch_t 0.606 (0.610)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:20:15,248 - Train: 27.66% [526000/1902000] [276.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:21:15,853 - Train: 27.66% [526100/1902000] [276.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 17:22:16,432 - Train: 27.67% [526200/1902000] [276.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:23:17,180 - Train: 27.67% [526300/1902000] [276.7/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 17:24:17,752 - Train: 27.68% [526400/1902000] [276.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:25:18,716 - Train: 27.68% [526500/1902000] [276.8/1000.0] [batch_t 0.604 (0.610)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:26:19,252 - Train: 27.69% [526600/1902000] [276.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:27:19,791 - Train: 27.69% [526700/1902000] [276.9/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 17:28:21,666 - Train: 27.70% [526800/1902000] [277.0/1000.0] [batch_t 0.606 (0.619)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:28:54,362 - ==> Total time: 4 days, 16:28:33 Eta: 12 days, 5:34:29 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 17:29:31,898 - Train: 27.70% [526900/1902000] [277.0/1000.0] [batch_t 0.605 (0.752)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:30:32,460 - Train: 27.71% [527000/1902000] [277.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:31:33,007 - Train: 27.71% [527100/1902000] [277.1/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 17:32:34,127 - Train: 27.72% [527200/1902000] [277.2/1000.0] [batch_t 0.606 (0.611)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:33:34,665 - Train: 27.72% [527300/1902000] [277.2/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 17:34:35,238 - Train: 27.73% [527400/1902000] [277.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:35:35,777 - Train: 27.73% [527500/1902000] [277.3/1000.0] [batch_t 0.610 (0.605)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-06 17:36:36,346 - Train: 27.74% [527600/1902000] [277.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:37:37,106 - Train: 27.74% [527700/1902000] [277.4/1000.0] [batch_t 0.609 (0.607)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 17:38:37,677 - Train: 27.75% [527800/1902000] [277.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:39:38,267 - Train: 27.75% [527900/1902000] [277.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 17:40:38,850 - Train: 27.76% [528000/1902000] [277.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:41:39,410 - Train: 27.77% [528100/1902000] [277.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:42:40,014 - Train: 27.77% [528200/1902000] [277.7/1000.0] [batch_t 0.613 (0.606)] [data_t 0.002] [optim_t 0.611] [lr 0.005000] +2024-04-06 17:43:40,590 - Train: 27.78% [528300/1902000] [277.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:44:41,294 - Train: 27.78% [528400/1902000] [277.8/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 17:45:41,861 - Train: 27.79% [528500/1902000] [277.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 17:46:43,246 - Train: 27.79% [528600/1902000] [277.9/1000.0] [batch_t 0.603 (0.614)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 17:47:44,568 - Train: 27.80% [528700/1902000] [278.0/1000.0] [batch_t 0.608 (0.613)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 17:48:18,513 - ==> Total time: 4 days, 16:47:58 Eta: 12 days, 4:57:14 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 17:48:47,615 - Train: 27.80% [528800/1902000] [278.0/1000.0] [batch_t 0.607 (0.612)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 17:49:48,177 - Train: 27.81% [528900/1902000] [278.1/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 17:50:48,731 - Train: 27.81% [529000/1902000] [278.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:51:50,107 - Train: 27.82% [529100/1902000] [278.2/1000.0] [batch_t 0.606 (0.614)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:52:50,637 - Train: 27.82% [529200/1902000] [278.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 17:53:51,228 - Train: 27.83% [529300/1902000] [278.3/1000.0] [batch_t 0.613 (0.606)] [data_t 0.002] [optim_t 0.611] [lr 0.005000] +2024-04-06 17:54:52,603 - Train: 27.83% [529400/1902000] [278.3/1000.0] [batch_t 0.607 (0.614)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 17:55:54,227 - Train: 27.84% [529500/1902000] [278.4/1000.0] [batch_t 0.605 (0.616)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:56:54,853 - Train: 27.84% [529600/1902000] [278.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 17:57:55,428 - Train: 27.85% [529700/1902000] [278.5/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 17:58:55,983 - Train: 27.85% [529800/1902000] [278.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 17:59:56,569 - Train: 27.86% [529900/1902000] [278.6/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-06 18:00:57,118 - Train: 27.87% [530000/1902000] [278.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 18:01:57,750 - Train: 27.87% [530100/1902000] [278.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 18:02:58,295 - Train: 27.88% [530200/1902000] [278.8/1000.0] [batch_t 0.601 (0.605)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-06 18:03:58,856 - Train: 27.88% [530300/1902000] [278.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 18:04:59,370 - Train: 27.89% [530400/1902000] [278.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 18:05:59,934 - Train: 27.89% [530500/1902000] [278.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 18:07:00,752 - Train: 27.90% [530600/1902000] [279.0/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 18:07:35,874 - ==> Total time: 4 days, 17:07:15 Eta: 12 days, 4:19:49 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 18:08:04,307 - Train: 27.90% [530700/1902000] [279.0/1000.0] [batch_t 0.605 (0.616)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 18:09:04,879 - Train: 27.91% [530800/1902000] [279.1/1000.0] [batch_t 0.611 (0.606)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-06 18:10:05,470 - Train: 27.91% [530900/1902000] [279.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 18:11:06,050 - Train: 27.92% [531000/1902000] [279.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 18:12:06,662 - Train: 27.92% [531100/1902000] [279.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 18:13:07,349 - Train: 27.93% [531200/1902000] [279.3/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 18:14:07,934 - Train: 27.93% [531300/1902000] [279.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 18:15:08,511 - Train: 27.94% [531400/1902000] [279.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 18:16:09,101 - Train: 27.94% [531500/1902000] [279.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 18:17:09,692 - Train: 27.95% [531600/1902000] [279.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 18:18:10,245 - Train: 27.95% [531700/1902000] [279.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 18:19:10,824 - Train: 27.96% [531800/1902000] [279.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 18:20:11,438 - Train: 27.97% [531900/1902000] [279.7/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 18:21:12,229 - Train: 27.97% [532000/1902000] [279.7/1000.0] [batch_t 0.609 (0.608)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 18:22:12,838 - Train: 27.98% [532100/1902000] [279.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 18:23:13,424 - Train: 27.98% [532200/1902000] [279.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 18:24:13,959 - Train: 27.99% [532300/1902000] [279.9/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 18:25:14,554 - Train: 27.99% [532400/1902000] [279.9/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-06 18:26:15,300 - Train: 28.00% [532500/1902000] [280.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 18:26:51,638 - ==> Total time: 4 days, 17:26:31 Eta: 12 days, 3:42:28 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 18:27:18,504 - Train: 28.00% [532600/1902000] [280.0/1000.0] [batch_t 0.604 (0.618)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 18:28:19,092 - Train: 28.01% [532700/1902000] [280.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 18:29:19,683 - Train: 28.01% [532800/1902000] [280.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 18:30:20,229 - Train: 28.02% [532900/1902000] [280.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 18:31:20,771 - Train: 28.02% [533000/1902000] [280.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 18:32:21,348 - Train: 28.03% [533100/1902000] [280.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 18:33:22,033 - Train: 28.03% [533200/1902000] [280.3/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 18:34:22,609 - Train: 28.04% [533300/1902000] [280.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 18:35:23,166 - Train: 28.04% [533400/1902000] [280.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 18:36:23,701 - Train: 28.05% [533500/1902000] [280.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 18:37:24,229 - Train: 28.05% [533600/1902000] [280.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 18:38:24,788 - Train: 28.06% [533700/1902000] [280.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 18:39:25,411 - Train: 28.07% [533800/1902000] [280.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 18:40:25,957 - Train: 28.07% [533900/1902000] [280.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 18:41:26,489 - Train: 28.08% [534000/1902000] [280.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 18:42:27,028 - Train: 28.08% [534100/1902000] [280.8/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 18:43:27,579 - Train: 28.09% [534200/1902000] [280.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 18:44:28,135 - Train: 28.09% [534300/1902000] [280.9/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 18:45:28,689 - Train: 28.10% [534400/1902000] [281.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 18:46:06,225 - ==> Total time: 4 days, 17:45:45 Eta: 12 days, 3:05:12 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 18:46:31,651 - Train: 28.10% [534500/1902000] [281.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 18:47:32,174 - Train: 28.11% [534600/1902000] [281.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 18:48:32,719 - Train: 28.11% [534700/1902000] [281.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 18:49:33,323 - Train: 28.12% [534800/1902000] [281.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 18:50:45,981 - Train: 28.12% [534900/1902000] [281.2/1000.0] [batch_t 0.605 (0.726)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 18:53:31,090 - Train: 28.13% [535000/1902000] [281.3/1000.0] [batch_t 0.604 (1.651)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 18:55:52,945 - Train: 28.13% [535100/1902000] [281.3/1000.0] [batch_t 2.911 (1.418)] [data_t 2.305] [optim_t 0.607] [lr 0.005000] +2024-04-06 18:58:19,051 - Train: 28.14% [535200/1902000] [281.4/1000.0] [batch_t 0.612 (1.461)] [data_t 0.002] [optim_t 0.610] [lr 0.005000] +2024-04-06 19:00:35,668 - Train: 28.14% [535300/1902000] [281.4/1000.0] [batch_t 4.550 (1.366)] [data_t 3.944] [optim_t 0.606] [lr 0.005000] +2024-04-06 19:04:36,165 - Train: 28.15% [535400/1902000] [281.5/1000.0] [batch_t 5.093 (2.405)] [data_t 4.489] [optim_t 0.604] [lr 0.005000] +2024-04-06 19:08:32,046 - Train: 28.15% [535500/1902000] [281.5/1000.0] [batch_t 3.103 (2.359)] [data_t 2.501] [optim_t 0.602] [lr 0.005000] +2024-04-06 19:11:27,373 - Train: 28.16% [535600/1902000] [281.6/1000.0] [batch_t 1.417 (1.753)] [data_t 0.002] [optim_t 1.415] [lr 0.005000] +2024-04-06 19:13:48,791 - Train: 28.17% [535700/1902000] [281.7/1000.0] [batch_t 1.418 (1.414)] [data_t 0.002] [optim_t 1.415] [lr 0.005000] +2024-04-06 19:16:10,132 - Train: 28.17% [535800/1902000] [281.7/1000.0] [batch_t 1.409 (1.413)] [data_t 0.002] [optim_t 1.407] [lr 0.005000] +2024-04-06 19:18:31,455 - Train: 28.18% [535900/1902000] [281.8/1000.0] [batch_t 1.412 (1.413)] [data_t 0.003] [optim_t 1.410] [lr 0.005000] +2024-04-06 19:20:52,798 - Train: 28.18% [536000/1902000] [281.8/1000.0] [batch_t 1.417 (1.413)] [data_t 0.002] [optim_t 1.415] [lr 0.005000] +2024-04-06 19:23:14,128 - Train: 28.19% [536100/1902000] [281.9/1000.0] [batch_t 1.409 (1.413)] [data_t 0.002] [optim_t 1.406] [lr 0.005000] +2024-04-06 19:25:35,439 - Train: 28.19% [536200/1902000] [281.9/1000.0] [batch_t 1.414 (1.413)] [data_t 0.002] [optim_t 1.411] [lr 0.005000] +2024-04-06 19:27:56,728 - Train: 28.20% [536300/1902000] [282.0/1000.0] [batch_t 1.414 (1.413)] [data_t 0.002] [optim_t 1.412] [lr 0.005000] +2024-04-06 19:29:27,205 - ==> Total time: 4 days, 18:29:06 Eta: 12 days, 3:29:26 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 19:30:20,830 - Train: 28.20% [536400/1902000] [282.0/1000.0] [batch_t 1.413 (1.421)] [data_t 0.002] [optim_t 1.411] [lr 0.005000] +2024-04-06 19:32:42,225 - Train: 28.21% [536500/1902000] [282.1/1000.0] [batch_t 1.408 (1.414)] [data_t 0.002] [optim_t 1.405] [lr 0.005000] +2024-04-06 19:35:03,553 - Train: 28.21% [536600/1902000] [282.1/1000.0] [batch_t 1.419 (1.413)] [data_t 0.002] [optim_t 1.417] [lr 0.005000] +2024-04-06 19:37:24,896 - Train: 28.22% [536700/1902000] [282.2/1000.0] [batch_t 1.409 (1.413)] [data_t 0.002] [optim_t 1.407] [lr 0.005000] +2024-04-06 19:39:46,160 - Train: 28.22% [536800/1902000] [282.2/1000.0] [batch_t 1.413 (1.413)] [data_t 0.002] [optim_t 1.411] [lr 0.005000] +2024-04-06 19:42:07,539 - Train: 28.23% [536900/1902000] [282.3/1000.0] [batch_t 1.407 (1.414)] [data_t 0.002] [optim_t 1.405] [lr 0.005000] +2024-04-06 19:44:28,934 - Train: 28.23% [537000/1902000] [282.3/1000.0] [batch_t 1.414 (1.414)] [data_t 0.002] [optim_t 1.412] [lr 0.005000] +2024-04-06 19:46:55,202 - Train: 28.24% [537100/1902000] [282.4/1000.0] [batch_t 1.413 (1.463)] [data_t 0.002] [optim_t 1.411] [lr 0.005000] +2024-04-06 19:49:16,555 - Train: 28.24% [537200/1902000] [282.4/1000.0] [batch_t 1.417 (1.413)] [data_t 0.002] [optim_t 1.415] [lr 0.005000] +2024-04-06 19:51:37,853 - Train: 28.25% [537300/1902000] [282.5/1000.0] [batch_t 1.409 (1.413)] [data_t 0.002] [optim_t 1.407] [lr 0.005000] +2024-04-06 19:53:59,215 - Train: 28.25% [537400/1902000] [282.5/1000.0] [batch_t 1.408 (1.414)] [data_t 0.002] [optim_t 1.405] [lr 0.005000] +2024-04-06 19:56:20,534 - Train: 28.26% [537500/1902000] [282.6/1000.0] [batch_t 1.420 (1.413)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-06 19:58:41,928 - Train: 28.26% [537600/1902000] [282.6/1000.0] [batch_t 1.413 (1.414)] [data_t 0.002] [optim_t 1.411] [lr 0.005000] +2024-04-06 20:01:03,312 - Train: 28.27% [537700/1902000] [282.7/1000.0] [batch_t 1.418 (1.414)] [data_t 0.002] [optim_t 1.416] [lr 0.005000] +2024-04-06 20:03:24,612 - Train: 28.28% [537800/1902000] [282.8/1000.0] [batch_t 1.418 (1.413)] [data_t 0.002] [optim_t 1.416] [lr 0.005000] +2024-04-06 20:05:45,995 - Train: 28.28% [537900/1902000] [282.8/1000.0] [batch_t 1.406 (1.414)] [data_t 0.002] [optim_t 1.404] [lr 0.005000] +2024-04-06 20:08:07,331 - Train: 28.29% [538000/1902000] [282.9/1000.0] [batch_t 1.408 (1.413)] [data_t 0.002] [optim_t 1.406] [lr 0.005000] +2024-04-06 20:10:28,717 - Train: 28.29% [538100/1902000] [282.9/1000.0] [batch_t 1.418 (1.414)] [data_t 0.002] [optim_t 1.416] [lr 0.005000] +2024-04-06 20:12:18,491 - Train: 28.30% [538200/1902000] [283.0/1000.0] [batch_t 0.603 (1.098)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 20:12:58,458 - ==> Total time: 4 days, 19:12:37 Eta: 12 days, 3:53:37 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 20:13:21,266 - Train: 28.30% [538300/1902000] [283.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 20:14:21,934 - Train: 28.31% [538400/1902000] [283.1/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 20:15:22,516 - Train: 28.31% [538500/1902000] [283.1/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 20:16:23,070 - Train: 28.32% [538600/1902000] [283.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 20:17:23,670 - Train: 28.32% [538700/1902000] [283.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 20:18:24,208 - Train: 28.33% [538800/1902000] [283.3/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 20:19:24,821 - Train: 28.33% [538900/1902000] [283.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 20:20:25,515 - Train: 28.34% [539000/1902000] [283.4/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 20:21:26,108 - Train: 28.34% [539100/1902000] [283.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 20:22:26,702 - Train: 28.35% [539200/1902000] [283.5/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 20:23:27,290 - Train: 28.35% [539300/1902000] [283.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 20:24:27,879 - Train: 28.36% [539400/1902000] [283.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 20:25:28,434 - Train: 28.36% [539500/1902000] [283.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 20:26:28,984 - Train: 28.37% [539600/1902000] [283.7/1000.0] [batch_t 0.609 (0.605)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-06 20:27:29,773 - Train: 28.38% [539700/1902000] [283.8/1000.0] [batch_t 0.607 (0.608)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-06 20:28:30,485 - Train: 28.38% [539800/1902000] [283.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-06 20:29:31,207 - Train: 28.39% [539900/1902000] [283.9/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 20:30:31,964 - Train: 28.39% [540000/1902000] [283.9/1000.0] [batch_t 0.607 (0.607)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-06 20:31:32,650 - Train: 28.40% [540100/1902000] [284.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-06 20:32:13,946 - ==> Total time: 4 days, 19:31:53 Eta: 12 days, 3:16:10 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 20:32:35,640 - Train: 28.40% [540200/1902000] [284.0/1000.0] [batch_t 0.608 (0.612)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-06 20:33:37,712 - Train: 28.41% [540300/1902000] [284.1/1000.0] [batch_t 0.605 (0.621)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 20:34:39,507 - Train: 28.41% [540400/1902000] [284.1/1000.0] [batch_t 0.616 (0.618)] [data_t 0.003] [optim_t 0.613] [lr 0.005000] +2024-04-06 20:35:40,230 - Train: 28.42% [540500/1902000] [284.2/1000.0] [batch_t 0.608 (0.607)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-06 20:36:40,929 - Train: 28.42% [540600/1902000] [284.2/1000.0] [batch_t 0.607 (0.607)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-06 20:37:41,666 - Train: 28.43% [540700/1902000] [284.3/1000.0] [batch_t 0.608 (0.607)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-06 20:38:42,386 - Train: 28.43% [540800/1902000] [284.3/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 20:39:43,079 - Train: 28.44% [540900/1902000] [284.4/1000.0] [batch_t 0.607 (0.607)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-06 20:40:43,942 - Train: 28.44% [541000/1902000] [284.4/1000.0] [batch_t 0.606 (0.609)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-06 20:41:44,636 - Train: 28.45% [541100/1902000] [284.5/1000.0] [batch_t 0.606 (0.607)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-06 20:42:45,312 - Train: 28.45% [541200/1902000] [284.5/1000.0] [batch_t 0.606 (0.607)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-06 20:43:46,009 - Train: 28.46% [541300/1902000] [284.6/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 20:44:47,171 - Train: 28.46% [541400/1902000] [284.6/1000.0] [batch_t 0.608 (0.612)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-06 20:45:48,072 - Train: 28.47% [541500/1902000] [284.7/1000.0] [batch_t 0.610 (0.609)] [data_t 0.003] [optim_t 0.607] [lr 0.005000] +2024-04-06 20:46:48,793 - Train: 28.48% [541600/1902000] [284.8/1000.0] [batch_t 0.606 (0.607)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-06 20:47:49,559 - Train: 28.48% [541700/1902000] [284.8/1000.0] [batch_t 0.610 (0.608)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-06 20:48:50,245 - Train: 28.49% [541800/1902000] [284.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-06 20:49:50,943 - Train: 28.49% [541900/1902000] [284.9/1000.0] [batch_t 0.605 (0.607)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-06 20:50:51,662 - Train: 28.50% [542000/1902000] [285.0/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 20:51:34,147 - ==> Total time: 4 days, 19:51:13 Eta: 12 days, 2:39:02 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 20:51:54,681 - Train: 28.50% [542100/1902000] [285.0/1000.0] [batch_t 0.605 (0.609)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-06 20:52:55,518 - Train: 28.51% [542200/1902000] [285.1/1000.0] [batch_t 0.608 (0.608)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-06 20:53:56,186 - Train: 28.51% [542300/1902000] [285.1/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 20:54:56,863 - Train: 28.52% [542400/1902000] [285.2/1000.0] [batch_t 0.606 (0.607)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-06 20:55:57,545 - Train: 28.52% [542500/1902000] [285.2/1000.0] [batch_t 0.606 (0.607)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-06 20:56:58,252 - Train: 28.53% [542600/1902000] [285.3/1000.0] [batch_t 0.608 (0.607)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-06 20:57:58,937 - Train: 28.53% [542700/1902000] [285.3/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 20:59:02,037 - Train: 28.54% [542800/1902000] [285.4/1000.0] [batch_t 0.608 (0.631)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 21:00:03,156 - Train: 28.54% [542900/1902000] [285.4/1000.0] [batch_t 0.608 (0.611)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-06 21:01:03,935 - Train: 28.55% [543000/1902000] [285.5/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 21:02:04,905 - Train: 28.55% [543100/1902000] [285.5/1000.0] [batch_t 0.604 (0.610)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 21:03:05,559 - Train: 28.56% [543200/1902000] [285.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 21:04:06,241 - Train: 28.56% [543300/1902000] [285.6/1000.0] [batch_t 0.606 (0.607)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-06 21:05:07,318 - Train: 28.57% [543400/1902000] [285.7/1000.0] [batch_t 0.608 (0.611)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 21:06:08,925 - Train: 28.58% [543500/1902000] [285.8/1000.0] [batch_t 0.609 (0.616)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 21:07:09,718 - Train: 28.58% [543600/1902000] [285.8/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:08:10,351 - Train: 28.59% [543700/1902000] [285.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 21:09:10,991 - Train: 28.59% [543800/1902000] [285.9/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-06 21:10:11,656 - Train: 28.60% [543900/1902000] [286.0/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 21:10:55,318 - ==> Total time: 4 days, 20:10:34 Eta: 12 days, 2:02:04 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 21:11:14,320 - Train: 28.60% [544000/1902000] [286.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 21:12:15,100 - Train: 28.61% [544100/1902000] [286.1/1000.0] [batch_t 0.608 (0.608)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 21:13:15,712 - Train: 28.61% [544200/1902000] [286.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 21:14:16,359 - Train: 28.62% [544300/1902000] [286.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 21:15:16,988 - Train: 28.62% [544400/1902000] [286.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 21:16:17,607 - Train: 28.63% [544500/1902000] [286.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:17:18,223 - Train: 28.63% [544600/1902000] [286.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 21:18:18,826 - Train: 28.64% [544700/1902000] [286.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 21:19:19,444 - Train: 28.64% [544800/1902000] [286.4/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 21:20:20,167 - Train: 28.65% [544900/1902000] [286.5/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:21:20,758 - Train: 28.65% [545000/1902000] [286.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:22:21,372 - Train: 28.66% [545100/1902000] [286.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:23:22,006 - Train: 28.66% [545200/1902000] [286.6/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 21:24:22,623 - Train: 28.67% [545300/1902000] [286.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 21:25:23,290 - Train: 28.68% [545400/1902000] [286.8/1000.0] [batch_t 0.606 (0.607)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-06 21:26:23,906 - Train: 28.68% [545500/1902000] [286.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-06 21:27:24,527 - Train: 28.69% [545600/1902000] [286.9/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 21:28:25,129 - Train: 28.69% [545700/1902000] [286.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 21:29:25,747 - Train: 28.70% [545800/1902000] [287.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:30:10,644 - ==> Total time: 4 days, 20:29:50 Eta: 12 days, 1:24:59 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 21:30:28,725 - Train: 28.70% [545900/1902000] [287.0/1000.0] [batch_t 0.608 (0.610)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 21:31:29,307 - Train: 28.71% [546000/1902000] [287.1/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 21:32:30,031 - Train: 28.71% [546100/1902000] [287.1/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 21:33:30,653 - Train: 28.72% [546200/1902000] [287.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 21:34:31,259 - Train: 28.72% [546300/1902000] [287.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:35:31,877 - Train: 28.73% [546400/1902000] [287.3/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 21:36:32,552 - Train: 28.73% [546500/1902000] [287.3/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 21:37:33,289 - Train: 28.74% [546600/1902000] [287.4/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 21:38:33,878 - Train: 28.74% [546700/1902000] [287.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 21:39:34,487 - Train: 28.75% [546800/1902000] [287.5/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 21:40:35,098 - Train: 28.75% [546900/1902000] [287.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 21:41:35,710 - Train: 28.76% [547000/1902000] [287.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:42:36,332 - Train: 28.76% [547100/1902000] [287.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 21:43:36,925 - Train: 28.77% [547200/1902000] [287.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 21:44:37,533 - Train: 28.77% [547300/1902000] [287.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 21:45:38,252 - Train: 28.78% [547400/1902000] [287.8/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 21:46:38,857 - Train: 28.79% [547500/1902000] [287.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 21:47:39,466 - Train: 28.79% [547600/1902000] [287.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:48:40,065 - Train: 28.80% [547700/1902000] [288.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 21:49:26,137 - ==> Total time: 4 days, 20:49:05 Eta: 12 days, 0:48:02 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 21:49:42,777 - Train: 28.80% [547800/1902000] [288.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:50:43,485 - Train: 28.81% [547900/1902000] [288.1/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:51:44,124 - Train: 28.81% [548000/1902000] [288.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 21:52:44,773 - Train: 28.82% [548100/1902000] [288.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 21:53:45,399 - Train: 28.82% [548200/1902000] [288.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:54:46,033 - Train: 28.83% [548300/1902000] [288.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 21:55:46,775 - Train: 28.83% [548400/1902000] [288.3/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 21:56:47,368 - Train: 28.84% [548500/1902000] [288.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 21:57:48,042 - Train: 28.84% [548600/1902000] [288.4/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 21:58:48,633 - Train: 28.85% [548700/1902000] [288.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 21:59:49,284 - Train: 28.85% [548800/1902000] [288.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:00:49,874 - Train: 28.86% [548900/1902000] [288.6/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 22:01:50,496 - Train: 28.86% [549000/1902000] [288.6/1000.0] [batch_t 0.608 (0.606)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-06 22:02:51,100 - Train: 28.87% [549100/1902000] [288.7/1000.0] [batch_t 0.608 (0.606)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-06 22:03:51,809 - Train: 28.87% [549200/1902000] [288.7/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 22:04:52,446 - Train: 28.88% [549300/1902000] [288.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:05:53,053 - Train: 28.89% [549400/1902000] [288.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:06:53,658 - Train: 28.89% [549500/1902000] [288.9/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 22:07:54,268 - Train: 28.90% [549600/1902000] [289.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 22:08:41,656 - ==> Total time: 4 days, 21:08:21 Eta: 12 days, 0:11:12 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 22:08:57,208 - Train: 28.90% [549700/1902000] [289.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:09:57,838 - Train: 28.91% [549800/1902000] [289.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 22:10:58,476 - Train: 28.91% [549900/1902000] [289.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:11:59,099 - Train: 28.92% [550000/1902000] [289.2/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 22:12:59,722 - Train: 28.92% [550100/1902000] [289.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:14:00,351 - Train: 28.93% [550200/1902000] [289.3/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 22:15:01,078 - Train: 28.93% [550300/1902000] [289.3/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:16:01,667 - Train: 28.94% [550400/1902000] [289.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 22:17:02,316 - Train: 28.94% [550500/1902000] [289.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:18:02,943 - Train: 28.95% [550600/1902000] [289.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 22:19:04,580 - Train: 28.95% [550700/1902000] [289.5/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:20:05,308 - Train: 28.96% [550800/1902000] [289.6/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:21:05,945 - Train: 28.96% [550900/1902000] [289.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 22:22:06,551 - Train: 28.97% [551000/1902000] [289.7/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 22:23:07,506 - Train: 28.97% [551100/1902000] [289.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:24:08,129 - Train: 28.98% [551200/1902000] [289.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:25:08,732 - Train: 28.99% [551300/1902000] [289.9/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 22:26:09,332 - Train: 28.99% [551400/1902000] [289.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:27:09,941 - Train: 29.00% [551500/1902000] [290.0/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 22:27:58,518 - ==> Total time: 4 days, 21:27:38 Eta: 11 days, 23:34:33 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 22:28:12,772 - Train: 29.00% [551600/1902000] [290.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 22:29:13,394 - Train: 29.01% [551700/1902000] [290.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:30:13,976 - Train: 29.01% [551800/1902000] [290.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 22:31:14,641 - Train: 29.02% [551900/1902000] [290.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:32:15,267 - Train: 29.02% [552000/1902000] [290.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:33:16,040 - Train: 29.03% [552100/1902000] [290.3/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:34:16,622 - Train: 29.03% [552200/1902000] [290.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:35:17,247 - Train: 29.04% [552300/1902000] [290.4/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 22:36:17,842 - Train: 29.04% [552400/1902000] [290.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:37:18,526 - Train: 29.05% [552500/1902000] [290.5/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 22:38:19,126 - Train: 29.05% [552600/1902000] [290.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:39:19,758 - Train: 29.06% [552700/1902000] [290.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 22:40:20,451 - Train: 29.06% [552800/1902000] [290.6/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 22:41:21,098 - Train: 29.07% [552900/1902000] [290.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:42:21,717 - Train: 29.07% [553000/1902000] [290.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:43:22,337 - Train: 29.08% [553100/1902000] [290.8/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 22:44:22,934 - Train: 29.09% [553200/1902000] [290.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:45:23,615 - Train: 29.09% [553300/1902000] [290.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:46:24,264 - Train: 29.10% [553400/1902000] [291.0/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 22:47:13,991 - ==> Total time: 4 days, 21:46:53 Eta: 11 days, 22:57:57 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 22:47:27,371 - Train: 29.10% [553500/1902000] [291.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:48:28,015 - Train: 29.11% [553600/1902000] [291.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:49:28,646 - Train: 29.11% [553700/1902000] [291.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:50:29,292 - Train: 29.12% [553800/1902000] [291.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:51:29,877 - Train: 29.12% [553900/1902000] [291.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 22:52:30,558 - Train: 29.13% [554000/1902000] [291.3/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 22:53:31,194 - Train: 29.13% [554100/1902000] [291.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:54:31,794 - Train: 29.14% [554200/1902000] [291.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:55:32,406 - Train: 29.14% [554300/1902000] [291.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 22:56:33,012 - Train: 29.15% [554400/1902000] [291.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:57:33,606 - Train: 29.15% [554500/1902000] [291.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 22:58:34,308 - Train: 29.16% [554600/1902000] [291.6/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 22:59:34,881 - Train: 29.16% [554700/1902000] [291.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 23:00:35,485 - Train: 29.17% [554800/1902000] [291.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 23:01:36,083 - Train: 29.17% [554900/1902000] [291.7/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 23:02:36,686 - Train: 29.18% [555000/1902000] [291.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 23:03:37,272 - Train: 29.19% [555100/1902000] [291.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 23:04:37,876 - Train: 29.19% [555200/1902000] [291.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 23:05:38,601 - Train: 29.20% [555300/1902000] [292.0/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 23:06:29,497 - ==> Total time: 4 days, 22:06:09 Eta: 11 days, 22:21:29 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 23:06:41,201 - Train: 29.20% [555400/1902000] [292.0/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 23:07:41,825 - Train: 29.21% [555500/1902000] [292.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 23:08:42,440 - Train: 29.21% [555600/1902000] [292.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 23:09:43,078 - Train: 29.22% [555700/1902000] [292.2/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 23:10:43,666 - Train: 29.22% [555800/1902000] [292.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 23:11:44,423 - Train: 29.23% [555900/1902000] [292.3/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 23:12:44,992 - Train: 29.23% [556000/1902000] [292.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 23:13:45,605 - Train: 29.24% [556100/1902000] [292.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 23:14:46,238 - Train: 29.24% [556200/1902000] [292.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 23:15:46,878 - Train: 29.25% [556300/1902000] [292.5/1000.0] [batch_t 0.604 (0.606)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-06 23:16:47,532 - Train: 29.25% [556400/1902000] [292.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 23:17:48,175 - Train: 29.26% [556500/1902000] [292.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 23:18:48,800 - Train: 29.26% [556600/1902000] [292.6/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-06 23:19:49,558 - Train: 29.27% [556700/1902000] [292.7/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 23:20:50,198 - Train: 29.27% [556800/1902000] [292.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 23:21:50,871 - Train: 29.28% [556900/1902000] [292.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 23:22:51,494 - Train: 29.28% [557000/1902000] [292.8/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 23:23:52,118 - Train: 29.29% [557100/1902000] [292.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 23:24:52,886 - Train: 29.30% [557200/1902000] [293.0/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 23:25:45,048 - ==> Total time: 4 days, 22:25:24 Eta: 11 days, 21:45:08 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 23:25:55,895 - Train: 29.30% [557300/1902000] [293.0/1000.0] [batch_t 0.608 (0.611)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 23:26:56,528 - Train: 29.31% [557400/1902000] [293.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-06 23:27:57,185 - Train: 29.31% [557500/1902000] [293.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 23:28:57,799 - Train: 29.32% [557600/1902000] [293.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 23:29:58,454 - Train: 29.32% [557700/1902000] [293.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 23:30:59,098 - Train: 29.33% [557800/1902000] [293.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 23:31:59,896 - Train: 29.33% [557900/1902000] [293.3/1000.0] [batch_t 0.608 (0.608)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-06 23:33:00,530 - Train: 29.34% [558000/1902000] [293.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 23:34:01,178 - Train: 29.34% [558100/1902000] [293.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 23:35:01,862 - Train: 29.35% [558200/1902000] [293.5/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 23:36:02,510 - Train: 29.35% [558300/1902000] [293.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 23:37:03,273 - Train: 29.36% [558400/1902000] [293.6/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 23:38:03,937 - Train: 29.36% [558500/1902000] [293.6/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 23:39:04,518 - Train: 29.37% [558600/1902000] [293.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-06 23:40:05,148 - Train: 29.37% [558700/1902000] [293.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 23:41:05,787 - Train: 29.38% [558800/1902000] [293.8/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 23:42:06,487 - Train: 29.38% [558900/1902000] [293.8/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 23:43:07,179 - Train: 29.39% [559000/1902000] [293.9/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 23:44:07,799 - Train: 29.40% [559100/1902000] [294.0/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-06 23:45:01,233 - ==> Total time: 4 days, 22:44:40 Eta: 11 days, 21:08:55 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-06 23:45:12,411 - Train: 29.40% [559200/1902000] [294.0/1000.0] [batch_t 0.605 (0.627)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 23:46:13,026 - Train: 29.41% [559300/1902000] [294.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 23:47:13,642 - Train: 29.41% [559400/1902000] [294.1/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 23:48:14,266 - Train: 29.42% [559500/1902000] [294.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-06 23:49:14,908 - Train: 29.42% [559600/1902000] [294.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 23:50:15,704 - Train: 29.43% [559700/1902000] [294.3/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 23:51:16,381 - Train: 29.43% [559800/1902000] [294.3/1000.0] [batch_t 0.604 (0.607)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-06 23:52:16,943 - Train: 29.44% [559900/1902000] [294.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-06 23:53:17,568 - Train: 29.44% [560000/1902000] [294.4/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 23:54:18,191 - Train: 29.45% [560100/1902000] [294.5/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 23:55:18,842 - Train: 29.45% [560200/1902000] [294.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 23:56:19,486 - Train: 29.46% [560300/1902000] [294.6/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-06 23:57:20,168 - Train: 29.46% [560400/1902000] [294.6/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-06 23:58:20,771 - Train: 29.47% [560500/1902000] [294.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-06 23:59:21,427 - Train: 29.47% [560600/1902000] [294.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 00:00:22,095 - Train: 29.48% [560700/1902000] [294.8/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:01:22,714 - Train: 29.48% [560800/1902000] [294.8/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-07 00:02:23,337 - Train: 29.49% [560900/1902000] [294.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 00:03:24,064 - Train: 29.50% [561000/1902000] [295.0/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 00:04:18,634 - ==> Total time: 4 days, 23:03:58 Eta: 11 days, 20:32:52 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 00:04:26,711 - Train: 29.50% [561100/1902000] [295.0/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:05:27,331 - Train: 29.51% [561200/1902000] [295.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 00:06:27,962 - Train: 29.51% [561300/1902000] [295.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:07:28,565 - Train: 29.52% [561400/1902000] [295.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:08:29,311 - Train: 29.52% [561500/1902000] [295.2/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 00:09:29,932 - Train: 29.53% [561600/1902000] [295.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 00:10:30,548 - Train: 29.53% [561700/1902000] [295.3/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 00:11:31,142 - Train: 29.54% [561800/1902000] [295.4/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 00:12:31,748 - Train: 29.54% [561900/1902000] [295.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 00:13:32,329 - Train: 29.55% [562000/1902000] [295.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:14:32,960 - Train: 29.55% [562100/1902000] [295.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:15:33,595 - Train: 29.56% [562200/1902000] [295.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:16:34,252 - Train: 29.56% [562300/1902000] [295.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:17:34,873 - Train: 29.57% [562400/1902000] [295.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 00:18:35,499 - Train: 29.57% [562500/1902000] [295.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.003] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:19:36,107 - Train: 29.58% [562600/1902000] [295.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:20:36,739 - Train: 29.58% [562700/1902000] [295.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:21:37,435 - Train: 29.59% [562800/1902000] [295.9/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:22:38,069 - Train: 29.60% [562900/1902000] [296.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:23:33,842 - ==> Total time: 4 days, 23:23:13 Eta: 11 days, 19:56:51 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 00:23:40,983 - Train: 29.60% [563000/1902000] [296.0/1000.0] [batch_t 0.605 (0.622)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:24:41,581 - Train: 29.61% [563100/1902000] [296.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 00:25:42,189 - Train: 29.61% [563200/1902000] [296.1/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 00:26:42,766 - Train: 29.62% [563300/1902000] [296.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 00:27:43,494 - Train: 29.62% [563400/1902000] [296.2/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:28:44,090 - Train: 29.63% [563500/1902000] [296.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:29:44,707 - Train: 29.63% [563600/1902000] [296.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:30:45,322 - Train: 29.64% [563700/1902000] [296.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 00:31:45,971 - Train: 29.64% [563800/1902000] [296.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:32:46,569 - Train: 29.65% [563900/1902000] [296.5/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 00:33:47,182 - Train: 29.65% [564000/1902000] [296.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:34:47,873 - Train: 29.66% [564100/1902000] [296.6/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:35:48,462 - Train: 29.66% [564200/1902000] [296.6/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 00:36:49,069 - Train: 29.67% [564300/1902000] [296.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:37:49,692 - Train: 29.67% [564400/1902000] [296.7/1000.0] [batch_t 0.601 (0.606)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-07 00:38:50,302 - Train: 29.68% [564500/1902000] [296.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 00:39:50,930 - Train: 29.68% [564600/1902000] [296.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:40:51,647 - Train: 29.69% [564700/1902000] [296.9/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 00:41:52,279 - Train: 29.70% [564800/1902000] [297.0/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 00:42:49,259 - ==> Total time: 4 days, 23:42:28 Eta: 11 days, 19:20:57 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 00:42:55,040 - Train: 29.70% [564900/1902000] [297.0/1000.0] [batch_t 0.605 (0.619)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:43:55,656 - Train: 29.71% [565000/1902000] [297.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:44:56,246 - Train: 29.71% [565100/1902000] [297.1/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 00:45:56,862 - Train: 29.72% [565200/1902000] [297.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:46:57,478 - Train: 29.72% [565300/1902000] [297.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 00:47:58,211 - Train: 29.73% [565400/1902000] [297.3/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:48:58,785 - Train: 29.73% [565500/1902000] [297.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:49:59,384 - Train: 29.74% [565600/1902000] [297.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:50:59,997 - Train: 29.74% [565700/1902000] [297.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:52:00,611 - Train: 29.75% [565800/1902000] [297.5/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 00:53:01,328 - Train: 29.75% [565900/1902000] [297.5/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:54:01,973 - Train: 29.76% [566000/1902000] [297.6/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 00:55:02,585 - Train: 29.76% [566100/1902000] [297.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:56:03,208 - Train: 29.77% [566200/1902000] [297.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 00:57:03,807 - Train: 29.77% [566300/1902000] [297.7/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 00:58:04,402 - Train: 29.78% [566400/1902000] [297.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 00:59:05,023 - Train: 29.78% [566500/1902000] [297.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:00:05,758 - Train: 29.79% [566600/1902000] [297.9/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 01:01:06,407 - Train: 29.79% [566700/1902000] [297.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:02:04,570 - ==> Total time: 5 days, 0:01:44 Eta: 11 days, 18:45:09 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 01:02:09,180 - Train: 29.80% [566800/1902000] [298.0/1000.0] [batch_t 0.605 (0.626)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 01:03:09,818 - Train: 29.81% [566900/1902000] [298.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:04:10,425 - Train: 29.81% [567000/1902000] [298.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 01:05:11,199 - Train: 29.82% [567100/1902000] [298.2/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:06:11,809 - Train: 29.82% [567200/1902000] [298.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 01:07:12,423 - Train: 29.83% [567300/1902000] [298.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:08:13,020 - Train: 29.83% [567400/1902000] [298.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 01:09:13,620 - Train: 29.84% [567500/1902000] [298.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 01:10:14,236 - Train: 29.84% [567600/1902000] [298.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 01:11:14,829 - Train: 29.85% [567700/1902000] [298.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:12:15,484 - Train: 29.85% [567800/1902000] [298.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 01:13:16,173 - Train: 29.86% [567900/1902000] [298.6/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:14:16,774 - Train: 29.86% [568000/1902000] [298.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 01:15:17,369 - Train: 29.87% [568100/1902000] [298.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 01:16:17,964 - Train: 29.87% [568200/1902000] [298.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:17:18,589 - Train: 29.88% [568300/1902000] [298.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 01:18:19,330 - Train: 29.88% [568400/1902000] [298.8/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 01:19:19,918 - Train: 29.89% [568500/1902000] [298.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:20:20,508 - Train: 29.89% [568600/1902000] [298.9/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 01:21:19,903 - ==> Total time: 5 days, 0:20:59 Eta: 11 days, 18:09:28 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 01:21:23,058 - Train: 29.90% [568700/1902000] [299.0/1000.0] [batch_t 0.608 (0.636)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 01:22:23,665 - Train: 29.91% [568800/1902000] [299.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 01:23:24,258 - Train: 29.91% [568900/1902000] [299.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:24:25,008 - Train: 29.92% [569000/1902000] [299.2/1000.0] [batch_t 0.607 (0.607)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:25:25,626 - Train: 29.92% [569100/1902000] [299.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:26:26,232 - Train: 29.93% [569200/1902000] [299.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 01:27:26,844 - Train: 29.93% [569300/1902000] [299.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:28:27,477 - Train: 29.94% [569400/1902000] [299.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:29:28,088 - Train: 29.94% [569500/1902000] [299.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:30:28,698 - Train: 29.95% [569600/1902000] [299.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 01:31:29,290 - Train: 29.95% [569700/1902000] [299.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 01:32:29,969 - Train: 29.96% [569800/1902000] [299.6/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 01:33:30,597 - Train: 29.96% [569900/1902000] [299.6/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 01:34:31,229 - Train: 29.97% [570000/1902000] [299.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:35:31,854 - Train: 29.97% [570100/1902000] [299.7/1000.0] [batch_t 0.608 (0.606)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-07 01:36:32,457 - Train: 29.98% [570200/1902000] [299.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 01:37:33,146 - Train: 29.98% [570300/1902000] [299.8/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 01:38:33,732 - Train: 29.99% [570400/1902000] [299.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 01:39:34,338 - Train: 29.99% [570500/1902000] [299.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 01:40:34,940 - Train: 30.00% [570600/1902000] [300.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:40:34,943 - ==> Total time: 5 days, 0:40:14 Eta: 11 days, 17:33:53 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 01:41:37,551 - Train: 30.01% [570700/1902000] [300.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 01:42:38,158 - Train: 30.01% [570800/1902000] [300.1/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 01:43:38,770 - Train: 30.02% [570900/1902000] [300.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 01:44:39,469 - Train: 30.02% [571000/1902000] [300.2/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 01:45:40,072 - Train: 30.03% [571100/1902000] [300.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 01:46:40,690 - Train: 30.03% [571200/1902000] [300.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 01:47:41,288 - Train: 30.04% [571300/1902000] [300.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:48:41,920 - Train: 30.04% [571400/1902000] [300.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 01:49:42,700 - Train: 30.05% [571500/1902000] [300.5/1000.0] [batch_t 0.603 (0.608)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 01:50:43,333 - Train: 30.05% [571600/1902000] [300.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 01:51:43,998 - Train: 30.06% [571700/1902000] [300.6/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 01:52:44,635 - Train: 30.06% [571800/1902000] [300.6/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 01:53:45,256 - Train: 30.07% [571900/1902000] [300.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 01:54:45,876 - Train: 30.07% [572000/1902000] [300.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 01:55:46,559 - Train: 30.08% [572100/1902000] [300.8/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 01:56:47,202 - Train: 30.08% [572200/1902000] [300.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 01:57:47,977 - Train: 30.09% [572300/1902000] [300.9/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 01:58:48,618 - Train: 30.09% [572400/1902000] [300.9/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 01:59:49,264 - Train: 30.10% [572500/1902000] [301.0/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-07 01:59:50,479 - ==> Total time: 5 days, 0:59:29 Eta: 11 days, 16:58:26 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 02:00:52,051 - Train: 30.11% [572600/1902000] [301.1/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 02:01:52,682 - Train: 30.11% [572700/1902000] [301.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:02:53,465 - Train: 30.12% [572800/1902000] [301.2/1000.0] [batch_t 0.602 (0.608)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-07 02:03:54,097 - Train: 30.12% [572900/1902000] [301.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:04:54,700 - Train: 30.13% [573000/1902000] [301.3/1000.0] [batch_t 0.608 (0.606)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-07 02:05:55,336 - Train: 30.13% [573100/1902000] [301.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 02:06:55,968 - Train: 30.14% [573200/1902000] [301.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:07:56,599 - Train: 30.14% [573300/1902000] [301.4/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 02:08:57,232 - Train: 30.15% [573400/1902000] [301.5/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 02:09:58,005 - Train: 30.15% [573500/1902000] [301.5/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 02:10:58,629 - Train: 30.16% [573600/1902000] [301.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:11:59,256 - Train: 30.16% [573700/1902000] [301.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:12:59,904 - Train: 30.17% [573800/1902000] [301.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-07 02:14:00,525 - Train: 30.17% [573900/1902000] [301.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 02:15:01,155 - Train: 30.18% [574000/1902000] [301.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 02:16:01,955 - Train: 30.18% [574100/1902000] [301.8/1000.0] [batch_t 0.607 (0.608)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:17:02,573 - Train: 30.19% [574200/1902000] [301.9/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 02:18:03,198 - Train: 30.19% [574300/1902000] [301.9/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 02:19:03,807 - Train: 30.20% [574400/1902000] [302.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:19:06,236 - ==> Total time: 5 days, 1:18:45 Eta: 11 days, 16:23:06 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 02:20:06,487 - Train: 30.21% [574500/1902000] [302.1/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:21:07,097 - Train: 30.21% [574600/1902000] [302.1/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 02:22:07,816 - Train: 30.22% [574700/1902000] [302.2/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 02:23:08,423 - Train: 30.22% [574800/1902000] [302.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 02:24:09,010 - Train: 30.23% [574900/1902000] [302.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 02:25:09,601 - Train: 30.23% [575000/1902000] [302.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 02:26:10,303 - Train: 30.24% [575100/1902000] [302.4/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 02:27:11,021 - Train: 30.24% [575200/1902000] [302.4/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:28:11,703 - Train: 30.25% [575300/1902000] [302.5/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 02:29:12,339 - Train: 30.25% [575400/1902000] [302.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 02:30:12,972 - Train: 30.26% [575500/1902000] [302.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 02:31:13,584 - Train: 30.26% [575600/1902000] [302.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 02:32:14,212 - Train: 30.27% [575700/1902000] [302.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 02:33:14,821 - Train: 30.27% [575800/1902000] [302.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 02:34:15,428 - Train: 30.28% [575900/1902000] [302.8/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 02:35:16,165 - Train: 30.28% [576000/1902000] [302.8/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 02:36:16,774 - Train: 30.29% [576100/1902000] [302.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 02:37:17,377 - Train: 30.29% [576200/1902000] [302.9/1000.0] [batch_t 0.609 (0.606)] [data_t 0.003] [optim_t 0.607] [lr 0.005000] +2024-04-07 02:38:18,022 - Train: 30.30% [576300/1902000] [303.0/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 02:38:21,667 - ==> Total time: 5 days, 1:38:01 Eta: 11 days, 15:47:51 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 02:39:20,582 - Train: 30.30% [576400/1902000] [303.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:40:21,321 - Train: 30.31% [576500/1902000] [303.1/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:41:21,919 - Train: 30.32% [576600/1902000] [303.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 02:42:22,510 - Train: 30.32% [576700/1902000] [303.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 02:43:23,078 - Train: 30.33% [576800/1902000] [303.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 02:44:23,658 - Train: 30.33% [576900/1902000] [303.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:45:24,337 - Train: 30.34% [577000/1902000] [303.4/1000.0] [batch_t 0.603 (0.607)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 02:46:24,948 - Train: 30.34% [577100/1902000] [303.4/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 02:47:25,565 - Train: 30.35% [577200/1902000] [303.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 02:48:26,187 - Train: 30.35% [577300/1902000] [303.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:49:26,791 - Train: 30.36% [577400/1902000] [303.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:50:27,436 - Train: 30.36% [577500/1902000] [303.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:51:28,060 - Train: 30.37% [577600/1902000] [303.7/1000.0] [batch_t 0.601 (0.606)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-07 02:52:28,649 - Train: 30.37% [577700/1902000] [303.7/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 02:53:29,461 - Train: 30.38% [577800/1902000] [303.8/1000.0] [batch_t 0.603 (0.608)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 02:54:30,096 - Train: 30.38% [577900/1902000] [303.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 02:55:30,743 - Train: 30.39% [578000/1902000] [303.9/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 02:56:31,395 - Train: 30.39% [578100/1902000] [303.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:57:32,066 - Train: 30.40% [578200/1902000] [304.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:57:36,919 - ==> Total time: 5 days, 1:57:16 Eta: 11 days, 15:12:42 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 02:58:35,046 - Train: 30.40% [578300/1902000] [304.0/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 02:59:35,670 - Train: 30.41% [578400/1902000] [304.1/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 03:00:36,288 - Train: 30.42% [578500/1902000] [304.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 03:01:36,907 - Train: 30.42% [578600/1902000] [304.2/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 03:02:37,557 - Train: 30.43% [578700/1902000] [304.3/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 03:03:38,150 - Train: 30.43% [578800/1902000] [304.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:04:38,833 - Train: 30.44% [578900/1902000] [304.4/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 03:05:39,428 - Train: 30.44% [579000/1902000] [304.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 03:06:40,181 - Train: 30.45% [579100/1902000] [304.5/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:07:40,825 - Train: 30.45% [579200/1902000] [304.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-07 03:08:41,450 - Train: 30.46% [579300/1902000] [304.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 03:09:42,060 - Train: 30.46% [579400/1902000] [304.6/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 03:10:42,675 - Train: 30.47% [579500/1902000] [304.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 03:11:43,461 - Train: 30.47% [579600/1902000] [304.7/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 03:12:44,122 - Train: 30.48% [579700/1902000] [304.8/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:13:44,737 - Train: 30.48% [579800/1902000] [304.8/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 03:14:45,324 - Train: 30.49% [579900/1902000] [304.9/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 03:15:45,921 - Train: 30.49% [580000/1902000] [304.9/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 03:16:46,552 - Train: 30.50% [580100/1902000] [305.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 03:16:52,611 - ==> Total time: 5 days, 2:16:32 Eta: 11 days, 14:37:40 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 03:17:49,156 - Train: 30.50% [580200/1902000] [305.0/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 03:18:49,903 - Train: 30.51% [580300/1902000] [305.1/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 03:19:50,503 - Train: 30.52% [580400/1902000] [305.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 03:20:51,103 - Train: 30.52% [580500/1902000] [305.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:21:51,761 - Train: 30.53% [580600/1902000] [305.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 03:22:52,337 - Train: 30.53% [580700/1902000] [305.3/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:23:53,028 - Train: 30.54% [580800/1902000] [305.4/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:24:53,631 - Train: 30.54% [580900/1902000] [305.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:25:54,220 - Train: 30.55% [581000/1902000] [305.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 03:26:54,829 - Train: 30.55% [581100/1902000] [305.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 03:27:55,466 - Train: 30.56% [581200/1902000] [305.6/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 03:28:56,117 - Train: 30.56% [581300/1902000] [305.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 03:29:56,790 - Train: 30.57% [581400/1902000] [305.7/1000.0] [batch_t 0.606 (0.607)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-07 03:30:57,416 - Train: 30.57% [581500/1902000] [305.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 03:31:58,208 - Train: 30.58% [581600/1902000] [305.8/1000.0] [batch_t 0.607 (0.608)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-07 03:32:58,806 - Train: 30.58% [581700/1902000] [305.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 03:33:59,425 - Train: 30.59% [581800/1902000] [305.9/1000.0] [batch_t 0.603 (0.606)] [data_t 0.003] [optim_t 0.601] [lr 0.005000] +2024-04-07 03:35:00,078 - Train: 30.59% [581900/1902000] [305.9/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 03:36:00,763 - Train: 30.60% [582000/1902000] [306.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:36:08,040 - ==> Total time: 5 days, 2:35:47 Eta: 11 days, 14:02:44 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 03:37:03,566 - Train: 30.60% [582100/1902000] [306.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:38:04,188 - Train: 30.61% [582200/1902000] [306.1/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 03:39:04,905 - Train: 30.62% [582300/1902000] [306.2/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:40:05,548 - Train: 30.62% [582400/1902000] [306.2/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 03:41:06,172 - Train: 30.63% [582500/1902000] [306.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 03:42:06,864 - Train: 30.63% [582600/1902000] [306.3/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:43:07,490 - Train: 30.64% [582700/1902000] [306.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:44:08,132 - Train: 30.64% [582800/1902000] [306.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 03:45:08,767 - Train: 30.65% [582900/1902000] [306.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 03:46:09,430 - Train: 30.65% [583000/1902000] [306.5/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:47:10,045 - Train: 30.66% [583100/1902000] [306.6/1000.0] [batch_t 0.609 (0.606)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-07 03:48:10,690 - Train: 30.66% [583200/1902000] [306.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:49:11,348 - Train: 30.67% [583300/1902000] [306.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:50:12,048 - Train: 30.67% [583400/1902000] [306.7/1000.0] [batch_t 0.608 (0.607)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-07 03:51:12,670 - Train: 30.68% [583500/1902000] [306.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 03:52:13,289 - Train: 30.68% [583600/1902000] [306.8/1000.0] [batch_t 0.609 (0.606)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-07 03:53:13,891 - Train: 30.69% [583700/1902000] [306.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 03:54:14,484 - Train: 30.69% [583800/1902000] [306.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 03:55:15,213 - Train: 30.70% [583900/1902000] [307.0/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 03:55:23,711 - ==> Total time: 5 days, 2:55:03 Eta: 11 days, 13:27:55 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 03:56:18,069 - Train: 30.70% [584000/1902000] [307.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 03:57:18,703 - Train: 30.71% [584100/1902000] [307.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 03:58:19,335 - Train: 30.72% [584200/1902000] [307.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 03:59:19,992 - Train: 30.72% [584300/1902000] [307.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 04:00:20,618 - Train: 30.73% [584400/1902000] [307.3/1000.0] [batch_t 0.601 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 04:01:21,273 - Train: 30.73% [584500/1902000] [307.3/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-07 04:02:22,036 - Train: 30.74% [584600/1902000] [307.4/1000.0] [batch_t 0.608 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 04:03:22,674 - Train: 30.74% [584700/1902000] [307.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 04:04:23,301 - Train: 30.75% [584800/1902000] [307.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 04:05:23,911 - Train: 30.75% [584900/1902000] [307.5/1000.0] [batch_t 0.608 (0.606)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-07 04:06:24,539 - Train: 30.76% [585000/1902000] [307.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 04:07:25,268 - Train: 30.76% [585100/1902000] [307.6/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:08:25,907 - Train: 30.77% [585200/1902000] [307.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:09:26,521 - Train: 30.77% [585300/1902000] [307.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 04:10:27,134 - Train: 30.78% [585400/1902000] [307.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 04:11:27,750 - Train: 30.78% [585500/1902000] [307.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:12:28,405 - Train: 30.79% [585600/1902000] [307.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 04:13:29,018 - Train: 30.79% [585700/1902000] [307.9/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 04:14:29,628 - Train: 30.80% [585800/1902000] [308.0/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 04:14:39,395 - ==> Total time: 5 days, 3:14:18 Eta: 11 days, 12:53:12 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 04:15:32,506 - Train: 30.80% [585900/1902000] [308.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 04:16:33,179 - Train: 30.81% [586000/1902000] [308.1/1000.0] [batch_t 0.612 (0.607)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-07 04:17:33,810 - Train: 30.81% [586100/1902000] [308.1/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 04:18:34,431 - Train: 30.82% [586200/1902000] [308.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:19:35,042 - Train: 30.83% [586300/1902000] [308.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 04:20:35,675 - Train: 30.83% [586400/1902000] [308.3/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 04:21:36,384 - Train: 30.84% [586500/1902000] [308.4/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 04:22:36,996 - Train: 30.84% [586600/1902000] [308.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:23:37,645 - Train: 30.85% [586700/1902000] [308.5/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 04:24:38,281 - Train: 30.85% [586800/1902000] [308.5/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 04:25:38,903 - Train: 30.86% [586900/1902000] [308.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 04:26:39,526 - Train: 30.86% [587000/1902000] [308.6/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 04:27:40,240 - Train: 30.87% [587100/1902000] [308.7/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 04:28:40,881 - Train: 30.87% [587200/1902000] [308.7/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 04:29:41,514 - Train: 30.88% [587300/1902000] [308.8/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 04:30:42,165 - Train: 30.88% [587400/1902000] [308.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 04:31:42,848 - Train: 30.89% [587500/1902000] [308.9/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 04:32:43,492 - Train: 30.89% [587600/1902000] [308.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 04:33:44,128 - Train: 30.90% [587700/1902000] [309.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:33:55,066 - ==> Total time: 5 days, 3:33:34 Eta: 11 days, 12:18:34 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 04:34:47,014 - Train: 30.90% [587800/1902000] [309.0/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:35:47,652 - Train: 30.91% [587900/1902000] [309.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 04:36:48,266 - Train: 30.91% [588000/1902000] [309.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 04:37:48,884 - Train: 30.92% [588100/1902000] [309.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:38:49,488 - Train: 30.93% [588200/1902000] [309.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 04:39:50,152 - Train: 30.93% [588300/1902000] [309.3/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:40:50,773 - Train: 30.94% [588400/1902000] [309.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:41:51,500 - Train: 30.94% [588500/1902000] [309.4/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 04:42:52,112 - Train: 30.95% [588600/1902000] [309.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:43:52,718 - Train: 30.95% [588700/1902000] [309.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:44:53,310 - Train: 30.96% [588800/1902000] [309.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:45:53,923 - Train: 30.96% [588900/1902000] [309.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:46:54,572 - Train: 30.97% [589000/1902000] [309.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 04:47:55,275 - Train: 30.97% [589100/1902000] [309.7/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 04:48:55,948 - Train: 30.98% [589200/1902000] [309.8/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:49:56,540 - Train: 30.98% [589300/1902000] [309.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:50:57,154 - Train: 30.99% [589400/1902000] [309.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 04:51:57,777 - Train: 30.99% [589500/1902000] [309.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 04:52:58,411 - Train: 31.00% [589600/1902000] [310.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 04:53:10,539 - ==> Total time: 5 days, 3:52:50 Eta: 11 days, 11:44:03 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 04:54:01,351 - Train: 31.00% [589700/1902000] [310.0/1000.0] [batch_t 0.606 (0.609)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:55:02,013 - Train: 31.01% [589800/1902000] [310.1/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 04:56:02,652 - Train: 31.01% [589900/1902000] [310.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 04:57:03,271 - Train: 31.02% [590000/1902000] [310.2/1000.0] [batch_t 0.608 (0.606)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-07 04:58:03,871 - Train: 31.03% [590100/1902000] [310.3/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-07 04:59:04,494 - Train: 31.03% [590200/1902000] [310.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:00:05,154 - Train: 31.04% [590300/1902000] [310.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 05:01:05,815 - Train: 31.04% [590400/1902000] [310.4/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:02:06,492 - Train: 31.05% [590500/1902000] [310.5/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 05:03:07,121 - Train: 31.05% [590600/1902000] [310.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 05:04:07,753 - Train: 31.06% [590700/1902000] [310.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:05:08,381 - Train: 31.06% [590800/1902000] [310.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 05:06:08,994 - Train: 31.07% [590900/1902000] [310.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 05:07:09,629 - Train: 31.07% [591000/1902000] [310.7/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 05:08:10,286 - Train: 31.08% [591100/1902000] [310.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 05:09:11,015 - Train: 31.08% [591200/1902000] [310.8/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 05:10:11,627 - Train: 31.09% [591300/1902000] [310.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 05:11:12,238 - Train: 31.09% [591400/1902000] [310.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:12:12,860 - Train: 31.10% [591500/1902000] [311.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:12:26,178 - ==> Total time: 5 days, 4:12:05 Eta: 11 days, 11:09:37 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 05:13:15,508 - Train: 31.10% [591600/1902000] [311.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 05:14:16,202 - Train: 31.11% [591700/1902000] [311.1/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:15:16,802 - Train: 31.11% [591800/1902000] [311.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 05:16:17,362 - Train: 31.12% [591900/1902000] [311.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 05:17:17,909 - Train: 31.13% [592000/1902000] [311.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 05:18:18,487 - Train: 31.13% [592100/1902000] [311.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:19:19,044 - Train: 31.14% [592200/1902000] [311.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:20:19,618 - Train: 31.14% [592300/1902000] [311.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 05:21:20,206 - Train: 31.15% [592400/1902000] [311.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 05:22:20,831 - Train: 31.15% [592500/1902000] [311.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 05:23:21,389 - Train: 31.16% [592600/1902000] [311.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 05:24:21,967 - Train: 31.16% [592700/1902000] [311.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:25:22,553 - Train: 31.17% [592800/1902000] [311.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 05:26:23,120 - Train: 31.17% [592900/1902000] [311.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:27:23,843 - Train: 31.18% [593000/1902000] [311.8/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 05:28:24,397 - Train: 31.18% [593100/1902000] [311.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 05:29:24,987 - Train: 31.19% [593200/1902000] [311.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 05:30:25,609 - Train: 31.19% [593300/1902000] [311.9/1000.0] [batch_t 0.611 (0.606)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-07 05:31:26,166 - Train: 31.20% [593400/1902000] [312.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:31:40,712 - ==> Total time: 5 days, 4:31:20 Eta: 11 days, 10:35:15 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 05:32:28,728 - Train: 31.20% [593500/1902000] [312.0/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 05:33:29,290 - Train: 31.21% [593600/1902000] [312.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 05:34:30,013 - Train: 31.21% [593700/1902000] [312.1/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:35:30,560 - Train: 31.22% [593800/1902000] [312.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 05:36:31,118 - Train: 31.23% [593900/1902000] [312.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 05:37:31,705 - Train: 31.23% [594000/1902000] [312.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 05:38:32,279 - Train: 31.24% [594100/1902000] [312.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 05:39:32,965 - Train: 31.24% [594200/1902000] [312.4/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:40:33,550 - Train: 31.25% [594300/1902000] [312.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 05:41:34,087 - Train: 31.25% [594400/1902000] [312.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:42:34,673 - Train: 31.26% [594500/1902000] [312.6/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 05:43:35,251 - Train: 31.26% [594600/1902000] [312.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 05:44:35,787 - Train: 31.27% [594700/1902000] [312.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:45:36,355 - Train: 31.27% [594800/1902000] [312.7/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 05:46:36,926 - Train: 31.28% [594900/1902000] [312.8/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 05:47:37,558 - Train: 31.28% [595000/1902000] [312.8/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 05:48:38,136 - Train: 31.29% [595100/1902000] [312.9/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 05:49:38,691 - Train: 31.29% [595200/1902000] [312.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 05:50:39,238 - Train: 31.30% [595300/1902000] [313.0/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 05:50:54,997 - ==> Total time: 5 days, 4:50:34 Eta: 11 days, 10:00:58 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 05:51:41,793 - Train: 31.30% [595400/1902000] [313.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 05:52:42,438 - Train: 31.31% [595500/1902000] [313.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 05:53:42,978 - Train: 31.31% [595600/1902000] [313.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 05:54:43,551 - Train: 31.32% [595700/1902000] [313.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 05:55:44,127 - Train: 31.32% [595800/1902000] [313.2/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 05:56:44,668 - Train: 31.33% [595900/1902000] [313.3/1000.0] [batch_t 0.609 (0.605)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 05:57:45,416 - Train: 31.34% [596000/1902000] [313.4/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 05:58:45,963 - Train: 31.34% [596100/1902000] [313.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 05:59:46,508 - Train: 31.35% [596200/1902000] [313.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 06:00:47,062 - Train: 31.35% [596300/1902000] [313.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:01:47,611 - Train: 31.36% [596400/1902000] [313.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 06:02:48,164 - Train: 31.36% [596500/1902000] [313.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 06:03:48,734 - Train: 31.37% [596600/1902000] [313.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 06:04:49,294 - Train: 31.37% [596700/1902000] [313.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 06:05:49,932 - Train: 31.38% [596800/1902000] [313.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 06:06:50,518 - Train: 31.38% [596900/1902000] [313.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 06:07:51,075 - Train: 31.39% [597000/1902000] [313.9/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 06:08:51,662 - Train: 31.39% [597100/1902000] [313.9/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 06:09:52,222 - Train: 31.40% [597200/1902000] [314.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 06:10:09,183 - ==> Total time: 5 days, 5:09:48 Eta: 11 days, 9:26:47 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 06:10:55,167 - Train: 31.40% [597300/1902000] [314.0/1000.0] [batch_t 0.609 (0.608)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 06:11:55,754 - Train: 31.41% [597400/1902000] [314.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 06:12:56,312 - Train: 31.41% [597500/1902000] [314.1/1000.0] [batch_t 0.611 (0.605)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-07 06:13:56,947 - Train: 31.42% [597600/1902000] [314.2/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 06:14:57,502 - Train: 31.42% [597700/1902000] [314.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 06:15:58,052 - Train: 31.43% [597800/1902000] [314.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 06:16:58,584 - Train: 31.44% [597900/1902000] [314.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 06:17:59,222 - Train: 31.44% [598000/1902000] [314.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 06:18:59,781 - Train: 31.45% [598100/1902000] [314.5/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 06:20:00,376 - Train: 31.45% [598200/1902000] [314.5/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 06:21:01,002 - Train: 31.46% [598300/1902000] [314.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:22:01,564 - Train: 31.46% [598400/1902000] [314.6/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 06:23:02,171 - Train: 31.47% [598500/1902000] [314.7/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 06:24:02,808 - Train: 31.47% [598600/1902000] [314.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 06:25:03,339 - Train: 31.48% [598700/1902000] [314.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 06:26:03,905 - Train: 31.48% [598800/1902000] [314.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 06:27:04,451 - Train: 31.49% [598900/1902000] [314.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 06:28:05,033 - Train: 31.49% [599000/1902000] [314.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 06:29:05,584 - Train: 31.50% [599100/1902000] [315.0/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 06:29:23,749 - ==> Total time: 5 days, 5:29:03 Eta: 11 days, 8:52:42 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 06:30:08,305 - Train: 31.50% [599200/1902000] [315.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:31:08,976 - Train: 31.51% [599300/1902000] [315.1/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:32:09,549 - Train: 31.51% [599400/1902000] [315.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:33:10,114 - Train: 31.52% [599500/1902000] [315.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:34:10,698 - Train: 31.52% [599600/1902000] [315.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 06:35:11,279 - Train: 31.53% [599700/1902000] [315.3/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 06:36:11,969 - Train: 31.54% [599800/1902000] [315.4/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 06:37:12,492 - Train: 31.54% [599900/1902000] [315.4/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:38:13,020 - Train: 31.55% [600000/1902000] [315.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:39:13,548 - Train: 31.55% [600100/1902000] [315.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 06:40:14,076 - Train: 31.56% [600200/1902000] [315.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 06:41:14,584 - Train: 31.56% [600300/1902000] [315.6/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 06:42:15,139 - Train: 31.57% [600400/1902000] [315.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:43:15,682 - Train: 31.57% [600500/1902000] [315.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:44:16,337 - Train: 31.58% [600600/1902000] [315.8/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:45:16,882 - Train: 31.58% [600700/1902000] [315.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:46:17,438 - Train: 31.59% [600800/1902000] [315.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 06:47:17,964 - Train: 31.59% [600900/1902000] [315.9/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 06:48:18,498 - Train: 31.60% [601000/1902000] [316.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:48:37,879 - ==> Total time: 5 days, 5:48:17 Eta: 11 days, 8:18:42 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 06:49:21,350 - Train: 31.60% [601100/1902000] [316.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 06:50:21,887 - Train: 31.61% [601200/1902000] [316.1/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 06:51:22,477 - Train: 31.61% [601300/1902000] [316.1/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 06:52:23,002 - Train: 31.62% [601400/1902000] [316.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:53:23,578 - Train: 31.62% [601500/1902000] [316.2/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 06:54:24,324 - Train: 31.63% [601600/1902000] [316.3/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 06:55:24,905 - Train: 31.64% [601700/1902000] [316.4/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-07 06:56:25,468 - Train: 31.64% [601800/1902000] [316.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 06:57:26,018 - Train: 31.65% [601900/1902000] [316.5/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:58:26,572 - Train: 31.65% [602000/1902000] [316.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 06:59:27,149 - Train: 31.66% [602100/1902000] [316.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:00:27,731 - Train: 31.66% [602200/1902000] [316.6/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 07:01:28,291 - Train: 31.67% [602300/1902000] [316.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 07:02:28,974 - Train: 31.67% [602400/1902000] [316.7/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:03:29,517 - Train: 31.68% [602500/1902000] [316.8/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 07:04:30,111 - Train: 31.68% [602600/1902000] [316.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 07:05:30,713 - Train: 31.69% [602700/1902000] [316.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 07:06:31,288 - Train: 31.69% [602800/1902000] [316.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:07:31,950 - Train: 31.70% [602900/1902000] [317.0/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:07:52,562 - ==> Total time: 5 days, 6:07:32 Eta: 11 days, 7:44:48 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 07:08:34,649 - Train: 31.70% [603000/1902000] [317.0/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 07:09:35,213 - Train: 31.71% [603100/1902000] [317.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 07:10:35,732 - Train: 31.71% [603200/1902000] [317.1/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 07:11:36,312 - Train: 31.72% [603300/1902000] [317.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:12:36,992 - Train: 31.72% [603400/1902000] [317.2/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 07:13:37,554 - Train: 31.73% [603500/1902000] [317.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 07:14:38,138 - Train: 31.74% [603600/1902000] [317.4/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 07:15:38,694 - Train: 31.74% [603700/1902000] [317.4/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 07:16:39,241 - Train: 31.75% [603800/1902000] [317.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:17:39,801 - Train: 31.75% [603900/1902000] [317.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:18:40,323 - Train: 31.76% [604000/1902000] [317.6/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:19:40,901 - Train: 31.76% [604100/1902000] [317.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 07:20:41,508 - Train: 31.77% [604200/1902000] [317.7/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 07:21:42,069 - Train: 31.77% [604300/1902000] [317.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 07:22:42,599 - Train: 31.78% [604400/1902000] [317.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 07:23:43,141 - Train: 31.78% [604500/1902000] [317.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 07:24:43,684 - Train: 31.79% [604600/1902000] [317.9/1000.0] [batch_t 0.602 (0.605)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 07:25:44,331 - Train: 31.79% [604700/1902000] [317.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 07:26:44,877 - Train: 31.80% [604800/1902000] [318.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:27:06,693 - ==> Total time: 5 days, 6:26:46 Eta: 11 days, 7:10:59 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 07:27:47,550 - Train: 31.80% [604900/1902000] [318.0/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 07:28:48,095 - Train: 31.81% [605000/1902000] [318.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 07:29:48,642 - Train: 31.81% [605100/1902000] [318.1/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:30:49,194 - Train: 31.82% [605200/1902000] [318.2/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:31:49,720 - Train: 31.82% [605300/1902000] [318.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 07:32:50,394 - Train: 31.83% [605400/1902000] [318.3/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 07:33:50,970 - Train: 31.83% [605500/1902000] [318.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 07:34:51,518 - Train: 31.84% [605600/1902000] [318.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 07:35:52,083 - Train: 31.85% [605700/1902000] [318.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 07:36:52,627 - Train: 31.85% [605800/1902000] [318.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:37:53,279 - Train: 31.86% [605900/1902000] [318.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 07:38:53,829 - Train: 31.86% [606000/1902000] [318.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 07:39:54,389 - Train: 31.87% [606100/1902000] [318.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:40:54,945 - Train: 31.87% [606200/1902000] [318.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:41:55,507 - Train: 31.88% [606300/1902000] [318.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 07:42:56,068 - Train: 31.88% [606400/1902000] [318.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 07:43:56,620 - Train: 31.89% [606500/1902000] [318.9/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 07:44:57,215 - Train: 31.89% [606600/1902000] [318.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 07:45:57,817 - Train: 31.90% [606700/1902000] [319.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:46:20,836 - ==> Total time: 5 days, 6:46:00 Eta: 11 days, 6:37:16 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 07:47:00,847 - Train: 31.90% [606800/1902000] [319.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:48:01,431 - Train: 31.91% [606900/1902000] [319.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:49:02,010 - Train: 31.91% [607000/1902000] [319.1/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 07:50:02,580 - Train: 31.92% [607100/1902000] [319.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 07:51:03,359 - Train: 31.92% [607200/1902000] [319.2/1000.0] [batch_t 0.611 (0.608)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-07 07:52:03,884 - Train: 31.93% [607300/1902000] [319.3/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 07:53:04,422 - Train: 31.93% [607400/1902000] [319.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 07:54:04,986 - Train: 31.94% [607500/1902000] [319.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:55:05,529 - Train: 31.95% [607600/1902000] [319.5/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:56:06,077 - Train: 31.95% [607700/1902000] [319.5/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 07:57:06,658 - Train: 31.96% [607800/1902000] [319.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 07:58:07,252 - Train: 31.96% [607900/1902000] [319.6/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 07:59:07,918 - Train: 31.97% [608000/1902000] [319.7/1000.0] [batch_t 0.610 (0.607)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-07 08:00:08,456 - Train: 31.97% [608100/1902000] [319.7/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 08:01:09,041 - Train: 31.98% [608200/1902000] [319.8/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 08:02:09,562 - Train: 31.98% [608300/1902000] [319.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 08:03:10,135 - Train: 31.99% [608400/1902000] [319.9/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 08:04:10,793 - Train: 31.99% [608500/1902000] [319.9/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 08:05:11,378 - Train: 32.00% [608600/1902000] [320.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 08:05:35,596 - ==> Total time: 5 days, 7:05:15 Eta: 11 days, 6:03:39 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 08:06:14,152 - Train: 32.00% [608700/1902000] [320.0/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 08:07:14,721 - Train: 32.01% [608800/1902000] [320.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 08:08:15,264 - Train: 32.01% [608900/1902000] [320.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 08:09:15,964 - Train: 32.02% [609000/1902000] [320.2/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 08:10:16,517 - Train: 32.02% [609100/1902000] [320.2/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 08:11:17,089 - Train: 32.03% [609200/1902000] [320.3/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-07 08:12:17,680 - Train: 32.03% [609300/1902000] [320.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 08:13:25,451 - Train: 32.04% [609400/1902000] [320.4/1000.0] [batch_t 0.605 (0.678)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 08:14:26,017 - Train: 32.05% [609500/1902000] [320.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 08:15:26,577 - Train: 32.05% [609600/1902000] [320.5/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 08:16:27,219 - Train: 32.06% [609700/1902000] [320.6/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 08:17:27,776 - Train: 32.06% [609800/1902000] [320.6/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 08:18:28,327 - Train: 32.07% [609900/1902000] [320.7/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 08:19:28,854 - Train: 32.07% [610000/1902000] [320.7/1000.0] [batch_t 0.603 (0.605)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 08:20:29,405 - Train: 32.08% [610100/1902000] [320.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 08:21:29,944 - Train: 32.08% [610200/1902000] [320.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 08:22:30,519 - Train: 32.09% [610300/1902000] [320.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 08:23:31,152 - Train: 32.09% [610400/1902000] [320.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 08:24:31,717 - Train: 32.10% [610500/1902000] [321.0/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 08:24:57,144 - ==> Total time: 5 days, 7:24:36 Eta: 11 days, 5:30:22 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 08:25:34,485 - Train: 32.10% [610600/1902000] [321.0/1000.0] [batch_t 0.602 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 08:26:35,018 - Train: 32.11% [610700/1902000] [321.1/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 08:27:35,620 - Train: 32.11% [610800/1902000] [321.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 08:28:36,355 - Train: 32.12% [610900/1902000] [321.2/1000.0] [batch_t 0.609 (0.607)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 08:29:36,934 - Train: 32.12% [611000/1902000] [321.2/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 08:30:37,528 - Train: 32.13% [611100/1902000] [321.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 08:31:38,105 - Train: 32.13% [611200/1902000] [321.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 08:32:38,663 - Train: 32.14% [611300/1902000] [321.4/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 08:33:39,234 - Train: 32.15% [611400/1902000] [321.5/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 08:34:39,862 - Train: 32.15% [611500/1902000] [321.5/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 08:35:40,473 - Train: 32.16% [611600/1902000] [321.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 08:36:41,136 - Train: 32.16% [611700/1902000] [321.6/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 08:37:41,693 - Train: 32.17% [611800/1902000] [321.7/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 08:38:42,257 - Train: 32.17% [611900/1902000] [321.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 08:39:42,800 - Train: 32.18% [612000/1902000] [321.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 08:40:43,356 - Train: 32.18% [612100/1902000] [321.8/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 08:41:43,997 - Train: 32.19% [612200/1902000] [321.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 08:42:44,558 - Train: 32.19% [612300/1902000] [321.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 08:43:45,126 - Train: 32.20% [612400/1902000] [322.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 08:44:11,767 - ==> Total time: 5 days, 7:43:51 Eta: 11 days, 4:56:56 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 08:44:47,907 - Train: 32.20% [612500/1902000] [322.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 08:45:48,477 - Train: 32.21% [612600/1902000] [322.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 08:46:48,993 - Train: 32.21% [612700/1902000] [322.1/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 08:47:49,578 - Train: 32.22% [612800/1902000] [322.2/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 08:48:50,393 - Train: 32.22% [612900/1902000] [322.2/1000.0] [batch_t 0.605 (0.608)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-07 08:49:51,082 - Train: 32.23% [613000/1902000] [322.3/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 08:50:51,801 - Train: 32.23% [613100/1902000] [322.3/1000.0] [batch_t 0.611 (0.607)] [data_t 0.003] [optim_t 0.608] [lr 0.005000] +2024-04-07 08:51:52,467 - Train: 32.24% [613200/1902000] [322.4/1000.0] [batch_t 0.607 (0.607)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-07 08:52:53,128 - Train: 32.25% [613300/1902000] [322.5/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 08:53:53,823 - Train: 32.25% [613400/1902000] [322.5/1000.0] [batch_t 0.608 (0.607)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-07 08:54:54,612 - Train: 32.26% [613500/1902000] [322.6/1000.0] [batch_t 0.604 (0.608)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 08:55:55,340 - Train: 32.26% [613600/1902000] [322.6/1000.0] [batch_t 0.604 (0.607)] [data_t 0.003] [optim_t 0.601] [lr 0.005000] +2024-04-07 08:56:56,055 - Train: 32.27% [613700/1902000] [322.7/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 08:57:56,761 - Train: 32.27% [613800/1902000] [322.7/1000.0] [batch_t 0.608 (0.607)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-07 08:58:57,449 - Train: 32.28% [613900/1902000] [322.8/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 08:59:58,160 - Train: 32.28% [614000/1902000] [322.8/1000.0] [batch_t 0.608 (0.607)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-07 09:00:58,851 - Train: 32.29% [614100/1902000] [322.9/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 09:01:59,647 - Train: 32.29% [614200/1902000] [322.9/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 09:03:00,363 - Train: 32.30% [614300/1902000] [323.0/1000.0] [batch_t 0.608 (0.607)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-07 09:03:28,273 - ==> Total time: 5 days, 8:03:07 Eta: 11 days, 4:23:38 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 09:04:03,334 - Train: 32.30% [614400/1902000] [323.0/1000.0] [batch_t 0.608 (0.607)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-07 09:05:04,030 - Train: 32.31% [614500/1902000] [323.1/1000.0] [batch_t 0.601 (0.607)] [data_t 0.002] [optim_t 0.599] [lr 0.005000] +2024-04-07 09:06:04,740 - Train: 32.31% [614600/1902000] [323.1/1000.0] [batch_t 0.609 (0.607)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-07 09:07:05,458 - Train: 32.32% [614700/1902000] [323.2/1000.0] [batch_t 0.605 (0.607)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-07 09:08:06,272 - Train: 32.32% [614800/1902000] [323.2/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 09:09:06,967 - Train: 32.33% [614900/1902000] [323.3/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 09:10:07,660 - Train: 32.33% [615000/1902000] [323.3/1000.0] [batch_t 0.608 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 09:11:08,366 - Train: 32.34% [615100/1902000] [323.4/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 09:12:09,081 - Train: 32.34% [615200/1902000] [323.4/1000.0] [batch_t 0.608 (0.607)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-07 09:13:09,777 - Train: 32.35% [615300/1902000] [323.5/1000.0] [batch_t 0.605 (0.607)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-07 09:14:10,560 - Train: 32.36% [615400/1902000] [323.6/1000.0] [batch_t 0.605 (0.608)] [data_t 0.003] [optim_t 0.602] [lr 0.005000] +2024-04-07 09:15:11,255 - Train: 32.36% [615500/1902000] [323.6/1000.0] [batch_t 0.608 (0.607)] [data_t 0.003] [optim_t 0.605] [lr 0.005000] +2024-04-07 09:16:12,002 - Train: 32.37% [615600/1902000] [323.7/1000.0] [batch_t 0.606 (0.607)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:17:12,714 - Train: 32.37% [615700/1902000] [323.7/1000.0] [batch_t 0.607 (0.607)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:18:13,419 - Train: 32.38% [615800/1902000] [323.8/1000.0] [batch_t 0.609 (0.607)] [data_t 0.003] [optim_t 0.606] [lr 0.005000] +2024-04-07 09:19:14,118 - Train: 32.38% [615900/1902000] [323.8/1000.0] [batch_t 0.606 (0.607)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:20:14,777 - Train: 32.39% [616000/1902000] [323.9/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:21:15,485 - Train: 32.39% [616100/1902000] [323.9/1000.0] [batch_t 0.609 (0.607)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 09:22:16,105 - Train: 32.40% [616200/1902000] [324.0/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.608] [lr 0.005000] +2024-04-07 09:22:45,194 - ==> Total time: 5 days, 8:22:24 Eta: 11 days, 3:50:27 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 09:23:18,915 - Train: 32.40% [616300/1902000] [324.0/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 09:24:19,523 - Train: 32.41% [616400/1902000] [324.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 09:25:20,144 - Train: 32.41% [616500/1902000] [324.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:26:20,791 - Train: 32.42% [616600/1902000] [324.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 09:27:21,556 - Train: 32.42% [616700/1902000] [324.2/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 09:28:22,204 - Train: 32.43% [616800/1902000] [324.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 09:29:22,858 - Train: 32.43% [616900/1902000] [324.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:30:23,494 - Train: 32.44% [617000/1902000] [324.4/1000.0] [batch_t 0.608 (0.606)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 09:31:24,145 - Train: 32.44% [617100/1902000] [324.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:32:24,816 - Train: 32.45% [617200/1902000] [324.5/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 09:33:25,506 - Train: 32.46% [617300/1902000] [324.6/1000.0] [batch_t 0.607 (0.607)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 09:34:26,107 - Train: 32.46% [617400/1902000] [324.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:35:26,753 - Train: 32.47% [617500/1902000] [324.7/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 09:36:27,395 - Train: 32.47% [617600/1902000] [324.7/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 09:37:28,029 - Train: 32.48% [617700/1902000] [324.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 09:38:28,632 - Train: 32.48% [617800/1902000] [324.8/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 09:39:29,237 - Train: 32.49% [617900/1902000] [324.9/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 09:40:29,945 - Train: 32.49% [618000/1902000] [324.9/1000.0] [batch_t 0.604 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 09:41:30,581 - Train: 32.50% [618100/1902000] [325.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 09:42:00,879 - ==> Total time: 5 days, 8:41:40 Eta: 11 days, 3:17:19 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 09:42:34,123 - Train: 32.50% [618200/1902000] [325.0/1000.0] [batch_t 0.606 (0.623)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 09:43:34,714 - Train: 32.51% [618300/1902000] [325.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:44:35,302 - Train: 32.51% [618400/1902000] [325.1/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 09:45:36,068 - Train: 32.52% [618500/1902000] [325.2/1000.0] [batch_t 0.607 (0.608)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 09:46:36,626 - Train: 32.52% [618600/1902000] [325.2/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:47:37,212 - Train: 32.53% [618700/1902000] [325.3/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 09:48:37,820 - Train: 32.53% [618800/1902000] [325.3/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:49:38,393 - Train: 32.54% [618900/1902000] [325.4/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:50:39,026 - Train: 32.54% [619000/1902000] [325.4/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 09:51:39,652 - Train: 32.55% [619100/1902000] [325.5/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 09:52:40,240 - Train: 32.56% [619200/1902000] [325.6/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 09:53:40,905 - Train: 32.56% [619300/1902000] [325.6/1000.0] [batch_t 0.605 (0.607)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 09:54:41,507 - Train: 32.57% [619400/1902000] [325.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 09:55:42,088 - Train: 32.57% [619500/1902000] [325.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 09:56:42,707 - Train: 32.58% [619600/1902000] [325.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 09:57:43,273 - Train: 32.58% [619700/1902000] [325.8/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 09:58:43,923 - Train: 32.59% [619800/1902000] [325.9/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 09:59:44,518 - Train: 32.59% [619900/1902000] [325.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 10:00:45,106 - Train: 32.60% [620000/1902000] [326.0/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 10:01:16,606 - ==> Total time: 5 days, 9:00:56 Eta: 11 days, 2:44:15 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 10:01:47,777 - Train: 32.60% [620100/1902000] [326.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 10:02:48,391 - Train: 32.61% [620200/1902000] [326.1/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 10:03:48,985 - Train: 32.61% [620300/1902000] [326.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 10:04:49,581 - Train: 32.62% [620400/1902000] [326.2/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 10:05:50,314 - Train: 32.62% [620500/1902000] [326.2/1000.0] [batch_t 0.607 (0.607)] [data_t 0.003] [optim_t 0.604] [lr 0.005000] +2024-04-07 10:06:50,892 - Train: 32.63% [620600/1902000] [326.3/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 10:07:51,514 - Train: 32.63% [620700/1902000] [326.3/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 10:08:52,111 - Train: 32.64% [620800/1902000] [326.4/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 10:09:52,739 - Train: 32.64% [620900/1902000] [326.4/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 10:10:53,513 - Train: 32.65% [621000/1902000] [326.5/1000.0] [batch_t 0.606 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 10:11:54,108 - Train: 32.66% [621100/1902000] [326.6/1000.0] [batch_t 0.609 (0.606)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 10:12:54,705 - Train: 32.66% [621200/1902000] [326.6/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 10:13:55,330 - Train: 32.67% [621300/1902000] [326.7/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 10:14:55,914 - Train: 32.67% [621400/1902000] [326.7/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 10:15:56,571 - Train: 32.68% [621500/1902000] [326.8/1000.0] [batch_t 0.610 (0.606)] [data_t 0.002] [optim_t 0.609] [lr 0.005000] +2024-04-07 10:16:57,198 - Train: 32.68% [621600/1902000] [326.8/1000.0] [batch_t 0.604 (0.606)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 10:17:57,824 - Train: 32.69% [621700/1902000] [326.9/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 10:18:58,560 - Train: 32.69% [621800/1902000] [326.9/1000.0] [batch_t 0.609 (0.607)] [data_t 0.002] [optim_t 0.607] [lr 0.005000] +2024-04-07 10:19:59,141 - Train: 32.70% [621900/1902000] [327.0/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 10:20:31,898 - ==> Total time: 5 days, 9:20:11 Eta: 11 days, 2:11:16 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 10:21:01,926 - Train: 32.70% [622000/1902000] [327.0/1000.0] [batch_t 0.605 (0.608)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 10:22:02,493 - Train: 32.71% [622100/1902000] [327.1/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 10:23:03,066 - Train: 32.71% [622200/1902000] [327.1/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 10:24:35,921 - Train: 32.72% [622300/1902000] [327.2/1000.0] [batch_t 0.613 (0.928)] [data_t 0.012] [optim_t 0.602] [lr 0.005000] +2024-04-07 10:40:12,700 - Train: 32.72% [622400/1902000] [327.2/1000.0] [batch_t 4.029 (9.368)] [data_t 2.587] [optim_t 1.442] [lr 0.005000] +2024-04-07 10:54:32,282 - Train: 32.73% [622500/1902000] [327.3/1000.0] [batch_t 1.431 (8.596)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-07 10:56:55,484 - Train: 32.73% [622600/1902000] [327.3/1000.0] [batch_t 1.441 (1.432)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-07 10:59:18,599 - Train: 32.74% [622700/1902000] [327.4/1000.0] [batch_t 1.431 (1.431)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-07 11:01:41,822 - Train: 32.74% [622800/1902000] [327.4/1000.0] [batch_t 1.437 (1.432)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-07 11:04:04,983 - Train: 32.75% [622900/1902000] [327.5/1000.0] [batch_t 1.440 (1.432)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-07 11:06:27,964 - Train: 32.75% [623000/1902000] [327.5/1000.0] [batch_t 1.427 (1.430)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-07 11:08:51,131 - Train: 32.76% [623100/1902000] [327.6/1000.0] [batch_t 1.419 (1.432)] [data_t 0.002] [optim_t 1.416] [lr 0.005000] +2024-04-07 11:11:14,306 - Train: 32.77% [623200/1902000] [327.7/1000.0] [batch_t 1.426 (1.432)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-07 11:12:53,116 - Train: 32.77% [623300/1902000] [327.7/1000.0] [batch_t 0.608 (0.988)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 11:13:55,809 - Train: 32.78% [623400/1902000] [327.8/1000.0] [batch_t 0.605 (0.627)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 11:14:56,385 - Train: 32.78% [623500/1902000] [327.8/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 11:16:00,626 - Train: 32.79% [623600/1902000] [327.9/1000.0] [batch_t 0.606 (0.642)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 11:17:02,887 - Train: 32.79% [623700/1902000] [327.9/1000.0] [batch_t 0.605 (0.623)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 11:18:09,315 - Train: 32.80% [623800/1902000] [328.0/1000.0] [batch_t 0.606 (0.664)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 11:18:44,455 - ==> Total time: 5 days, 10:18:23 Eta: 11 days, 2:58:11 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 11:19:22,318 - Train: 32.80% [623900/1902000] [328.0/1000.0] [batch_t 0.606 (0.776)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 11:20:22,822 - Train: 32.81% [624000/1902000] [328.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 11:21:23,316 - Train: 32.81% [624100/1902000] [328.1/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 11:22:23,836 - Train: 32.82% [624200/1902000] [328.2/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 11:23:24,364 - Train: 32.82% [624300/1902000] [328.2/1000.0] [batch_t 0.604 (0.605)] [data_t 0.002] [optim_t 0.602] [lr 0.005000] +2024-04-07 11:24:25,041 - Train: 32.83% [624400/1902000] [328.3/1000.0] [batch_t 0.602 (0.607)] [data_t 0.002] [optim_t 0.600] [lr 0.005000] +2024-04-07 11:25:25,568 - Train: 32.83% [624500/1902000] [328.3/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 11:26:26,127 - Train: 32.84% [624600/1902000] [328.4/1000.0] [batch_t 0.608 (0.605)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 11:27:26,672 - Train: 32.84% [624700/1902000] [328.4/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 11:28:27,224 - Train: 32.85% [624800/1902000] [328.5/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 11:29:27,796 - Train: 32.85% [624900/1902000] [328.5/1000.0] [batch_t 0.605 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 11:30:28,344 - Train: 32.86% [625000/1902000] [328.6/1000.0] [batch_t 0.605 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 11:31:28,871 - Train: 32.87% [625100/1902000] [328.7/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 11:32:29,460 - Train: 32.87% [625200/1902000] [328.7/1000.0] [batch_t 0.603 (0.606)] [data_t 0.002] [optim_t 0.601] [lr 0.005000] +2024-04-07 11:33:30,011 - Train: 32.88% [625300/1902000] [328.8/1000.0] [batch_t 0.606 (0.605)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 11:34:30,544 - Train: 32.88% [625400/1902000] [328.8/1000.0] [batch_t 0.607 (0.605)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 11:35:31,792 - Train: 32.89% [625500/1902000] [328.9/1000.0] [batch_t 0.608 (0.612)] [data_t 0.002] [optim_t 0.606] [lr 0.005000] +2024-04-07 11:36:33,161 - Train: 32.89% [625600/1902000] [328.9/1000.0] [batch_t 0.605 (0.614)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 11:37:33,801 - Train: 32.90% [625700/1902000] [329.0/1000.0] [batch_t 0.606 (0.606)] [data_t 0.002] [optim_t 0.604] [lr 0.005000] +2024-04-07 11:38:12,494 - ==> Total time: 5 days, 10:37:52 Eta: 11 days, 2:25:26 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 11:38:43,644 - Train: 32.90% [625800/1902000] [329.0/1000.0] [batch_t 0.605 (0.666)] [data_t 0.002] [optim_t 0.603] [lr 0.005000] +2024-04-07 11:39:44,209 - Train: 32.91% [625900/1902000] [329.1/1000.0] [batch_t 0.607 (0.606)] [data_t 0.002] [optim_t 0.605] [lr 0.005000] +2024-04-07 11:41:29,527 - Train: 32.91% [626000/1902000] [329.1/1000.0] [batch_t 7.430 (1.053)] [data_t 6.825] [optim_t 0.605] [lr 0.005000] +2024-04-07 11:51:44,972 - Train: 32.92% [626100/1902000] [329.2/1000.0] [batch_t 2.481 (6.154)] [data_t 1.066] [optim_t 1.415] [lr 0.005000] +2024-04-07 12:01:47,796 - Train: 32.92% [626200/1902000] [329.2/1000.0] [batch_t 15.830 (6.028)] [data_t 14.429] [optim_t 1.401] [lr 0.005000] +2024-04-07 12:15:43,639 - Train: 32.93% [626300/1902000] [329.3/1000.0] [batch_t 14.253 (8.358)] [data_t 12.809] [optim_t 1.444] [lr 0.005000] +2024-04-07 12:20:15,068 - Train: 32.93% [626400/1902000] [329.3/1000.0] [batch_t 1.426 (2.714)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-07 12:22:38,330 - Train: 32.94% [626500/1902000] [329.4/1000.0] [batch_t 1.436 (1.433)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-07 12:25:01,505 - Train: 32.94% [626600/1902000] [329.4/1000.0] [batch_t 1.441 (1.432)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-07 12:27:24,870 - Train: 32.95% [626700/1902000] [329.5/1000.0] [batch_t 1.426 (1.434)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-07 12:29:48,185 - Train: 32.95% [626800/1902000] [329.5/1000.0] [batch_t 1.433 (1.433)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-07 12:32:11,418 - Train: 32.96% [626900/1902000] [329.6/1000.0] [batch_t 1.421 (1.432)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-07 12:34:34,675 - Train: 32.97% [627000/1902000] [329.7/1000.0] [batch_t 1.422 (1.432)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-07 12:36:57,770 - Train: 32.97% [627100/1902000] [329.7/1000.0] [batch_t 1.437 (1.431)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-07 12:39:21,018 - Train: 32.98% [627200/1902000] [329.8/1000.0] [batch_t 1.436 (1.432)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-07 12:41:44,237 - Train: 32.98% [627300/1902000] [329.8/1000.0] [batch_t 1.433 (1.432)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-07 12:44:07,379 - Train: 32.99% [627400/1902000] [329.9/1000.0] [batch_t 1.431 (1.431)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-07 12:46:30,688 - Train: 32.99% [627500/1902000] [329.9/1000.0] [batch_t 1.440 (1.433)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-07 12:48:53,880 - Train: 33.00% [627600/1902000] [330.0/1000.0] [batch_t 1.431 (1.432)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-07 12:50:19,820 - ==> Total time: 5 days, 11:49:59 Eta: 11 days, 3:39:40 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 12:51:20,819 - Train: 33.00% [627700/1902000] [330.0/1000.0] [batch_t 1.441 (1.449)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-07 12:53:44,067 - Train: 33.01% [627800/1902000] [330.1/1000.0] [batch_t 1.441 (1.432)] [data_t 0.003] [optim_t 1.439] [lr 0.005000] +2024-04-07 12:56:07,259 - Train: 33.01% [627900/1902000] [330.1/1000.0] [batch_t 1.452 (1.432)] [data_t 0.003] [optim_t 1.449] [lr 0.005000] +2024-04-07 12:58:30,419 - Train: 33.02% [628000/1902000] [330.2/1000.0] [batch_t 1.425 (1.432)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-07 13:00:53,783 - Train: 33.02% [628100/1902000] [330.2/1000.0] [batch_t 1.436 (1.434)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-07 13:03:17,023 - Train: 33.03% [628200/1902000] [330.3/1000.0] [batch_t 1.425 (1.432)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-07 13:05:40,314 - Train: 33.03% [628300/1902000] [330.3/1000.0] [batch_t 1.430 (1.433)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-07 13:08:03,558 - Train: 33.04% [628400/1902000] [330.4/1000.0] [batch_t 1.440 (1.432)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-07 13:10:26,871 - Train: 33.04% [628500/1902000] [330.4/1000.0] [batch_t 1.424 (1.433)] [data_t 0.002] [optim_t 1.421] [lr 0.005000] +2024-04-07 13:12:50,195 - Train: 33.05% [628600/1902000] [330.5/1000.0] [batch_t 1.428 (1.433)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-07 13:15:13,597 - Train: 33.05% [628700/1902000] [330.5/1000.0] [batch_t 1.436 (1.434)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-07 13:17:36,774 - Train: 33.06% [628800/1902000] [330.6/1000.0] [batch_t 1.431 (1.432)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-07 13:20:00,004 - Train: 33.07% [628900/1902000] [330.7/1000.0] [batch_t 1.428 (1.432)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-07 13:22:23,288 - Train: 33.07% [629000/1902000] [330.7/1000.0] [batch_t 1.432 (1.433)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-07 13:24:46,570 - Train: 33.08% [629100/1902000] [330.8/1000.0] [batch_t 1.416 (1.433)] [data_t 0.002] [optim_t 1.414] [lr 0.005000] +2024-04-07 13:27:09,852 - Train: 33.08% [629200/1902000] [330.8/1000.0] [batch_t 1.431 (1.433)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-07 13:29:33,139 - Train: 33.09% [629300/1902000] [330.9/1000.0] [batch_t 1.426 (1.433)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-07 13:31:56,612 - Train: 33.09% [629400/1902000] [330.9/1000.0] [batch_t 1.426 (1.435)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-07 13:34:19,924 - Train: 33.10% [629500/1902000] [331.0/1000.0] [batch_t 1.436 (1.433)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-07 13:35:48,763 - ==> Total time: 5 days, 12:35:28 Eta: 11 days, 3:59:11 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 13:36:45,147 - Train: 33.10% [629600/1902000] [331.0/1000.0] [batch_t 1.427 (1.433)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-07 13:39:08,461 - Train: 33.11% [629700/1902000] [331.1/1000.0] [batch_t 1.435 (1.433)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-07 13:41:31,622 - Train: 33.11% [629800/1902000] [331.1/1000.0] [batch_t 1.422 (1.431)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-07 13:43:54,875 - Train: 33.12% [629900/1902000] [331.2/1000.0] [batch_t 1.426 (1.432)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-07 13:46:18,092 - Train: 33.12% [630000/1902000] [331.2/1000.0] [batch_t 1.436 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-07 13:48:41,296 - Train: 33.13% [630100/1902000] [331.3/1000.0] [batch_t 1.418 (1.432)] [data_t 0.003] [optim_t 1.416] [lr 0.005000] +2024-04-07 13:51:04,509 - Train: 33.13% [630200/1902000] [331.3/1000.0] [batch_t 1.443 (1.432)] [data_t 0.002] [optim_t 1.441] [lr 0.005000] +2024-04-07 13:53:27,747 - Train: 33.14% [630300/1902000] [331.4/1000.0] [batch_t 1.450 (1.432)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-07 13:55:51,010 - Train: 33.14% [630400/1902000] [331.4/1000.0] [batch_t 1.430 (1.433)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-07 13:58:14,177 - Train: 33.15% [630500/1902000] [331.5/1000.0] [batch_t 1.444 (1.432)] [data_t 0.003] [optim_t 1.441] [lr 0.005000] +2024-04-07 14:00:37,308 - Train: 33.15% [630600/1902000] [331.5/1000.0] [batch_t 1.422 (1.431)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-07 14:03:00,573 - Train: 33.16% [630700/1902000] [331.6/1000.0] [batch_t 1.433 (1.433)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-07 14:05:23,833 - Train: 33.17% [630800/1902000] [331.7/1000.0] [batch_t 1.431 (1.433)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-07 14:07:46,993 - Train: 33.17% [630900/1902000] [331.7/1000.0] [batch_t 1.438 (1.431)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-07 14:10:10,134 - Train: 33.18% [631000/1902000] [331.8/1000.0] [batch_t 1.440 (1.431)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-07 14:13:47,137 - Train: 33.18% [631100/1902000] [331.8/1000.0] [batch_t 4.703 (2.170)] [data_t 3.262] [optim_t 1.441] [lr 0.005000] +2024-04-07 14:27:17,229 - Train: 33.19% [631200/1902000] [331.9/1000.0] [batch_t 1.436 (8.101)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-07 14:29:40,318 - Train: 33.19% [631300/1902000] [331.9/1000.0] [batch_t 1.424 (1.431)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-07 14:32:03,450 - Train: 33.20% [631400/1902000] [332.0/1000.0] [batch_t 1.427 (1.431)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-07 14:33:35,199 - ==> Total time: 5 days, 13:33:14 Eta: 11 days, 4:43:02 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 14:34:29,675 - Train: 33.20% [631500/1902000] [332.0/1000.0] [batch_t 1.423 (1.439)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-07 14:36:52,904 - Train: 33.21% [631600/1902000] [332.1/1000.0] [batch_t 1.436 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-07 14:39:16,224 - Train: 33.21% [631700/1902000] [332.1/1000.0] [batch_t 1.435 (1.433)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-07 14:41:39,358 - Train: 33.22% [631800/1902000] [332.2/1000.0] [batch_t 1.429 (1.431)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-07 14:44:02,500 - Train: 33.22% [631900/1902000] [332.2/1000.0] [batch_t 1.437 (1.431)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-07 14:46:25,809 - Train: 33.23% [632000/1902000] [332.3/1000.0] [batch_t 1.434 (1.433)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-07 14:48:48,925 - Train: 33.23% [632100/1902000] [332.3/1000.0] [batch_t 1.426 (1.431)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-07 14:51:12,170 - Train: 33.24% [632200/1902000] [332.4/1000.0] [batch_t 1.416 (1.432)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-07 14:53:35,329 - Train: 33.24% [632300/1902000] [332.4/1000.0] [batch_t 1.432 (1.431)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-07 14:55:58,467 - Train: 33.25% [632400/1902000] [332.5/1000.0] [batch_t 1.435 (1.431)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-07 14:58:21,705 - Train: 33.25% [632500/1902000] [332.5/1000.0] [batch_t 1.420 (1.432)] [data_t 0.003] [optim_t 1.417] [lr 0.005000] +2024-04-07 15:00:45,100 - Train: 33.26% [632600/1902000] [332.6/1000.0] [batch_t 1.430 (1.434)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-07 15:03:08,250 - Train: 33.26% [632700/1902000] [332.6/1000.0] [batch_t 1.422 (1.431)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-07 15:05:31,602 - Train: 33.27% [632800/1902000] [332.7/1000.0] [batch_t 1.431 (1.433)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-07 15:07:54,899 - Train: 33.28% [632900/1902000] [332.8/1000.0] [batch_t 1.440 (1.433)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-07 15:10:17,991 - Train: 33.28% [633000/1902000] [332.8/1000.0] [batch_t 1.432 (1.431)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-07 15:12:41,003 - Train: 33.29% [633100/1902000] [332.9/1000.0] [batch_t 1.444 (1.430)] [data_t 0.002] [optim_t 1.442] [lr 0.005000] +2024-04-07 15:15:04,188 - Train: 33.29% [633200/1902000] [332.9/1000.0] [batch_t 1.427 (1.432)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-07 15:17:27,256 - Train: 33.30% [633300/1902000] [333.0/1000.0] [batch_t 1.439 (1.431)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-07 15:19:01,676 - ==> Total time: 5 days, 14:18:41 Eta: 11 days, 5:01:34 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 15:19:54,077 - Train: 33.30% [633400/1902000] [333.0/1000.0] [batch_t 1.431 (1.434)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-07 15:22:17,075 - Train: 33.31% [633500/1902000] [333.1/1000.0] [batch_t 1.427 (1.430)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-07 15:24:40,113 - Train: 33.31% [633600/1902000] [333.1/1000.0] [batch_t 1.430 (1.430)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-07 15:27:03,450 - Train: 33.32% [633700/1902000] [333.2/1000.0] [batch_t 1.436 (1.433)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-07 15:29:26,815 - Train: 33.32% [633800/1902000] [333.2/1000.0] [batch_t 1.437 (1.434)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-07 15:31:50,009 - Train: 33.33% [633900/1902000] [333.3/1000.0] [batch_t 1.436 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-07 15:34:13,272 - Train: 33.33% [634000/1902000] [333.3/1000.0] [batch_t 1.427 (1.433)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-07 15:36:36,499 - Train: 33.34% [634100/1902000] [333.4/1000.0] [batch_t 1.436 (1.432)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-07 15:38:59,809 - Train: 33.34% [634200/1902000] [333.4/1000.0] [batch_t 1.427 (1.433)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-07 15:41:40,090 - Train: 33.35% [634300/1902000] [333.5/1000.0] [batch_t 1.754 (1.603)] [data_t 0.320] [optim_t 1.434] [lr 0.005000] +2024-04-07 15:44:27,698 - Train: 33.35% [634400/1902000] [333.5/1000.0] [batch_t 1.430 (1.676)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-07 15:47:22,071 - Train: 33.36% [634500/1902000] [333.6/1000.0] [batch_t 1.430 (1.744)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-07 15:49:49,164 - Train: 33.36% [634600/1902000] [333.6/1000.0] [batch_t 2.041 (1.471)] [data_t 0.622] [optim_t 1.419] [lr 0.005000] +2024-04-07 15:52:33,513 - Train: 33.37% [634700/1902000] [333.7/1000.0] [batch_t 1.439 (1.643)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-07 15:54:56,697 - Train: 33.38% [634800/1902000] [333.8/1000.0] [batch_t 1.436 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-07 15:57:19,978 - Train: 33.38% [634900/1902000] [333.8/1000.0] [batch_t 1.441 (1.433)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-07 15:59:43,295 - Train: 33.39% [635000/1902000] [333.9/1000.0] [batch_t 1.431 (1.433)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-07 16:02:06,600 - Train: 33.39% [635100/1902000] [333.9/1000.0] [batch_t 1.430 (1.433)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-07 16:04:29,836 - Train: 33.40% [635200/1902000] [334.0/1000.0] [batch_t 1.412 (1.432)] [data_t 0.003] [optim_t 1.410] [lr 0.005000] +2024-04-07 16:06:07,345 - ==> Total time: 5 days, 15:05:46 Eta: 11 days, 5:23:01 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 16:06:56,490 - Train: 33.40% [635300/1902000] [334.0/1000.0] [batch_t 1.435 (1.436)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-07 16:09:19,892 - Train: 33.41% [635400/1902000] [334.1/1000.0] [batch_t 1.439 (1.434)] [data_t 0.003] [optim_t 1.436] [lr 0.005000] +2024-04-07 16:11:43,288 - Train: 33.41% [635500/1902000] [334.1/1000.0] [batch_t 1.442 (1.434)] [data_t 0.002] [optim_t 1.440] [lr 0.005000] +2024-04-07 16:14:06,719 - Train: 33.42% [635600/1902000] [334.2/1000.0] [batch_t 1.440 (1.434)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-07 16:16:30,106 - Train: 33.42% [635700/1902000] [334.2/1000.0] [batch_t 1.438 (1.434)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-07 16:18:53,504 - Train: 33.43% [635800/1902000] [334.3/1000.0] [batch_t 1.434 (1.434)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-07 16:21:16,825 - Train: 33.43% [635900/1902000] [334.3/1000.0] [batch_t 1.438 (1.433)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-07 16:23:40,184 - Train: 33.44% [636000/1902000] [334.4/1000.0] [batch_t 1.429 (1.433)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-07 16:26:03,398 - Train: 33.44% [636100/1902000] [334.4/1000.0] [batch_t 1.428 (1.432)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-07 16:28:26,832 - Train: 33.45% [636200/1902000] [334.5/1000.0] [batch_t 1.436 (1.434)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-07 16:30:50,124 - Train: 33.45% [636300/1902000] [334.5/1000.0] [batch_t 1.423 (1.433)] [data_t 0.003] [optim_t 1.421] [lr 0.005000] +2024-04-07 16:33:13,480 - Train: 33.46% [636400/1902000] [334.6/1000.0] [batch_t 1.428 (1.433)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-07 16:35:36,793 - Train: 33.46% [636500/1902000] [334.6/1000.0] [batch_t 1.438 (1.433)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-07 16:38:00,270 - Train: 33.47% [636600/1902000] [334.7/1000.0] [batch_t 1.422 (1.435)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-07 16:40:23,614 - Train: 33.48% [636700/1902000] [334.8/1000.0] [batch_t 1.426 (1.433)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-07 16:42:46,925 - Train: 33.48% [636800/1902000] [334.8/1000.0] [batch_t 1.452 (1.433)] [data_t 0.003] [optim_t 1.449] [lr 0.005000] +2024-04-07 16:45:10,232 - Train: 33.49% [636900/1902000] [334.9/1000.0] [batch_t 1.436 (1.433)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-07 16:47:33,525 - Train: 33.49% [637000/1902000] [334.9/1000.0] [batch_t 1.419 (1.433)] [data_t 0.003] [optim_t 1.416] [lr 0.005000] +2024-04-07 16:49:56,806 - Train: 33.50% [637100/1902000] [335.0/1000.0] [batch_t 1.433 (1.433)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-07 16:51:37,176 - ==> Total time: 5 days, 15:51:16 Eta: 11 days, 5:40:53 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 16:52:22,531 - Train: 33.50% [637200/1902000] [335.0/1000.0] [batch_t 1.428 (1.434)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-07 16:54:45,780 - Train: 33.51% [637300/1902000] [335.1/1000.0] [batch_t 1.445 (1.432)] [data_t 0.003] [optim_t 1.442] [lr 0.005000] +2024-04-07 16:57:08,821 - Train: 33.51% [637400/1902000] [335.1/1000.0] [batch_t 1.447 (1.430)] [data_t 0.002] [optim_t 1.445] [lr 0.005000] +2024-04-07 16:59:31,818 - Train: 33.52% [637500/1902000] [335.2/1000.0] [batch_t 1.422 (1.430)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-07 17:01:55,093 - Train: 33.52% [637600/1902000] [335.2/1000.0] [batch_t 1.431 (1.433)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-07 17:04:18,296 - Train: 33.53% [637700/1902000] [335.3/1000.0] [batch_t 1.428 (1.432)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-07 17:06:41,566 - Train: 33.53% [637800/1902000] [335.3/1000.0] [batch_t 1.427 (1.433)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-07 17:09:04,689 - Train: 33.54% [637900/1902000] [335.4/1000.0] [batch_t 1.437 (1.431)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-07 17:11:27,774 - Train: 33.54% [638000/1902000] [335.4/1000.0] [batch_t 1.425 (1.431)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-07 17:13:50,674 - Train: 33.55% [638100/1902000] [335.5/1000.0] [batch_t 1.436 (1.429)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-07 17:16:13,917 - Train: 33.55% [638200/1902000] [335.5/1000.0] [batch_t 1.432 (1.432)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-07 17:18:37,097 - Train: 33.56% [638300/1902000] [335.6/1000.0] [batch_t 1.429 (1.432)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-07 17:21:00,258 - Train: 33.56% [638400/1902000] [335.6/1000.0] [batch_t 1.431 (1.432)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-07 17:23:23,574 - Train: 33.57% [638500/1902000] [335.7/1000.0] [batch_t 1.426 (1.433)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-07 17:25:46,711 - Train: 33.58% [638600/1902000] [335.8/1000.0] [batch_t 1.438 (1.431)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-07 17:28:09,775 - Train: 33.58% [638700/1902000] [335.8/1000.0] [batch_t 1.432 (1.431)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-07 17:30:32,952 - Train: 33.59% [638800/1902000] [335.9/1000.0] [batch_t 1.430 (1.432)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-07 17:32:56,030 - Train: 33.59% [638900/1902000] [335.9/1000.0] [batch_t 1.426 (1.431)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-07 17:35:19,286 - Train: 33.60% [639000/1902000] [336.0/1000.0] [batch_t 1.430 (1.432)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-07 17:37:02,394 - ==> Total time: 5 days, 16:36:41 Eta: 11 days, 5:58:14 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 17:37:47,570 - Train: 33.60% [639100/1902000] [336.0/1000.0] [batch_t 1.435 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-07 17:40:10,776 - Train: 33.61% [639200/1902000] [336.1/1000.0] [batch_t 1.429 (1.432)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-07 17:42:34,048 - Train: 33.61% [639300/1902000] [336.1/1000.0] [batch_t 1.433 (1.433)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-07 17:44:57,317 - Train: 33.62% [639400/1902000] [336.2/1000.0] [batch_t 1.434 (1.433)] [data_t 0.003] [optim_t 1.431] [lr 0.005000] +2024-04-07 17:47:20,670 - Train: 33.62% [639500/1902000] [336.2/1000.0] [batch_t 1.430 (1.433)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-07 17:49:43,903 - Train: 33.63% [639600/1902000] [336.3/1000.0] [batch_t 1.431 (1.432)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-07 17:52:07,155 - Train: 33.63% [639700/1902000] [336.3/1000.0] [batch_t 1.435 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-07 17:54:30,489 - Train: 33.64% [639800/1902000] [336.4/1000.0] [batch_t 1.431 (1.433)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-07 17:56:53,975 - Train: 33.64% [639900/1902000] [336.4/1000.0] [batch_t 1.430 (1.435)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-07 17:59:17,333 - Train: 33.65% [640000/1902000] [336.5/1000.0] [batch_t 1.447 (1.433)] [data_t 0.002] [optim_t 1.445] [lr 0.005000] +2024-04-07 18:01:40,609 - Train: 33.65% [640100/1902000] [336.5/1000.0] [batch_t 1.431 (1.433)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-07 18:04:03,799 - Train: 33.66% [640200/1902000] [336.6/1000.0] [batch_t 1.417 (1.432)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-07 18:06:27,113 - Train: 33.66% [640300/1902000] [336.6/1000.0] [batch_t 1.434 (1.433)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-07 18:08:50,320 - Train: 33.67% [640400/1902000] [336.7/1000.0] [batch_t 1.435 (1.432)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-07 18:11:13,693 - Train: 33.68% [640500/1902000] [336.8/1000.0] [batch_t 1.436 (1.434)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-07 18:13:36,938 - Train: 33.68% [640600/1902000] [336.8/1000.0] [batch_t 1.442 (1.432)] [data_t 0.002] [optim_t 1.440] [lr 0.005000] +2024-04-07 18:16:00,234 - Train: 33.69% [640700/1902000] [336.9/1000.0] [batch_t 1.440 (1.433)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-07 18:18:23,535 - Train: 33.69% [640800/1902000] [336.9/1000.0] [batch_t 1.427 (1.433)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-07 18:20:47,069 - Train: 33.70% [640900/1902000] [337.0/1000.0] [batch_t 1.437 (1.435)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-07 18:22:33,053 - ==> Total time: 5 days, 17:22:12 Eta: 11 days, 6:15:23 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 18:23:12,432 - Train: 33.70% [641000/1902000] [337.0/1000.0] [batch_t 1.438 (1.431)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-07 18:25:35,651 - Train: 33.71% [641100/1902000] [337.1/1000.0] [batch_t 1.432 (1.432)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-07 18:27:59,023 - Train: 33.71% [641200/1902000] [337.1/1000.0] [batch_t 1.432 (1.434)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-07 18:30:22,234 - Train: 33.72% [641300/1902000] [337.2/1000.0] [batch_t 1.436 (1.432)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-07 18:32:45,561 - Train: 33.72% [641400/1902000] [337.2/1000.0] [batch_t 1.441 (1.433)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-07 18:35:08,848 - Train: 33.73% [641500/1902000] [337.3/1000.0] [batch_t 1.442 (1.433)] [data_t 0.003] [optim_t 1.439] [lr 0.005000] +2024-04-07 18:37:32,042 - Train: 33.73% [641600/1902000] [337.3/1000.0] [batch_t 1.442 (1.432)] [data_t 0.003] [optim_t 1.439] [lr 0.005000] +2024-04-07 18:39:55,237 - Train: 33.74% [641700/1902000] [337.4/1000.0] [batch_t 1.422 (1.432)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-07 18:42:18,253 - Train: 33.74% [641800/1902000] [337.4/1000.0] [batch_t 1.424 (1.430)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-07 18:44:41,333 - Train: 33.75% [641900/1902000] [337.5/1000.0] [batch_t 1.435 (1.431)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-07 18:47:04,381 - Train: 33.75% [642000/1902000] [337.5/1000.0] [batch_t 1.435 (1.430)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-07 18:49:27,486 - Train: 33.76% [642100/1902000] [337.6/1000.0] [batch_t 1.422 (1.431)] [data_t 0.002] [optim_t 1.420] [lr 0.005000] +2024-04-07 18:51:50,519 - Train: 33.76% [642200/1902000] [337.6/1000.0] [batch_t 1.444 (1.430)] [data_t 0.002] [optim_t 1.441] [lr 0.005000] +2024-04-07 18:54:13,684 - Train: 33.77% [642300/1902000] [337.7/1000.0] [batch_t 1.437 (1.432)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-07 18:56:36,828 - Train: 33.77% [642400/1902000] [337.7/1000.0] [batch_t 1.441 (1.431)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-07 18:58:59,880 - Train: 33.78% [642500/1902000] [337.8/1000.0] [batch_t 1.423 (1.430)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-07 19:01:23,024 - Train: 33.79% [642600/1902000] [337.9/1000.0] [batch_t 1.432 (1.431)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-07 19:03:46,143 - Train: 33.79% [642700/1902000] [337.9/1000.0] [batch_t 1.440 (1.431)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-07 19:06:09,278 - Train: 33.80% [642800/1902000] [338.0/1000.0] [batch_t 1.438 (1.431)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-07 19:07:57,945 - ==> Total time: 5 days, 18:07:37 Eta: 11 days, 6:31:58 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 19:08:34,156 - Train: 33.80% [642900/1902000] [338.0/1000.0] [batch_t 1.424 (1.429)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-07 19:10:57,224 - Train: 33.81% [643000/1902000] [338.1/1000.0] [batch_t 1.426 (1.431)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-07 19:13:20,264 - Train: 33.81% [643100/1902000] [338.1/1000.0] [batch_t 1.439 (1.430)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-07 19:15:43,293 - Train: 33.82% [643200/1902000] [338.2/1000.0] [batch_t 1.427 (1.430)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-07 19:18:06,450 - Train: 33.82% [643300/1902000] [338.2/1000.0] [batch_t 1.440 (1.431)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-07 19:20:29,537 - Train: 33.83% [643400/1902000] [338.3/1000.0] [batch_t 1.435 (1.431)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-07 19:22:52,745 - Train: 33.83% [643500/1902000] [338.3/1000.0] [batch_t 1.417 (1.432)] [data_t 0.003] [optim_t 1.414] [lr 0.005000] +2024-04-07 19:25:15,740 - Train: 33.84% [643600/1902000] [338.4/1000.0] [batch_t 1.426 (1.430)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-07 19:27:38,920 - Train: 33.84% [643700/1902000] [338.4/1000.0] [batch_t 1.432 (1.432)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-07 19:30:01,978 - Train: 33.85% [643800/1902000] [338.5/1000.0] [batch_t 1.425 (1.430)] [data_t 0.002] [optim_t 1.422] [lr 0.005000] +2024-04-07 19:32:25,209 - Train: 33.85% [643900/1902000] [338.5/1000.0] [batch_t 1.431 (1.432)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-07 19:34:48,342 - Train: 33.86% [644000/1902000] [338.6/1000.0] [batch_t 1.426 (1.431)] [data_t 0.002] [optim_t 1.423] [lr 0.005000] +2024-04-07 19:37:11,469 - Train: 33.86% [644100/1902000] [338.6/1000.0] [batch_t 1.436 (1.431)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-07 19:39:34,632 - Train: 33.87% [644200/1902000] [338.7/1000.0] [batch_t 1.441 (1.432)] [data_t 0.003] [optim_t 1.439] [lr 0.005000] +2024-04-07 19:41:57,993 - Train: 33.87% [644300/1902000] [338.7/1000.0] [batch_t 1.438 (1.434)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-07 19:44:21,232 - Train: 33.88% [644400/1902000] [338.8/1000.0] [batch_t 1.440 (1.432)] [data_t 0.002] [optim_t 1.438] [lr 0.005000] +2024-04-07 19:46:44,492 - Train: 33.89% [644500/1902000] [338.9/1000.0] [batch_t 1.441 (1.433)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-07 19:49:07,720 - Train: 33.89% [644600/1902000] [338.9/1000.0] [batch_t 1.435 (1.432)] [data_t 0.002] [optim_t 1.433] [lr 0.005000] +2024-04-07 19:51:30,974 - Train: 33.90% [644700/1902000] [339.0/1000.0] [batch_t 1.427 (1.432)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-07 19:53:22,704 - ==> Total time: 5 days, 18:53:02 Eta: 11 days, 6:48:11 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 19:53:56,320 - Train: 33.90% [644800/1902000] [339.0/1000.0] [batch_t 1.437 (1.437)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-07 19:56:19,469 - Train: 33.91% [644900/1902000] [339.1/1000.0] [batch_t 1.441 (1.431)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-07 19:58:42,697 - Train: 33.91% [645000/1902000] [339.1/1000.0] [batch_t 1.433 (1.432)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-07 20:01:06,066 - Train: 33.92% [645100/1902000] [339.2/1000.0] [batch_t 1.431 (1.434)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-07 20:03:29,480 - Train: 33.92% [645200/1902000] [339.2/1000.0] [batch_t 1.433 (1.434)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-07 20:05:52,704 - Train: 33.93% [645300/1902000] [339.3/1000.0] [batch_t 1.444 (1.432)] [data_t 0.002] [optim_t 1.442] [lr 0.005000] +2024-04-07 20:08:15,965 - Train: 33.93% [645400/1902000] [339.3/1000.0] [batch_t 1.430 (1.433)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-07 20:10:39,021 - Train: 33.94% [645500/1902000] [339.4/1000.0] [batch_t 1.429 (1.430)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-07 20:13:02,220 - Train: 33.94% [645600/1902000] [339.4/1000.0] [batch_t 1.436 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-07 20:15:25,487 - Train: 33.95% [645700/1902000] [339.5/1000.0] [batch_t 1.423 (1.433)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-07 20:17:48,578 - Train: 33.95% [645800/1902000] [339.5/1000.0] [batch_t 1.420 (1.431)] [data_t 0.002] [optim_t 1.417] [lr 0.005000] +2024-04-07 20:20:11,896 - Train: 33.96% [645900/1902000] [339.6/1000.0] [batch_t 1.424 (1.433)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-07 20:22:35,156 - Train: 33.96% [646000/1902000] [339.6/1000.0] [batch_t 1.438 (1.433)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-07 20:24:58,398 - Train: 33.97% [646100/1902000] [339.7/1000.0] [batch_t 1.438 (1.432)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-07 20:27:21,614 - Train: 33.97% [646200/1902000] [339.7/1000.0] [batch_t 1.423 (1.432)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-07 20:29:44,819 - Train: 33.98% [646300/1902000] [339.8/1000.0] [batch_t 1.426 (1.432)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-07 20:32:08,280 - Train: 33.99% [646400/1902000] [339.9/1000.0] [batch_t 1.435 (1.435)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-07 20:34:31,500 - Train: 33.99% [646500/1902000] [339.9/1000.0] [batch_t 1.427 (1.432)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-07 20:36:54,632 - Train: 34.00% [646600/1902000] [340.0/1000.0] [batch_t 1.438 (1.431)] [data_t 0.002] [optim_t 1.436] [lr 0.005000] +2024-04-07 20:38:49,254 - ==> Total time: 5 days, 19:38:28 Eta: 11 days, 7:04:06 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 20:39:19,934 - Train: 34.00% [646700/1902000] [340.0/1000.0] [batch_t 1.430 (1.436)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-07 20:41:43,129 - Train: 34.01% [646800/1902000] [340.1/1000.0] [batch_t 1.424 (1.432)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-07 20:44:06,386 - Train: 34.01% [646900/1902000] [340.1/1000.0] [batch_t 1.436 (1.432)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-07 20:46:29,608 - Train: 34.02% [647000/1902000] [340.2/1000.0] [batch_t 1.432 (1.432)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-07 20:48:52,930 - Train: 34.02% [647100/1902000] [340.2/1000.0] [batch_t 1.438 (1.433)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-07 20:51:16,268 - Train: 34.03% [647200/1902000] [340.3/1000.0] [batch_t 1.427 (1.433)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-07 20:53:39,450 - Train: 34.03% [647300/1902000] [340.3/1000.0] [batch_t 1.436 (1.432)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-07 20:56:02,645 - Train: 34.04% [647400/1902000] [340.4/1000.0] [batch_t 1.441 (1.432)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-07 20:58:25,896 - Train: 34.04% [647500/1902000] [340.4/1000.0] [batch_t 1.435 (1.432)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-07 21:00:49,134 - Train: 34.05% [647600/1902000] [340.5/1000.0] [batch_t 1.433 (1.432)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-07 21:03:12,189 - Train: 34.05% [647700/1902000] [340.5/1000.0] [batch_t 1.425 (1.430)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-07 21:05:35,374 - Train: 34.06% [647800/1902000] [340.6/1000.0] [batch_t 1.422 (1.432)] [data_t 0.002] [optim_t 1.419] [lr 0.005000] +2024-04-07 21:07:58,589 - Train: 34.06% [647900/1902000] [340.6/1000.0] [batch_t 1.430 (1.432)] [data_t 0.002] [optim_t 1.428] [lr 0.005000] +2024-04-07 21:10:21,660 - Train: 34.07% [648000/1902000] [340.7/1000.0] [batch_t 1.447 (1.431)] [data_t 0.003] [optim_t 1.445] [lr 0.005000] +2024-04-07 21:12:44,781 - Train: 34.07% [648100/1902000] [340.7/1000.0] [batch_t 1.432 (1.431)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-07 21:15:08,037 - Train: 34.08% [648200/1902000] [340.8/1000.0] [batch_t 1.430 (1.432)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-07 21:17:31,243 - Train: 34.09% [648300/1902000] [340.9/1000.0] [batch_t 1.436 (1.432)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-07 21:19:54,522 - Train: 34.09% [648400/1902000] [340.9/1000.0] [batch_t 1.421 (1.433)] [data_t 0.003] [optim_t 1.418] [lr 0.005000] +2024-04-07 21:22:17,686 - Train: 34.10% [648500/1902000] [341.0/1000.0] [batch_t 1.442 (1.432)] [data_t 0.003] [optim_t 1.439] [lr 0.005000] +2024-04-07 21:24:15,185 - ==> Total time: 5 days, 20:23:54 Eta: 11 days, 7:19:38 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 21:24:42,913 - Train: 34.10% [648600/1902000] [341.0/1000.0] [batch_t 1.427 (1.431)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-07 21:27:06,073 - Train: 34.11% [648700/1902000] [341.1/1000.0] [batch_t 1.431 (1.432)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-07 21:29:29,437 - Train: 34.11% [648800/1902000] [341.1/1000.0] [batch_t 1.437 (1.434)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-07 21:31:52,634 - Train: 34.12% [648900/1902000] [341.2/1000.0] [batch_t 1.437 (1.432)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-07 21:34:15,752 - Train: 34.12% [649000/1902000] [341.2/1000.0] [batch_t 1.429 (1.431)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-07 21:36:38,974 - Train: 34.13% [649100/1902000] [341.3/1000.0] [batch_t 1.435 (1.432)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-07 21:39:02,207 - Train: 34.13% [649200/1902000] [341.3/1000.0] [batch_t 1.426 (1.432)] [data_t 0.002] [optim_t 1.424] [lr 0.005000] +2024-04-07 21:41:25,362 - Train: 34.14% [649300/1902000] [341.4/1000.0] [batch_t 1.427 (1.431)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-07 21:43:48,597 - Train: 34.14% [649400/1902000] [341.4/1000.0] [batch_t 1.439 (1.432)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-07 21:46:11,743 - Train: 34.15% [649500/1902000] [341.5/1000.0] [batch_t 1.420 (1.431)] [data_t 0.002] [optim_t 1.417] [lr 0.005000] +2024-04-07 21:48:34,942 - Train: 34.15% [649600/1902000] [341.5/1000.0] [batch_t 1.437 (1.432)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-07 21:50:58,283 - Train: 34.16% [649700/1902000] [341.6/1000.0] [batch_t 1.431 (1.433)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-07 21:53:21,574 - Train: 34.16% [649800/1902000] [341.6/1000.0] [batch_t 1.442 (1.433)] [data_t 0.002] [optim_t 1.440] [lr 0.005000] +2024-04-07 21:55:44,675 - Train: 34.17% [649900/1902000] [341.7/1000.0] [batch_t 1.431 (1.431)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-07 21:58:08,004 - Train: 34.17% [650000/1902000] [341.7/1000.0] [batch_t 1.432 (1.433)] [data_t 0.003] [optim_t 1.430] [lr 0.005000] +2024-04-07 22:00:31,419 - Train: 34.18% [650100/1902000] [341.8/1000.0] [batch_t 1.442 (1.434)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-07 22:02:54,715 - Train: 34.19% [650200/1902000] [341.9/1000.0] [batch_t 1.436 (1.433)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-07 22:05:17,996 - Train: 34.19% [650300/1902000] [341.9/1000.0] [batch_t 1.445 (1.433)] [data_t 0.002] [optim_t 1.443] [lr 0.005000] +2024-04-07 22:07:41,418 - Train: 34.20% [650400/1902000] [342.0/1000.0] [batch_t 1.436 (1.434)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-07 22:09:41,802 - ==> Total time: 5 days, 21:09:21 Eta: 11 days, 7:34:50 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 22:10:07,122 - Train: 34.20% [650500/1902000] [342.0/1000.0] [batch_t 1.438 (1.437)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-07 22:12:30,388 - Train: 34.21% [650600/1902000] [342.1/1000.0] [batch_t 1.425 (1.433)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-07 22:14:53,876 - Train: 34.21% [650700/1902000] [342.1/1000.0] [batch_t 1.428 (1.435)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-07 22:17:17,255 - Train: 34.22% [650800/1902000] [342.2/1000.0] [batch_t 1.422 (1.434)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-07 22:19:40,510 - Train: 34.22% [650900/1902000] [342.2/1000.0] [batch_t 1.437 (1.432)] [data_t 0.002] [optim_t 1.435] [lr 0.005000] +2024-04-07 22:22:03,894 - Train: 34.23% [651000/1902000] [342.3/1000.0] [batch_t 1.433 (1.434)] [data_t 0.002] [optim_t 1.430] [lr 0.005000] +2024-04-07 22:24:27,243 - Train: 34.23% [651100/1902000] [342.3/1000.0] [batch_t 1.420 (1.433)] [data_t 0.002] [optim_t 1.418] [lr 0.005000] +2024-04-07 22:26:50,562 - Train: 34.24% [651200/1902000] [342.4/1000.0] [batch_t 1.432 (1.433)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-07 22:29:13,920 - Train: 34.24% [651300/1902000] [342.4/1000.0] [batch_t 1.428 (1.433)] [data_t 0.003] [optim_t 1.425] [lr 0.005000] +2024-04-07 22:31:37,204 - Train: 34.25% [651400/1902000] [342.5/1000.0] [batch_t 1.430 (1.433)] [data_t 0.003] [optim_t 1.427] [lr 0.005000] +2024-04-07 22:34:00,427 - Train: 34.25% [651500/1902000] [342.5/1000.0] [batch_t 1.439 (1.432)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-07 22:36:23,628 - Train: 34.26% [651600/1902000] [342.6/1000.0] [batch_t 1.427 (1.432)] [data_t 0.002] [optim_t 1.425] [lr 0.005000] +2024-04-07 22:38:47,022 - Train: 34.26% [651700/1902000] [342.6/1000.0] [batch_t 1.421 (1.434)] [data_t 0.003] [optim_t 1.419] [lr 0.005000] +2024-04-07 22:41:10,288 - Train: 34.27% [651800/1902000] [342.7/1000.0] [batch_t 1.446 (1.433)] [data_t 0.002] [optim_t 1.444] [lr 0.005000] +2024-04-07 22:43:33,560 - Train: 34.27% [651900/1902000] [342.7/1000.0] [batch_t 1.432 (1.433)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-07 22:45:56,731 - Train: 34.28% [652000/1902000] [342.8/1000.0] [batch_t 1.427 (1.432)] [data_t 0.003] [optim_t 1.424] [lr 0.005000] +2024-04-07 22:48:19,991 - Train: 34.28% [652100/1902000] [342.8/1000.0] [batch_t 1.429 (1.432)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-07 22:50:43,301 - Train: 34.29% [652200/1902000] [342.9/1000.0] [batch_t 1.428 (1.433)] [data_t 0.002] [optim_t 1.426] [lr 0.005000] +2024-04-07 22:53:06,595 - Train: 34.30% [652300/1902000] [343.0/1000.0] [batch_t 1.437 (1.433)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-07 22:55:09,674 - ==> Total time: 5 days, 21:54:49 Eta: 11 days, 7:49:43 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 22:55:31,970 - Train: 34.30% [652400/1902000] [343.0/1000.0] [batch_t 1.444 (1.432)] [data_t 0.003] [optim_t 1.442] [lr 0.005000] +2024-04-07 22:57:55,255 - Train: 34.31% [652500/1902000] [343.1/1000.0] [batch_t 1.441 (1.433)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-07 23:00:18,595 - Train: 34.31% [652600/1902000] [343.1/1000.0] [batch_t 1.437 (1.433)] [data_t 0.002] [optim_t 1.434] [lr 0.005000] +2024-04-07 23:02:41,865 - Train: 34.32% [652700/1902000] [343.2/1000.0] [batch_t 1.434 (1.433)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-07 23:05:05,151 - Train: 34.32% [652800/1902000] [343.2/1000.0] [batch_t 1.445 (1.433)] [data_t 0.002] [optim_t 1.442] [lr 0.005000] +2024-04-07 23:07:28,321 - Train: 34.33% [652900/1902000] [343.3/1000.0] [batch_t 1.431 (1.432)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-07 23:09:51,508 - Train: 34.33% [653000/1902000] [343.3/1000.0] [batch_t 1.431 (1.432)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-07 23:12:14,791 - Train: 34.34% [653100/1902000] [343.4/1000.0] [batch_t 1.426 (1.433)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-07 23:14:38,073 - Train: 34.34% [653200/1902000] [343.4/1000.0] [batch_t 1.432 (1.433)] [data_t 0.003] [optim_t 1.429] [lr 0.005000] +2024-04-07 23:17:01,217 - Train: 34.35% [653300/1902000] [343.5/1000.0] [batch_t 1.434 (1.431)] [data_t 0.002] [optim_t 1.432] [lr 0.005000] +2024-04-07 23:19:24,561 - Train: 34.35% [653400/1902000] [343.5/1000.0] [batch_t 1.423 (1.433)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-07 23:21:47,749 - Train: 34.36% [653500/1902000] [343.6/1000.0] [batch_t 1.426 (1.432)] [data_t 0.003] [optim_t 1.423] [lr 0.005000] +2024-04-07 23:24:11,067 - Train: 34.36% [653600/1902000] [343.6/1000.0] [batch_t 1.434 (1.433)] [data_t 0.003] [optim_t 1.432] [lr 0.005000] +2024-04-07 23:26:34,175 - Train: 34.37% [653700/1902000] [343.7/1000.0] [batch_t 1.425 (1.431)] [data_t 0.003] [optim_t 1.422] [lr 0.005000] +2024-04-07 23:28:57,479 - Train: 34.37% [653800/1902000] [343.7/1000.0] [batch_t 1.423 (1.433)] [data_t 0.002] [optim_t 1.420] [lr 0.005000] +2024-04-07 23:31:20,848 - Train: 34.38% [653900/1902000] [343.8/1000.0] [batch_t 1.448 (1.434)] [data_t 0.003] [optim_t 1.445] [lr 0.005000] +2024-04-07 23:33:44,190 - Train: 34.38% [654000/1902000] [343.8/1000.0] [batch_t 1.431 (1.433)] [data_t 0.003] [optim_t 1.428] [lr 0.005000] +2024-04-07 23:36:07,523 - Train: 34.39% [654100/1902000] [343.9/1000.0] [batch_t 1.423 (1.433)] [data_t 0.003] [optim_t 1.420] [lr 0.005000] +2024-04-07 23:38:30,832 - Train: 34.40% [654200/1902000] [344.0/1000.0] [batch_t 1.439 (1.433)] [data_t 0.002] [optim_t 1.437] [lr 0.005000] +2024-04-07 23:40:36,927 - ==> Total time: 5 days, 22:40:16 Eta: 11 days, 8:04:14 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-07 23:40:56,518 - Train: 34.40% [654300/1902000] [344.0/1000.0] [batch_t 1.431 (1.441)] [data_t 0.002] [optim_t 1.429] [lr 0.005000] +2024-04-07 23:43:19,783 - Train: 34.41% [654400/1902000] [344.1/1000.0] [batch_t 1.433 (1.433)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-07 23:45:43,129 - Train: 34.41% [654500/1902000] [344.1/1000.0] [batch_t 1.441 (1.433)] [data_t 0.003] [optim_t 1.438] [lr 0.005000] +2024-04-07 23:48:06,381 - Train: 34.42% [654600/1902000] [344.2/1000.0] [batch_t 1.429 (1.432)] [data_t 0.002] [optim_t 1.427] [lr 0.005000] +2024-04-07 23:50:29,668 - Train: 34.42% [654700/1902000] [344.2/1000.0] [batch_t 1.433 (1.433)] [data_t 0.002] [optim_t 1.431] [lr 0.005000] +2024-04-07 23:52:52,860 - Train: 34.43% [654800/1902000] [344.3/1000.0] [batch_t 1.436 (1.432)] [data_t 0.003] [optim_t 1.433] [lr 0.005000] +2024-04-07 23:55:16,167 - Train: 34.43% [654900/1902000] [344.3/1000.0] [batch_t 1.429 (1.433)] [data_t 0.003] [optim_t 1.426] [lr 0.005000] +2024-04-07 23:57:39,440 - Train: 34.44% [655000/1902000] [344.4/1000.0] [batch_t 1.441 (1.433)] [data_t 0.002] [optim_t 1.439] [lr 0.005000] +2024-04-08 00:00:02,679 - Train: 34.44% [655100/1902000] [344.4/1000.0] [batch_t 1.437 (1.432)] [data_t 0.003] [optim_t 1.434] [lr 0.005000] +2024-04-08 00:02:27,031 - Train: 34.45% [655200/1902000] [344.5/1000.0] [batch_t 1.460 (1.443)] [data_t 0.003] [optim_t 1.456] [lr 0.005000] +2024-04-08 00:04:52,674 - Train: 34.45% [655300/1902000] [344.5/1000.0] [batch_t 1.455 (1.456)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 00:07:18,344 - Train: 34.46% [655400/1902000] [344.6/1000.0] [batch_t 1.460 (1.457)] [data_t 0.003] [optim_t 1.456] [lr 0.005000] +2024-04-08 00:09:44,010 - Train: 34.46% [655500/1902000] [344.6/1000.0] [batch_t 1.459 (1.457)] [data_t 0.003] [optim_t 1.456] [lr 0.005000] +2024-04-08 00:12:09,545 - Train: 34.47% [655600/1902000] [344.7/1000.0] [batch_t 1.457 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 00:14:35,101 - Train: 34.47% [655700/1902000] [344.7/1000.0] [batch_t 1.461 (1.455)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 00:17:00,724 - Train: 34.48% [655800/1902000] [344.8/1000.0] [batch_t 1.455 (1.456)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 00:19:26,291 - Train: 34.48% [655900/1902000] [344.8/1000.0] [batch_t 1.450 (1.456)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-08 00:21:51,884 - Train: 34.49% [656000/1902000] [344.9/1000.0] [batch_t 1.451 (1.456)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-08 00:24:17,482 - Train: 34.50% [656100/1902000] [345.0/1000.0] [batch_t 1.451 (1.456)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-08 00:26:28,503 - ==> Total time: 5 days, 23:26:08 Eta: 11 days, 8:19:10 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-08 00:26:45,244 - Train: 34.50% [656200/1902000] [345.0/1000.0] [batch_t 1.449 (1.456)] [data_t 0.003] [optim_t 1.446] [lr 0.005000] +2024-04-08 00:29:10,853 - Train: 34.51% [656300/1902000] [345.1/1000.0] [batch_t 1.462 (1.456)] [data_t 0.003] [optim_t 1.459] [lr 0.005000] +2024-04-08 00:31:36,400 - Train: 34.51% [656400/1902000] [345.1/1000.0] [batch_t 1.456 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 00:34:01,934 - Train: 34.52% [656500/1902000] [345.2/1000.0] [batch_t 1.448 (1.455)] [data_t 0.003] [optim_t 1.444] [lr 0.005000] +2024-04-08 00:36:27,503 - Train: 34.52% [656600/1902000] [345.2/1000.0] [batch_t 1.457 (1.456)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 00:38:53,168 - Train: 34.53% [656700/1902000] [345.3/1000.0] [batch_t 1.457 (1.457)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 00:41:18,717 - Train: 34.53% [656800/1902000] [345.3/1000.0] [batch_t 1.457 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 00:43:44,269 - Train: 34.54% [656900/1902000] [345.4/1000.0] [batch_t 1.458 (1.455)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 00:46:09,948 - Train: 34.54% [657000/1902000] [345.4/1000.0] [batch_t 1.452 (1.457)] [data_t 0.004] [optim_t 1.449] [lr 0.005000] +2024-04-08 00:48:35,533 - Train: 34.55% [657100/1902000] [345.5/1000.0] [batch_t 1.457 (1.456)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 00:51:01,166 - Train: 34.55% [657200/1902000] [345.5/1000.0] [batch_t 1.461 (1.456)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 00:53:26,776 - Train: 34.56% [657300/1902000] [345.6/1000.0] [batch_t 1.457 (1.456)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 00:55:52,346 - Train: 34.56% [657400/1902000] [345.6/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 00:58:17,887 - Train: 34.57% [657500/1902000] [345.7/1000.0] [batch_t 1.461 (1.455)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 01:00:43,434 - Train: 34.57% [657600/1902000] [345.7/1000.0] [batch_t 1.456 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 01:03:09,011 - Train: 34.58% [657700/1902000] [345.8/1000.0] [batch_t 1.450 (1.456)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-08 01:05:34,567 - Train: 34.58% [657800/1902000] [345.8/1000.0] [batch_t 1.459 (1.455)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 01:08:00,163 - Train: 34.59% [657900/1902000] [345.9/1000.0] [batch_t 1.462 (1.456)] [data_t 0.003] [optim_t 1.459] [lr 0.005000] +2024-04-08 01:10:25,763 - Train: 34.60% [658000/1902000] [346.0/1000.0] [batch_t 1.451 (1.456)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-08 01:12:39,682 - ==> Total time: 6 days, 0:12:19 Eta: 11 days, 8:34:23 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-08 01:12:53,281 - Train: 34.60% [658100/1902000] [346.0/1000.0] [batch_t 1.455 (1.457)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 01:15:18,875 - Train: 34.61% [658200/1902000] [346.1/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 01:17:44,479 - Train: 34.61% [658300/1902000] [346.1/1000.0] [batch_t 1.454 (1.456)] [data_t 0.003] [optim_t 1.450] [lr 0.005000] +2024-04-08 01:20:10,052 - Train: 34.62% [658400/1902000] [346.2/1000.0] [batch_t 1.460 (1.456)] [data_t 0.003] [optim_t 1.456] [lr 0.005000] +2024-04-08 01:22:35,599 - Train: 34.62% [658500/1902000] [346.2/1000.0] [batch_t 1.451 (1.455)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-08 01:25:01,248 - Train: 34.63% [658600/1902000] [346.3/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 01:27:26,879 - Train: 34.63% [658700/1902000] [346.3/1000.0] [batch_t 1.451 (1.456)] [data_t 0.004] [optim_t 1.447] [lr 0.005000] +2024-04-08 01:29:52,529 - Train: 34.64% [658800/1902000] [346.4/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 01:32:18,124 - Train: 34.64% [658900/1902000] [346.4/1000.0] [batch_t 1.458 (1.456)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 01:34:43,695 - Train: 34.65% [659000/1902000] [346.5/1000.0] [batch_t 1.461 (1.456)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 01:37:09,350 - Train: 34.65% [659100/1902000] [346.5/1000.0] [batch_t 1.455 (1.456)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 01:39:34,900 - Train: 34.66% [659200/1902000] [346.6/1000.0] [batch_t 1.458 (1.455)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 01:42:00,573 - Train: 34.66% [659300/1902000] [346.6/1000.0] [batch_t 1.458 (1.457)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 01:44:26,167 - Train: 34.67% [659400/1902000] [346.7/1000.0] [batch_t 1.461 (1.456)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 01:46:51,804 - Train: 34.67% [659500/1902000] [346.7/1000.0] [batch_t 1.452 (1.456)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-08 01:49:17,385 - Train: 34.68% [659600/1902000] [346.8/1000.0] [batch_t 1.461 (1.456)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 01:51:42,933 - Train: 34.68% [659700/1902000] [346.8/1000.0] [batch_t 1.459 (1.455)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 01:54:08,452 - Train: 34.69% [659800/1902000] [346.9/1000.0] [batch_t 1.460 (1.455)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 01:56:33,999 - Train: 34.70% [659900/1902000] [347.0/1000.0] [batch_t 1.462 (1.455)] [data_t 0.003] [optim_t 1.459] [lr 0.005000] +2024-04-08 01:58:50,901 - ==> Total time: 6 days, 0:58:30 Eta: 11 days, 8:49:14 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-08 01:59:01,807 - Train: 34.70% [660000/1902000] [347.0/1000.0] [batch_t 1.457 (1.455)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 02:01:27,452 - Train: 34.71% [660100/1902000] [347.1/1000.0] [batch_t 1.461 (1.456)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 02:03:52,978 - Train: 34.71% [660200/1902000] [347.1/1000.0] [batch_t 1.461 (1.455)] [data_t 0.004] [optim_t 1.458] [lr 0.005000] +2024-04-08 02:06:18,518 - Train: 34.72% [660300/1902000] [347.2/1000.0] [batch_t 1.452 (1.455)] [data_t 0.003] [optim_t 1.449] [lr 0.005000] +2024-04-08 02:08:44,062 - Train: 34.72% [660400/1902000] [347.2/1000.0] [batch_t 1.444 (1.455)] [data_t 0.003] [optim_t 1.441] [lr 0.005000] +2024-04-08 02:11:09,622 - Train: 34.73% [660500/1902000] [347.3/1000.0] [batch_t 1.461 (1.456)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 02:13:35,201 - Train: 34.73% [660600/1902000] [347.3/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 02:16:00,767 - Train: 34.74% [660700/1902000] [347.4/1000.0] [batch_t 1.458 (1.456)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 02:18:26,339 - Train: 34.74% [660800/1902000] [347.4/1000.0] [batch_t 1.460 (1.456)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 02:20:51,955 - Train: 34.75% [660900/1902000] [347.5/1000.0] [batch_t 1.457 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 02:23:17,614 - Train: 34.75% [661000/1902000] [347.5/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 02:25:43,215 - Train: 34.76% [661100/1902000] [347.6/1000.0] [batch_t 1.462 (1.456)] [data_t 0.003] [optim_t 1.459] [lr 0.005000] +2024-04-08 02:28:08,796 - Train: 34.76% [661200/1902000] [347.6/1000.0] [batch_t 1.461 (1.456)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 02:30:34,433 - Train: 34.77% [661300/1902000] [347.7/1000.0] [batch_t 1.455 (1.456)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 02:33:00,016 - Train: 34.77% [661400/1902000] [347.7/1000.0] [batch_t 1.468 (1.456)] [data_t 0.003] [optim_t 1.464] [lr 0.005000] +2024-04-08 02:35:25,617 - Train: 34.78% [661500/1902000] [347.8/1000.0] [batch_t 1.450 (1.456)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-08 02:37:51,146 - Train: 34.78% [661600/1902000] [347.8/1000.0] [batch_t 1.456 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 02:40:16,743 - Train: 34.79% [661700/1902000] [347.9/1000.0] [batch_t 1.461 (1.456)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 02:42:42,338 - Train: 34.79% [661800/1902000] [347.9/1000.0] [batch_t 1.462 (1.456)] [data_t 0.003] [optim_t 1.459] [lr 0.005000] +2024-04-08 02:45:02,112 - ==> Total time: 6 days, 1:44:41 Eta: 11 days, 9:03:44 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-08 02:45:12,105 - Train: 34.80% [661900/1902000] [348.0/1000.0] [batch_t 1.444 (1.508)] [data_t 0.003] [optim_t 1.441] [lr 0.005000] +2024-04-08 02:47:37,671 - Train: 34.81% [662000/1902000] [348.1/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 02:50:03,211 - Train: 34.81% [662100/1902000] [348.1/1000.0] [batch_t 1.456 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 02:52:28,872 - Train: 34.82% [662200/1902000] [348.2/1000.0] [batch_t 1.455 (1.457)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 02:54:54,491 - Train: 34.82% [662300/1902000] [348.2/1000.0] [batch_t 1.461 (1.456)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 02:57:20,125 - Train: 34.83% [662400/1902000] [348.3/1000.0] [batch_t 1.454 (1.456)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 02:59:45,743 - Train: 34.83% [662500/1902000] [348.3/1000.0] [batch_t 1.450 (1.456)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-08 03:02:11,370 - Train: 34.84% [662600/1902000] [348.4/1000.0] [batch_t 1.460 (1.456)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 03:04:36,916 - Train: 34.84% [662700/1902000] [348.4/1000.0] [batch_t 1.455 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 03:07:02,581 - Train: 34.85% [662800/1902000] [348.5/1000.0] [batch_t 1.456 (1.457)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 03:09:28,132 - Train: 34.85% [662900/1902000] [348.5/1000.0] [batch_t 1.456 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 03:11:53,726 - Train: 34.86% [663000/1902000] [348.6/1000.0] [batch_t 1.452 (1.456)] [data_t 0.003] [optim_t 1.449] [lr 0.005000] +2024-04-08 03:14:19,423 - Train: 34.86% [663100/1902000] [348.6/1000.0] [batch_t 1.459 (1.457)] [data_t 0.003] [optim_t 1.456] [lr 0.005000] +2024-04-08 03:16:45,021 - Train: 34.87% [663200/1902000] [348.7/1000.0] [batch_t 1.457 (1.456)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 03:19:10,579 - Train: 34.87% [663300/1902000] [348.7/1000.0] [batch_t 1.461 (1.455)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 03:21:36,176 - Train: 34.88% [663400/1902000] [348.8/1000.0] [batch_t 1.455 (1.456)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 03:24:01,781 - Train: 34.88% [663500/1902000] [348.8/1000.0] [batch_t 1.452 (1.456)] [data_t 0.003] [optim_t 1.449] [lr 0.005000] +2024-04-08 03:26:27,417 - Train: 34.89% [663600/1902000] [348.9/1000.0] [batch_t 1.454 (1.456)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 03:28:52,950 - Train: 34.89% [663700/1902000] [348.9/1000.0] [batch_t 1.452 (1.455)] [data_t 0.003] [optim_t 1.449] [lr 0.005000] +2024-04-08 03:31:15,687 - ==> Total time: 6 days, 2:30:55 Eta: 11 days, 9:17:57 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-08 03:31:20,743 - Train: 34.90% [663800/1902000] [349.0/1000.0] [batch_t 1.454 (1.455)] [data_t 0.004] [optim_t 1.451] [lr 0.005000] +2024-04-08 03:33:46,327 - Train: 34.91% [663900/1902000] [349.1/1000.0] [batch_t 1.451 (1.456)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-08 03:36:11,981 - Train: 34.91% [664000/1902000] [349.1/1000.0] [batch_t 1.447 (1.456)] [data_t 0.003] [optim_t 1.444] [lr 0.005000] +2024-04-08 03:38:37,545 - Train: 34.92% [664100/1902000] [349.2/1000.0] [batch_t 1.462 (1.456)] [data_t 0.003] [optim_t 1.459] [lr 0.005000] +2024-04-08 03:41:03,163 - Train: 34.92% [664200/1902000] [349.2/1000.0] [batch_t 1.457 (1.456)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 03:43:28,642 - Train: 34.93% [664300/1902000] [349.3/1000.0] [batch_t 1.450 (1.455)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-08 03:45:54,196 - Train: 34.93% [664400/1902000] [349.3/1000.0] [batch_t 1.465 (1.455)] [data_t 0.003] [optim_t 1.462] [lr 0.005000] +2024-04-08 03:48:19,742 - Train: 34.94% [664500/1902000] [349.4/1000.0] [batch_t 1.459 (1.455)] [data_t 0.003] [optim_t 1.456] [lr 0.005000] +2024-04-08 03:50:45,439 - Train: 34.94% [664600/1902000] [349.4/1000.0] [batch_t 1.455 (1.457)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 03:53:10,958 - Train: 34.95% [664700/1902000] [349.5/1000.0] [batch_t 1.460 (1.455)] [data_t 0.003] [optim_t 1.456] [lr 0.005000] +2024-04-08 03:55:36,554 - Train: 34.95% [664800/1902000] [349.5/1000.0] [batch_t 1.446 (1.456)] [data_t 0.003] [optim_t 1.443] [lr 0.005000] +2024-04-08 03:58:02,157 - Train: 34.96% [664900/1902000] [349.6/1000.0] [batch_t 1.446 (1.456)] [data_t 0.003] [optim_t 1.443] [lr 0.005000] +2024-04-08 04:00:27,744 - Train: 34.96% [665000/1902000] [349.6/1000.0] [batch_t 1.460 (1.456)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 04:02:53,220 - Train: 34.97% [665100/1902000] [349.7/1000.0] [batch_t 1.454 (1.455)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 04:05:18,840 - Train: 34.97% [665200/1902000] [349.7/1000.0] [batch_t 1.451 (1.456)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-08 04:07:44,493 - Train: 34.98% [665300/1902000] [349.8/1000.0] [batch_t 1.455 (1.456)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 04:10:10,058 - Train: 34.98% [665400/1902000] [349.8/1000.0] [batch_t 1.448 (1.456)] [data_t 0.003] [optim_t 1.444] [lr 0.005000] +2024-04-08 04:12:35,667 - Train: 34.99% [665500/1902000] [349.9/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 04:15:01,372 - Train: 34.99% [665600/1902000] [349.9/1000.0] [batch_t 1.457 (1.457)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 04:17:26,936 - Train: 35.00% [665700/1902000] [350.0/1000.0] [batch_t 1.455 (1.456)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 04:17:26,940 - ==> Total time: 6 days, 3:17:06 Eta: 11 days, 9:31:46 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-08 04:19:54,799 - Train: 35.01% [665800/1902000] [350.1/1000.0] [batch_t 1.455 (1.456)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 04:22:20,375 - Train: 35.01% [665900/1902000] [350.1/1000.0] [batch_t 1.460 (1.456)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 04:24:46,013 - Train: 35.02% [666000/1902000] [350.2/1000.0] [batch_t 1.459 (1.456)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 04:27:11,611 - Train: 35.02% [666100/1902000] [350.2/1000.0] [batch_t 1.455 (1.456)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 04:29:37,157 - Train: 35.03% [666200/1902000] [350.3/1000.0] [batch_t 1.461 (1.455)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 04:32:02,708 - Train: 35.03% [666300/1902000] [350.3/1000.0] [batch_t 1.453 (1.455)] [data_t 0.003] [optim_t 1.449] [lr 0.005000] +2024-04-08 04:34:28,180 - Train: 35.04% [666400/1902000] [350.4/1000.0] [batch_t 1.457 (1.455)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 04:36:53,734 - Train: 35.04% [666500/1902000] [350.4/1000.0] [batch_t 1.460 (1.455)] [data_t 0.003] [optim_t 1.456] [lr 0.005000] +2024-04-08 04:39:19,327 - Train: 35.05% [666600/1902000] [350.5/1000.0] [batch_t 1.447 (1.456)] [data_t 0.003] [optim_t 1.444] [lr 0.005000] +2024-04-08 04:41:44,807 - Train: 35.05% [666700/1902000] [350.5/1000.0] [batch_t 1.455 (1.455)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 04:44:10,423 - Train: 35.06% [666800/1902000] [350.6/1000.0] [batch_t 1.457 (1.456)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 04:46:35,964 - Train: 35.06% [666900/1902000] [350.6/1000.0] [batch_t 1.457 (1.455)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 04:49:01,511 - Train: 35.07% [667000/1902000] [350.7/1000.0] [batch_t 1.457 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 04:51:27,089 - Train: 35.07% [667100/1902000] [350.7/1000.0] [batch_t 1.460 (1.456)] [data_t 0.003] [optim_t 1.456] [lr 0.005000] +2024-04-08 04:53:52,719 - Train: 35.08% [667200/1902000] [350.8/1000.0] [batch_t 1.451 (1.456)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-08 04:56:18,339 - Train: 35.08% [667300/1902000] [350.8/1000.0] [batch_t 1.455 (1.456)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 04:58:43,862 - Train: 35.09% [667400/1902000] [350.9/1000.0] [batch_t 1.462 (1.455)] [data_t 0.003] [optim_t 1.459] [lr 0.005000] +2024-04-08 05:01:09,436 - Train: 35.09% [667500/1902000] [350.9/1000.0] [batch_t 1.451 (1.456)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-08 05:03:34,947 - Train: 35.10% [667600/1902000] [351.0/1000.0] [batch_t 1.457 (1.455)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 05:03:37,870 - ==> Total time: 6 days, 4:03:17 Eta: 11 days, 9:45:13 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-08 05:06:02,793 - Train: 35.11% [667700/1902000] [351.1/1000.0] [batch_t 1.462 (1.456)] [data_t 0.003] [optim_t 1.459] [lr 0.005000] +2024-04-08 05:08:28,429 - Train: 35.11% [667800/1902000] [351.1/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 05:10:53,978 - Train: 35.12% [667900/1902000] [351.2/1000.0] [batch_t 1.461 (1.455)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 05:13:19,571 - Train: 35.12% [668000/1902000] [351.2/1000.0] [batch_t 1.468 (1.456)] [data_t 0.004] [optim_t 1.465] [lr 0.005000] +2024-04-08 05:15:45,180 - Train: 35.13% [668100/1902000] [351.3/1000.0] [batch_t 1.455 (1.456)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 05:18:10,841 - Train: 35.13% [668200/1902000] [351.3/1000.0] [batch_t 1.451 (1.457)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-08 05:20:36,507 - Train: 35.14% [668300/1902000] [351.4/1000.0] [batch_t 1.456 (1.457)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 05:23:02,085 - Train: 35.14% [668400/1902000] [351.4/1000.0] [batch_t 1.448 (1.456)] [data_t 0.003] [optim_t 1.445] [lr 0.005000] +2024-04-08 05:25:27,730 - Train: 35.15% [668500/1902000] [351.5/1000.0] [batch_t 1.455 (1.456)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 05:27:53,421 - Train: 35.15% [668600/1902000] [351.5/1000.0] [batch_t 1.456 (1.457)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 05:30:19,001 - Train: 35.16% [668700/1902000] [351.6/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 05:32:44,563 - Train: 35.16% [668800/1902000] [351.6/1000.0] [batch_t 1.457 (1.456)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 05:35:10,021 - Train: 35.17% [668900/1902000] [351.7/1000.0] [batch_t 1.460 (1.454)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 05:37:35,565 - Train: 35.17% [669000/1902000] [351.7/1000.0] [batch_t 1.454 (1.455)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 05:40:01,194 - Train: 35.18% [669100/1902000] [351.8/1000.0] [batch_t 1.447 (1.456)] [data_t 0.003] [optim_t 1.443] [lr 0.005000] +2024-04-08 05:42:26,723 - Train: 35.18% [669200/1902000] [351.8/1000.0] [batch_t 1.460 (1.455)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 05:44:52,274 - Train: 35.19% [669300/1902000] [351.9/1000.0] [batch_t 1.460 (1.455)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 05:47:17,767 - Train: 35.19% [669400/1902000] [351.9/1000.0] [batch_t 1.455 (1.455)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 05:49:43,252 - Train: 35.20% [669500/1902000] [352.0/1000.0] [batch_t 1.456 (1.455)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 05:49:49,077 - ==> Total time: 6 days, 4:49:28 Eta: 11 days, 9:58:21 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-08 05:52:10,818 - Train: 35.21% [669600/1902000] [352.1/1000.0] [batch_t 1.452 (1.455)] [data_t 0.003] [optim_t 1.449] [lr 0.005000] +2024-04-08 05:54:36,365 - Train: 35.21% [669700/1902000] [352.1/1000.0] [batch_t 1.452 (1.455)] [data_t 0.003] [optim_t 1.449] [lr 0.005000] +2024-04-08 05:57:01,960 - Train: 35.22% [669800/1902000] [352.2/1000.0] [batch_t 1.462 (1.456)] [data_t 0.003] [optim_t 1.459] [lr 0.005000] +2024-04-08 05:59:27,437 - Train: 35.22% [669900/1902000] [352.2/1000.0] [batch_t 1.455 (1.455)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 06:01:53,017 - Train: 35.23% [670000/1902000] [352.3/1000.0] [batch_t 1.457 (1.456)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 06:04:18,536 - Train: 35.23% [670100/1902000] [352.3/1000.0] [batch_t 1.459 (1.455)] [data_t 0.003] [optim_t 1.456] [lr 0.005000] +2024-04-08 06:06:44,085 - Train: 35.24% [670200/1902000] [352.4/1000.0] [batch_t 1.455 (1.455)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 06:09:09,591 - Train: 35.24% [670300/1902000] [352.4/1000.0] [batch_t 1.458 (1.455)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 06:11:35,024 - Train: 35.25% [670400/1902000] [352.5/1000.0] [batch_t 1.460 (1.454)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 06:14:00,565 - Train: 35.25% [670500/1902000] [352.5/1000.0] [batch_t 1.457 (1.455)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 06:16:26,147 - Train: 35.26% [670600/1902000] [352.6/1000.0] [batch_t 1.462 (1.456)] [data_t 0.003] [optim_t 1.459] [lr 0.005000] +2024-04-08 06:18:51,737 - Train: 35.26% [670700/1902000] [352.6/1000.0] [batch_t 1.461 (1.456)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 06:21:17,339 - Train: 35.27% [670800/1902000] [352.7/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 06:23:42,847 - Train: 35.27% [670900/1902000] [352.7/1000.0] [batch_t 1.456 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 06:26:08,418 - Train: 35.28% [671000/1902000] [352.8/1000.0] [batch_t 1.459 (1.456)] [data_t 0.003] [optim_t 1.456] [lr 0.005000] +2024-04-08 06:28:33,997 - Train: 35.28% [671100/1902000] [352.8/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 06:30:59,497 - Train: 35.29% [671200/1902000] [352.9/1000.0] [batch_t 1.456 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 06:33:25,023 - Train: 35.29% [671300/1902000] [352.9/1000.0] [batch_t 1.462 (1.455)] [data_t 0.003] [optim_t 1.459] [lr 0.005000] +2024-04-08 06:35:50,579 - Train: 35.30% [671400/1902000] [353.0/1000.0] [batch_t 1.457 (1.455)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 06:35:59,290 - ==> Total time: 6 days, 5:35:38 Eta: 11 days, 10:11:06 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-08 06:38:18,387 - Train: 35.30% [671500/1902000] [353.0/1000.0] [batch_t 1.457 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 06:40:43,855 - Train: 35.31% [671600/1902000] [353.1/1000.0] [batch_t 1.454 (1.455)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 06:43:09,318 - Train: 35.32% [671700/1902000] [353.2/1000.0] [batch_t 1.456 (1.455)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 06:45:34,802 - Train: 35.32% [671800/1902000] [353.2/1000.0] [batch_t 1.449 (1.455)] [data_t 0.003] [optim_t 1.446] [lr 0.005000] +2024-04-08 06:48:00,330 - Train: 35.33% [671900/1902000] [353.3/1000.0] [batch_t 1.458 (1.455)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 06:50:25,894 - Train: 35.33% [672000/1902000] [353.3/1000.0] [batch_t 1.461 (1.456)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 06:52:51,463 - Train: 35.34% [672100/1902000] [353.4/1000.0] [batch_t 1.451 (1.456)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-08 06:55:17,037 - Train: 35.34% [672200/1902000] [353.4/1000.0] [batch_t 1.461 (1.456)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 06:57:42,563 - Train: 35.35% [672300/1902000] [353.5/1000.0] [batch_t 1.453 (1.455)] [data_t 0.003] [optim_t 1.450] [lr 0.005000] +2024-04-08 07:00:08,156 - Train: 35.35% [672400/1902000] [353.5/1000.0] [batch_t 1.446 (1.456)] [data_t 0.003] [optim_t 1.443] [lr 0.005000] +2024-04-08 07:02:33,620 - Train: 35.36% [672500/1902000] [353.6/1000.0] [batch_t 1.454 (1.455)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 07:04:59,096 - Train: 35.36% [672600/1902000] [353.6/1000.0] [batch_t 1.455 (1.455)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 07:07:24,592 - Train: 35.37% [672700/1902000] [353.7/1000.0] [batch_t 1.461 (1.455)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 07:09:50,127 - Train: 35.37% [672800/1902000] [353.7/1000.0] [batch_t 1.454 (1.455)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 07:12:15,668 - Train: 35.38% [672900/1902000] [353.8/1000.0] [batch_t 1.452 (1.455)] [data_t 0.003] [optim_t 1.449] [lr 0.005000] +2024-04-08 07:14:41,283 - Train: 35.38% [673000/1902000] [353.8/1000.0] [batch_t 1.451 (1.456)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-08 07:17:06,847 - Train: 35.39% [673100/1902000] [353.9/1000.0] [batch_t 1.454 (1.456)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 07:19:32,486 - Train: 35.39% [673200/1902000] [353.9/1000.0] [batch_t 1.455 (1.456)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 07:21:58,022 - Train: 35.40% [673300/1902000] [354.0/1000.0] [batch_t 1.453 (1.455)] [data_t 0.003] [optim_t 1.450] [lr 0.005000] +2024-04-08 07:22:09,676 - ==> Total time: 6 days, 6:21:49 Eta: 11 days, 10:23:32 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-08 07:24:25,539 - Train: 35.40% [673400/1902000] [354.0/1000.0] [batch_t 1.462 (1.455)] [data_t 0.003] [optim_t 1.459] [lr 0.005000] +2024-04-08 07:26:51,116 - Train: 35.41% [673500/1902000] [354.1/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 07:29:16,702 - Train: 35.42% [673600/1902000] [354.2/1000.0] [batch_t 1.450 (1.456)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-08 07:31:42,086 - Train: 35.42% [673700/1902000] [354.2/1000.0] [batch_t 1.458 (1.454)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 07:34:07,550 - Train: 35.43% [673800/1902000] [354.3/1000.0] [batch_t 1.455 (1.455)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 07:36:33,082 - Train: 35.43% [673900/1902000] [354.3/1000.0] [batch_t 1.455 (1.455)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 07:38:58,588 - Train: 35.44% [674000/1902000] [354.4/1000.0] [batch_t 1.460 (1.455)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 07:41:24,175 - Train: 35.44% [674100/1902000] [354.4/1000.0] [batch_t 1.448 (1.456)] [data_t 0.003] [optim_t 1.445] [lr 0.005000] +2024-04-08 07:43:49,759 - Train: 35.45% [674200/1902000] [354.5/1000.0] [batch_t 1.457 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 07:46:15,315 - Train: 35.45% [674300/1902000] [354.5/1000.0] [batch_t 1.466 (1.455)] [data_t 0.003] [optim_t 1.462] [lr 0.005000] +2024-04-08 07:48:40,852 - Train: 35.46% [674400/1902000] [354.6/1000.0] [batch_t 1.449 (1.455)] [data_t 0.003] [optim_t 1.446] [lr 0.005000] +2024-04-08 07:51:06,418 - Train: 35.46% [674500/1902000] [354.6/1000.0] [batch_t 1.455 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 07:53:31,911 - Train: 35.47% [674600/1902000] [354.7/1000.0] [batch_t 1.457 (1.455)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 07:55:57,460 - Train: 35.47% [674700/1902000] [354.7/1000.0] [batch_t 1.454 (1.455)] [data_t 0.003] [optim_t 1.450] [lr 0.005000] +2024-04-08 07:58:22,966 - Train: 35.48% [674800/1902000] [354.8/1000.0] [batch_t 1.453 (1.455)] [data_t 0.003] [optim_t 1.449] [lr 0.005000] +2024-04-08 08:00:48,495 - Train: 35.48% [674900/1902000] [354.8/1000.0] [batch_t 1.459 (1.455)] [data_t 0.003] [optim_t 1.456] [lr 0.005000] +2024-04-08 08:03:14,038 - Train: 35.49% [675000/1902000] [354.9/1000.0] [batch_t 1.451 (1.455)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-08 08:05:39,525 - Train: 35.49% [675100/1902000] [354.9/1000.0] [batch_t 1.457 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 08:08:05,084 - Train: 35.50% [675200/1902000] [355.0/1000.0] [batch_t 1.457 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 08:08:19,627 - ==> Total time: 6 days, 7:07:59 Eta: 11 days, 10:35:38 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-08 08:10:32,797 - Train: 35.50% [675300/1902000] [355.0/1000.0] [batch_t 1.455 (1.454)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 08:12:58,268 - Train: 35.51% [675400/1902000] [355.1/1000.0] [batch_t 1.454 (1.455)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 08:15:23,724 - Train: 35.52% [675500/1902000] [355.2/1000.0] [batch_t 1.456 (1.454)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 08:17:49,244 - Train: 35.52% [675600/1902000] [355.2/1000.0] [batch_t 1.454 (1.455)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 08:20:14,832 - Train: 35.53% [675700/1902000] [355.3/1000.0] [batch_t 1.446 (1.456)] [data_t 0.003] [optim_t 1.443] [lr 0.005000] +2024-04-08 08:22:40,295 - Train: 35.53% [675800/1902000] [355.3/1000.0] [batch_t 1.451 (1.455)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-08 08:25:05,815 - Train: 35.54% [675900/1902000] [355.4/1000.0] [batch_t 1.460 (1.455)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 08:27:31,222 - Train: 35.54% [676000/1902000] [355.4/1000.0] [batch_t 1.457 (1.454)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 08:29:56,798 - Train: 35.55% [676100/1902000] [355.5/1000.0] [batch_t 1.458 (1.456)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 08:32:22,319 - Train: 35.55% [676200/1902000] [355.5/1000.0] [batch_t 1.450 (1.455)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-08 08:34:47,872 - Train: 35.56% [676300/1902000] [355.6/1000.0] [batch_t 1.455 (1.455)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 08:37:13,365 - Train: 35.56% [676400/1902000] [355.6/1000.0] [batch_t 1.461 (1.455)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 08:39:38,765 - Train: 35.57% [676500/1902000] [355.7/1000.0] [batch_t 1.461 (1.454)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 08:42:04,333 - Train: 35.57% [676600/1902000] [355.7/1000.0] [batch_t 1.452 (1.456)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-08 08:44:29,888 - Train: 35.58% [676700/1902000] [355.8/1000.0] [batch_t 1.456 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 08:46:55,414 - Train: 35.58% [676800/1902000] [355.8/1000.0] [batch_t 1.456 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 08:49:20,898 - Train: 35.59% [676900/1902000] [355.9/1000.0] [batch_t 1.458 (1.455)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 08:51:46,376 - Train: 35.59% [677000/1902000] [355.9/1000.0] [batch_t 1.465 (1.455)] [data_t 0.003] [optim_t 1.462] [lr 0.005000] +2024-04-08 08:54:11,859 - Train: 35.60% [677100/1902000] [356.0/1000.0] [batch_t 1.450 (1.455)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-08 08:54:29,310 - ==> Total time: 6 days, 7:54:08 Eta: 11 days, 10:47:23 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-08 08:56:39,404 - Train: 35.60% [677200/1902000] [356.0/1000.0] [batch_t 1.457 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 08:59:05,043 - Train: 35.61% [677300/1902000] [356.1/1000.0] [batch_t 1.458 (1.456)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 09:01:30,637 - Train: 35.62% [677400/1902000] [356.2/1000.0] [batch_t 1.462 (1.456)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 09:03:56,161 - Train: 35.62% [677500/1902000] [356.2/1000.0] [batch_t 1.453 (1.455)] [data_t 0.003] [optim_t 1.450] [lr 0.005000] +2024-04-08 09:06:21,733 - Train: 35.63% [677600/1902000] [356.3/1000.0] [batch_t 1.463 (1.456)] [data_t 0.003] [optim_t 1.459] [lr 0.005000] +2024-04-08 09:08:47,302 - Train: 35.63% [677700/1902000] [356.3/1000.0] [batch_t 1.457 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 09:11:12,766 - Train: 35.64% [677800/1902000] [356.4/1000.0] [batch_t 1.450 (1.455)] [data_t 0.004] [optim_t 1.447] [lr 0.005000] +2024-04-08 09:13:38,234 - Train: 35.64% [677900/1902000] [356.4/1000.0] [batch_t 1.462 (1.455)] [data_t 0.003] [optim_t 1.459] [lr 0.005000] +2024-04-08 09:16:03,819 - Train: 35.65% [678000/1902000] [356.5/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 09:18:29,417 - Train: 35.65% [678100/1902000] [356.5/1000.0] [batch_t 1.452 (1.456)] [data_t 0.003] [optim_t 1.449] [lr 0.005000] +2024-04-08 09:20:55,066 - Train: 35.66% [678200/1902000] [356.6/1000.0] [batch_t 1.459 (1.456)] [data_t 0.003] [optim_t 1.456] [lr 0.005000] +2024-04-08 09:23:20,652 - Train: 35.66% [678300/1902000] [356.6/1000.0] [batch_t 1.458 (1.456)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 09:25:46,176 - Train: 35.67% [678400/1902000] [356.7/1000.0] [batch_t 1.455 (1.455)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 09:28:11,690 - Train: 35.67% [678500/1902000] [356.7/1000.0] [batch_t 1.459 (1.455)] [data_t 0.003] [optim_t 1.456] [lr 0.005000] +2024-04-08 09:30:37,200 - Train: 35.68% [678600/1902000] [356.8/1000.0] [batch_t 1.456 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 09:33:02,784 - Train: 35.68% [678700/1902000] [356.8/1000.0] [batch_t 1.457 (1.456)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 09:35:28,281 - Train: 35.69% [678800/1902000] [356.9/1000.0] [batch_t 1.456 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 09:37:53,752 - Train: 35.69% [678900/1902000] [356.9/1000.0] [batch_t 1.450 (1.455)] [data_t 0.004] [optim_t 1.447] [lr 0.005000] +2024-04-08 09:40:19,287 - Train: 35.70% [679000/1902000] [357.0/1000.0] [batch_t 1.450 (1.455)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-08 09:40:39,704 - ==> Total time: 6 days, 8:40:19 Eta: 11 days, 10:58:50 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-08 09:42:47,344 - Train: 35.70% [679100/1902000] [357.0/1000.0] [batch_t 1.453 (1.455)] [data_t 0.003] [optim_t 1.450] [lr 0.005000] +2024-04-08 09:45:12,894 - Train: 35.71% [679200/1902000] [357.1/1000.0] [batch_t 1.461 (1.455)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 09:47:38,351 - Train: 35.72% [679300/1902000] [357.2/1000.0] [batch_t 1.453 (1.454)] [data_t 0.003] [optim_t 1.450] [lr 0.005000] +2024-04-08 09:50:03,808 - Train: 35.72% [679400/1902000] [357.2/1000.0] [batch_t 1.461 (1.454)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 09:52:29,304 - Train: 35.73% [679500/1902000] [357.3/1000.0] [batch_t 1.452 (1.455)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-08 09:54:54,942 - Train: 35.73% [679600/1902000] [357.3/1000.0] [batch_t 1.455 (1.456)] [data_t 0.004] [optim_t 1.452] [lr 0.005000] +2024-04-08 09:57:20,454 - Train: 35.74% [679700/1902000] [357.4/1000.0] [batch_t 1.452 (1.455)] [data_t 0.003] [optim_t 1.449] [lr 0.005000] +2024-04-08 09:59:46,006 - Train: 35.74% [679800/1902000] [357.4/1000.0] [batch_t 1.448 (1.455)] [data_t 0.003] [optim_t 1.445] [lr 0.005000] +2024-04-08 10:02:11,552 - Train: 35.75% [679900/1902000] [357.5/1000.0] [batch_t 1.457 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 10:04:37,096 - Train: 35.75% [680000/1902000] [357.5/1000.0] [batch_t 1.461 (1.455)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 10:07:02,550 - Train: 35.76% [680100/1902000] [357.6/1000.0] [batch_t 1.456 (1.454)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 10:09:28,109 - Train: 35.76% [680200/1902000] [357.6/1000.0] [batch_t 1.457 (1.455)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 10:11:53,649 - Train: 35.77% [680300/1902000] [357.7/1000.0] [batch_t 1.462 (1.455)] [data_t 0.003] [optim_t 1.459] [lr 0.005000] +2024-04-08 10:14:19,163 - Train: 35.77% [680400/1902000] [357.7/1000.0] [batch_t 1.451 (1.455)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-08 10:16:44,703 - Train: 35.78% [680500/1902000] [357.8/1000.0] [batch_t 1.457 (1.455)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 10:19:10,209 - Train: 35.78% [680600/1902000] [357.8/1000.0] [batch_t 1.462 (1.455)] [data_t 0.003] [optim_t 1.459] [lr 0.005000] +2024-04-08 10:21:35,783 - Train: 35.79% [680700/1902000] [357.9/1000.0] [batch_t 1.450 (1.456)] [data_t 0.003] [optim_t 1.446] [lr 0.005000] +2024-04-08 10:24:01,329 - Train: 35.79% [680800/1902000] [357.9/1000.0] [batch_t 1.457 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 10:26:26,890 - Train: 35.80% [680900/1902000] [358.0/1000.0] [batch_t 1.457 (1.456)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 10:26:50,200 - ==> Total time: 6 days, 9:26:29 Eta: 11 days, 11:09:58 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-08 10:28:54,675 - Train: 35.80% [681000/1902000] [358.0/1000.0] [batch_t 1.455 (1.456)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 10:31:20,189 - Train: 35.81% [681100/1902000] [358.1/1000.0] [batch_t 1.455 (1.455)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 10:33:45,712 - Train: 35.81% [681200/1902000] [358.1/1000.0] [batch_t 1.461 (1.455)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 10:36:11,266 - Train: 35.82% [681300/1902000] [358.2/1000.0] [batch_t 1.457 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 10:38:36,834 - Train: 35.83% [681400/1902000] [358.3/1000.0] [batch_t 1.451 (1.456)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-08 10:41:02,449 - Train: 35.83% [681500/1902000] [358.3/1000.0] [batch_t 1.465 (1.456)] [data_t 0.003] [optim_t 1.462] [lr 0.005000] +2024-04-08 10:43:28,026 - Train: 35.84% [681600/1902000] [358.4/1000.0] [batch_t 1.454 (1.456)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 10:45:53,586 - Train: 35.84% [681700/1902000] [358.4/1000.0] [batch_t 1.455 (1.456)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 10:48:19,114 - Train: 35.85% [681800/1902000] [358.5/1000.0] [batch_t 1.456 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 10:50:44,643 - Train: 35.85% [681900/1902000] [358.5/1000.0] [batch_t 1.461 (1.455)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 10:53:10,245 - Train: 35.86% [682000/1902000] [358.6/1000.0] [batch_t 1.458 (1.456)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 10:55:35,790 - Train: 35.86% [682100/1902000] [358.6/1000.0] [batch_t 1.461 (1.455)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 10:58:01,368 - Train: 35.87% [682200/1902000] [358.7/1000.0] [batch_t 1.455 (1.456)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 11:00:27,006 - Train: 35.87% [682300/1902000] [358.7/1000.0] [batch_t 1.457 (1.456)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 11:02:52,550 - Train: 35.88% [682400/1902000] [358.8/1000.0] [batch_t 1.460 (1.455)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 11:05:18,051 - Train: 35.88% [682500/1902000] [358.8/1000.0] [batch_t 1.456 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 11:07:43,556 - Train: 35.89% [682600/1902000] [358.9/1000.0] [batch_t 1.454 (1.455)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 11:10:09,087 - Train: 35.89% [682700/1902000] [358.9/1000.0] [batch_t 1.461 (1.455)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 11:12:34,634 - Train: 35.90% [682800/1902000] [359.0/1000.0] [batch_t 1.455 (1.455)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 11:13:00,851 - ==> Total time: 6 days, 10:12:40 Eta: 11 days, 11:20:47 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-08 11:15:05,708 - Train: 35.90% [682900/1902000] [359.0/1000.0] [batch_t 1.453 (1.494)] [data_t 0.003] [optim_t 1.449] [lr 0.005000] +2024-04-08 11:17:31,246 - Train: 35.91% [683000/1902000] [359.1/1000.0] [batch_t 1.460 (1.455)] [data_t 0.003] [optim_t 1.456] [lr 0.005000] +2024-04-08 11:19:56,794 - Train: 35.91% [683100/1902000] [359.1/1000.0] [batch_t 1.458 (1.455)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 11:22:22,444 - Train: 35.92% [683200/1902000] [359.2/1000.0] [batch_t 1.457 (1.456)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 11:24:48,008 - Train: 35.93% [683300/1902000] [359.3/1000.0] [batch_t 1.459 (1.456)] [data_t 0.003] [optim_t 1.456] [lr 0.005000] +2024-04-08 11:27:13,589 - Train: 35.93% [683400/1902000] [359.3/1000.0] [batch_t 1.455 (1.456)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 11:29:39,128 - Train: 35.94% [683500/1902000] [359.4/1000.0] [batch_t 1.452 (1.455)] [data_t 0.003] [optim_t 1.449] [lr 0.005000] +2024-04-08 11:32:04,643 - Train: 35.94% [683600/1902000] [359.4/1000.0] [batch_t 1.455 (1.455)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 11:34:30,105 - Train: 35.95% [683700/1902000] [359.5/1000.0] [batch_t 1.455 (1.455)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 11:36:55,601 - Train: 35.95% [683800/1902000] [359.5/1000.0] [batch_t 1.457 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 11:39:21,118 - Train: 35.96% [683900/1902000] [359.6/1000.0] [batch_t 1.455 (1.455)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 11:41:46,592 - Train: 35.96% [684000/1902000] [359.6/1000.0] [batch_t 1.456 (1.455)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 11:44:12,179 - Train: 35.97% [684100/1902000] [359.7/1000.0] [batch_t 1.454 (1.456)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 11:46:37,730 - Train: 35.97% [684200/1902000] [359.7/1000.0] [batch_t 1.445 (1.455)] [data_t 0.003] [optim_t 1.442] [lr 0.005000] +2024-04-08 11:49:03,293 - Train: 35.98% [684300/1902000] [359.8/1000.0] [batch_t 1.455 (1.456)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 11:51:28,725 - Train: 35.98% [684400/1902000] [359.8/1000.0] [batch_t 1.448 (1.454)] [data_t 0.003] [optim_t 1.445] [lr 0.005000] +2024-04-08 11:53:54,177 - Train: 35.99% [684500/1902000] [359.9/1000.0] [batch_t 1.455 (1.454)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 11:56:19,680 - Train: 35.99% [684600/1902000] [359.9/1000.0] [batch_t 1.451 (1.455)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-08 11:58:45,235 - Train: 36.00% [684700/1902000] [360.0/1000.0] [batch_t 1.455 (1.455)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 11:59:14,331 - ==> Total time: 6 days, 10:58:53 Eta: 11 days, 11:31:22 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-08 12:01:13,873 - Train: 36.00% [684800/1902000] [360.0/1000.0] [batch_t 1.456 (1.465)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 12:03:39,383 - Train: 36.01% [684900/1902000] [360.1/1000.0] [batch_t 1.460 (1.455)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 12:06:04,984 - Train: 36.01% [685000/1902000] [360.1/1000.0] [batch_t 1.454 (1.456)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 12:08:30,589 - Train: 36.02% [685100/1902000] [360.2/1000.0] [batch_t 1.459 (1.456)] [data_t 0.003] [optim_t 1.456] [lr 0.005000] +2024-04-08 12:10:56,140 - Train: 36.03% [685200/1902000] [360.3/1000.0] [batch_t 1.447 (1.455)] [data_t 0.003] [optim_t 1.444] [lr 0.005000] +2024-04-08 12:13:21,716 - Train: 36.03% [685300/1902000] [360.3/1000.0] [batch_t 1.457 (1.456)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 12:15:47,264 - Train: 36.04% [685400/1902000] [360.4/1000.0] [batch_t 1.460 (1.455)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 12:18:12,816 - Train: 36.04% [685500/1902000] [360.4/1000.0] [batch_t 1.455 (1.455)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 12:20:38,382 - Train: 36.05% [685600/1902000] [360.5/1000.0] [batch_t 1.451 (1.456)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-08 12:23:03,959 - Train: 36.05% [685700/1902000] [360.5/1000.0] [batch_t 1.460 (1.456)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 12:25:29,461 - Train: 36.06% [685800/1902000] [360.6/1000.0] [batch_t 1.451 (1.455)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-08 12:27:54,961 - Train: 36.06% [685900/1902000] [360.6/1000.0] [batch_t 1.452 (1.455)] [data_t 0.003] [optim_t 1.449] [lr 0.005000] +2024-04-08 12:30:20,491 - Train: 36.07% [686000/1902000] [360.7/1000.0] [batch_t 1.458 (1.455)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 12:32:46,093 - Train: 36.07% [686100/1902000] [360.7/1000.0] [batch_t 1.460 (1.456)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 12:35:11,649 - Train: 36.08% [686200/1902000] [360.8/1000.0] [batch_t 1.452 (1.455)] [data_t 0.003] [optim_t 1.449] [lr 0.005000] +2024-04-08 12:37:37,207 - Train: 36.08% [686300/1902000] [360.8/1000.0] [batch_t 1.461 (1.455)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 12:40:02,774 - Train: 36.09% [686400/1902000] [360.9/1000.0] [batch_t 1.450 (1.456)] [data_t 0.003] [optim_t 1.446] [lr 0.005000] +2024-04-08 12:42:28,287 - Train: 36.09% [686500/1902000] [360.9/1000.0] [batch_t 1.457 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 12:44:53,839 - Train: 36.10% [686600/1902000] [361.0/1000.0] [batch_t 1.456 (1.455)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 12:45:25,880 - ==> Total time: 6 days, 11:45:05 Eta: 11 days, 11:41:35 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-08 12:47:23,332 - Train: 36.10% [686700/1902000] [361.0/1000.0] [batch_t 1.458 (1.465)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 12:49:48,837 - Train: 36.11% [686800/1902000] [361.1/1000.0] [batch_t 1.455 (1.455)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 12:52:14,362 - Train: 36.11% [686900/1902000] [361.1/1000.0] [batch_t 1.459 (1.455)] [data_t 0.003] [optim_t 1.456] [lr 0.005000] +2024-04-08 12:54:39,960 - Train: 36.12% [687000/1902000] [361.2/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 12:57:05,543 - Train: 36.13% [687100/1902000] [361.3/1000.0] [batch_t 1.451 (1.456)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-08 12:59:31,117 - Train: 36.13% [687200/1902000] [361.3/1000.0] [batch_t 1.459 (1.456)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 13:01:56,610 - Train: 36.14% [687300/1902000] [361.4/1000.0] [batch_t 1.447 (1.455)] [data_t 0.003] [optim_t 1.444] [lr 0.005000] +2024-04-08 13:04:22,127 - Train: 36.14% [687400/1902000] [361.4/1000.0] [batch_t 1.446 (1.455)] [data_t 0.003] [optim_t 1.443] [lr 0.005000] +2024-04-08 13:06:47,626 - Train: 36.15% [687500/1902000] [361.5/1000.0] [batch_t 1.456 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 13:09:13,137 - Train: 36.15% [687600/1902000] [361.5/1000.0] [batch_t 1.455 (1.455)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 13:11:38,733 - Train: 36.16% [687700/1902000] [361.6/1000.0] [batch_t 1.458 (1.456)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 13:14:04,318 - Train: 36.16% [687800/1902000] [361.6/1000.0] [batch_t 1.462 (1.456)] [data_t 0.003] [optim_t 1.459] [lr 0.005000] +2024-04-08 13:16:29,886 - Train: 36.17% [687900/1902000] [361.7/1000.0] [batch_t 1.450 (1.456)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-08 13:18:55,392 - Train: 36.17% [688000/1902000] [361.7/1000.0] [batch_t 1.456 (1.455)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 13:21:21,007 - Train: 36.18% [688100/1902000] [361.8/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 13:23:46,543 - Train: 36.18% [688200/1902000] [361.8/1000.0] [batch_t 1.455 (1.455)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 13:26:12,021 - Train: 36.19% [688300/1902000] [361.9/1000.0] [batch_t 1.448 (1.455)] [data_t 0.003] [optim_t 1.445] [lr 0.005000] +2024-04-08 13:28:37,462 - Train: 36.19% [688400/1902000] [361.9/1000.0] [batch_t 1.455 (1.454)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 13:31:03,038 - Train: 36.20% [688500/1902000] [362.0/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 13:31:37,918 - ==> Total time: 6 days, 12:31:17 Eta: 11 days, 11:51:30 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-08 13:33:32,701 - Train: 36.20% [688600/1902000] [362.0/1000.0] [batch_t 1.451 (1.466)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-08 13:35:58,332 - Train: 36.21% [688700/1902000] [362.1/1000.0] [batch_t 1.461 (1.456)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 13:38:23,888 - Train: 36.21% [688800/1902000] [362.1/1000.0] [batch_t 1.456 (1.455)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 13:40:49,488 - Train: 36.22% [688900/1902000] [362.2/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 13:43:15,032 - Train: 36.23% [689000/1902000] [362.3/1000.0] [batch_t 1.454 (1.455)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 13:45:40,543 - Train: 36.23% [689100/1902000] [362.3/1000.0] [batch_t 1.452 (1.455)] [data_t 0.003] [optim_t 1.449] [lr 0.005000] +2024-04-08 13:48:06,100 - Train: 36.24% [689200/1902000] [362.4/1000.0] [batch_t 1.456 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 13:50:31,604 - Train: 36.24% [689300/1902000] [362.4/1000.0] [batch_t 1.461 (1.455)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 13:52:57,070 - Train: 36.25% [689400/1902000] [362.5/1000.0] [batch_t 1.454 (1.455)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 13:55:22,619 - Train: 36.25% [689500/1902000] [362.5/1000.0] [batch_t 1.451 (1.455)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-08 13:57:48,143 - Train: 36.26% [689600/1902000] [362.6/1000.0] [batch_t 1.454 (1.455)] [data_t 0.003] [optim_t 1.450] [lr 0.005000] +2024-04-08 14:00:13,624 - Train: 36.26% [689700/1902000] [362.6/1000.0] [batch_t 1.453 (1.455)] [data_t 0.003] [optim_t 1.449] [lr 0.005000] +2024-04-08 14:02:39,133 - Train: 36.27% [689800/1902000] [362.7/1000.0] [batch_t 1.452 (1.455)] [data_t 0.003] [optim_t 1.449] [lr 0.005000] +2024-04-08 14:05:04,602 - Train: 36.27% [689900/1902000] [362.7/1000.0] [batch_t 1.461 (1.455)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 14:07:30,215 - Train: 36.28% [690000/1902000] [362.8/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 14:09:55,682 - Train: 36.28% [690100/1902000] [362.8/1000.0] [batch_t 1.450 (1.455)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-08 14:12:21,191 - Train: 36.29% [690200/1902000] [362.9/1000.0] [batch_t 1.456 (1.455)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 14:14:46,676 - Train: 36.29% [690300/1902000] [362.9/1000.0] [batch_t 1.454 (1.455)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 14:17:12,228 - Train: 36.30% [690400/1902000] [363.0/1000.0] [batch_t 1.456 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 14:17:50,037 - ==> Total time: 6 days, 13:17:29 Eta: 11 days, 12:01:06 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-08 14:19:40,261 - Train: 36.30% [690500/1902000] [363.0/1000.0] [batch_t 1.454 (1.460)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 14:22:05,734 - Train: 36.31% [690600/1902000] [363.1/1000.0] [batch_t 1.455 (1.455)] [data_t 0.004] [optim_t 1.451] [lr 0.005000] +2024-04-08 14:24:31,120 - Train: 36.31% [690700/1902000] [363.1/1000.0] [batch_t 1.460 (1.454)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 14:26:56,659 - Train: 36.32% [690800/1902000] [363.2/1000.0] [batch_t 1.458 (1.455)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 14:29:22,155 - Train: 36.32% [690900/1902000] [363.2/1000.0] [batch_t 1.456 (1.455)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 14:31:47,717 - Train: 36.33% [691000/1902000] [363.3/1000.0] [batch_t 1.455 (1.456)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 14:34:13,284 - Train: 36.34% [691100/1902000] [363.4/1000.0] [batch_t 1.459 (1.456)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 14:36:38,714 - Train: 36.34% [691200/1902000] [363.4/1000.0] [batch_t 1.463 (1.454)] [data_t 0.003] [optim_t 1.460] [lr 0.005000] +2024-04-08 14:39:04,245 - Train: 36.35% [691300/1902000] [363.5/1000.0] [batch_t 1.456 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 14:41:29,714 - Train: 36.35% [691400/1902000] [363.5/1000.0] [batch_t 1.460 (1.455)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 14:43:55,230 - Train: 36.36% [691500/1902000] [363.6/1000.0] [batch_t 1.454 (1.455)] [data_t 0.003] [optim_t 1.450] [lr 0.005000] +2024-04-08 14:46:20,669 - Train: 36.36% [691600/1902000] [363.6/1000.0] [batch_t 1.456 (1.454)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 14:48:46,165 - Train: 36.37% [691700/1902000] [363.7/1000.0] [batch_t 1.454 (1.455)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 14:51:11,709 - Train: 36.37% [691800/1902000] [363.7/1000.0] [batch_t 1.461 (1.455)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 14:53:37,157 - Train: 36.38% [691900/1902000] [363.8/1000.0] [batch_t 1.451 (1.454)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-08 14:56:02,687 - Train: 36.38% [692000/1902000] [363.8/1000.0] [batch_t 1.453 (1.455)] [data_t 0.003] [optim_t 1.450] [lr 0.005000] +2024-04-08 14:58:28,311 - Train: 36.39% [692100/1902000] [363.9/1000.0] [batch_t 1.451 (1.456)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-08 15:00:54,026 - Train: 36.39% [692200/1902000] [363.9/1000.0] [batch_t 1.455 (1.457)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 15:03:19,582 - Train: 36.40% [692300/1902000] [364.0/1000.0] [batch_t 1.463 (1.455)] [data_t 0.003] [optim_t 1.459] [lr 0.005000] +2024-04-08 15:04:00,366 - ==> Total time: 6 days, 14:03:39 Eta: 11 days, 12:10:21 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-08 15:05:50,315 - Train: 36.40% [692400/1902000] [364.0/1000.0] [batch_t 1.462 (1.460)] [data_t 0.003] [optim_t 1.459] [lr 0.005000] +2024-04-08 15:08:15,973 - Train: 36.41% [692500/1902000] [364.1/1000.0] [batch_t 1.451 (1.456)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-08 15:10:41,540 - Train: 36.41% [692600/1902000] [364.1/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 15:13:07,198 - Train: 36.42% [692700/1902000] [364.2/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 15:15:32,766 - Train: 36.42% [692800/1902000] [364.2/1000.0] [batch_t 1.444 (1.456)] [data_t 0.003] [optim_t 1.441] [lr 0.005000] +2024-04-08 15:17:58,312 - Train: 36.43% [692900/1902000] [364.3/1000.0] [batch_t 1.457 (1.455)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 15:20:23,897 - Train: 36.44% [693000/1902000] [364.4/1000.0] [batch_t 1.452 (1.456)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-08 15:22:49,473 - Train: 36.44% [693100/1902000] [364.4/1000.0] [batch_t 1.454 (1.456)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 15:25:15,176 - Train: 36.45% [693200/1902000] [364.5/1000.0] [batch_t 1.466 (1.457)] [data_t 0.003] [optim_t 1.462] [lr 0.005000] +2024-04-08 15:27:49,109 - Train: 36.45% [693300/1902000] [364.5/1000.0] [batch_t 1.461 (1.539)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 15:30:27,766 - Train: 36.46% [693400/1902000] [364.6/1000.0] [batch_t 2.872 (1.586)] [data_t 1.418] [optim_t 1.454] [lr 0.005000] +2024-04-08 15:33:45,352 - Train: 36.46% [693500/1902000] [364.6/1000.0] [batch_t 1.460 (1.976)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 15:36:33,439 - Train: 36.47% [693600/1902000] [364.7/1000.0] [batch_t 1.456 (1.681)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 15:40:08,872 - Train: 36.47% [693700/1902000] [364.7/1000.0] [batch_t 1.438 (2.154)] [data_t 0.003] [optim_t 1.435] [lr 0.005000] +2024-04-08 15:42:34,521 - Train: 36.48% [693800/1902000] [364.8/1000.0] [batch_t 1.467 (1.456)] [data_t 0.003] [optim_t 1.464] [lr 0.005000] +2024-04-08 15:45:05,215 - Train: 36.48% [693900/1902000] [364.8/1000.0] [batch_t 1.461 (1.507)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 15:47:30,855 - Train: 36.49% [694000/1902000] [364.9/1000.0] [batch_t 1.458 (1.456)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 15:49:56,504 - Train: 36.49% [694100/1902000] [364.9/1000.0] [batch_t 1.462 (1.456)] [data_t 0.003] [optim_t 1.459] [lr 0.005000] +2024-04-08 15:52:22,029 - Train: 36.50% [694200/1902000] [365.0/1000.0] [batch_t 1.450 (1.455)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-08 15:53:05,722 - ==> Total time: 6 days, 14:52:45 Eta: 11 days, 12:24:22 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-08 15:54:49,894 - Train: 36.50% [694300/1902000] [365.0/1000.0] [batch_t 1.448 (1.456)] [data_t 0.003] [optim_t 1.445] [lr 0.005000] +2024-04-08 15:57:15,531 - Train: 36.51% [694400/1902000] [365.1/1000.0] [batch_t 1.460 (1.456)] [data_t 0.003] [optim_t 1.456] [lr 0.005000] +2024-04-08 15:59:41,159 - Train: 36.51% [694500/1902000] [365.1/1000.0] [batch_t 1.458 (1.456)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 16:02:06,811 - Train: 36.52% [694600/1902000] [365.2/1000.0] [batch_t 1.450 (1.456)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-08 16:04:32,524 - Train: 36.52% [694700/1902000] [365.2/1000.0] [batch_t 1.458 (1.457)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 16:06:58,138 - Train: 36.53% [694800/1902000] [365.3/1000.0] [batch_t 1.463 (1.456)] [data_t 0.003] [optim_t 1.460] [lr 0.005000] +2024-04-08 16:09:23,777 - Train: 36.54% [694900/1902000] [365.4/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 16:11:49,354 - Train: 36.54% [695000/1902000] [365.4/1000.0] [batch_t 1.466 (1.456)] [data_t 0.003] [optim_t 1.463] [lr 0.005000] +2024-04-08 16:14:15,025 - Train: 36.55% [695100/1902000] [365.5/1000.0] [batch_t 1.454 (1.457)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 16:16:40,616 - Train: 36.55% [695200/1902000] [365.5/1000.0] [batch_t 1.452 (1.456)] [data_t 0.003] [optim_t 1.449] [lr 0.005000] +2024-04-08 16:19:06,296 - Train: 36.56% [695300/1902000] [365.6/1000.0] [batch_t 1.461 (1.457)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 16:21:31,961 - Train: 36.56% [695400/1902000] [365.6/1000.0] [batch_t 1.451 (1.457)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-08 16:23:57,568 - Train: 36.57% [695500/1902000] [365.7/1000.0] [batch_t 1.458 (1.456)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 16:26:23,191 - Train: 36.57% [695600/1902000] [365.7/1000.0] [batch_t 1.461 (1.456)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 16:28:48,800 - Train: 36.58% [695700/1902000] [365.8/1000.0] [batch_t 1.452 (1.456)] [data_t 0.003] [optim_t 1.449] [lr 0.005000] +2024-04-08 16:31:14,384 - Train: 36.58% [695800/1902000] [365.8/1000.0] [batch_t 1.464 (1.456)] [data_t 0.004] [optim_t 1.460] [lr 0.005000] +2024-04-08 16:33:40,016 - Train: 36.59% [695900/1902000] [365.9/1000.0] [batch_t 1.457 (1.456)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 16:36:05,649 - Train: 36.59% [696000/1902000] [365.9/1000.0] [batch_t 1.455 (1.456)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 16:38:31,304 - Train: 36.60% [696100/1902000] [366.0/1000.0] [batch_t 1.461 (1.456)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 16:39:17,979 - ==> Total time: 6 days, 15:38:57 Eta: 11 days, 12:33:03 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-08 16:41:01,847 - Train: 36.60% [696200/1902000] [366.0/1000.0] [batch_t 1.459 (1.474)] [data_t 0.003] [optim_t 1.456] [lr 0.005000] +2024-04-08 16:43:27,479 - Train: 36.61% [696300/1902000] [366.1/1000.0] [batch_t 1.457 (1.456)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 16:45:53,091 - Train: 36.61% [696400/1902000] [366.1/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 16:48:18,691 - Train: 36.62% [696500/1902000] [366.2/1000.0] [batch_t 1.460 (1.456)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 16:50:44,290 - Train: 36.62% [696600/1902000] [366.2/1000.0] [batch_t 1.449 (1.456)] [data_t 0.003] [optim_t 1.446] [lr 0.005000] +2024-04-08 16:53:09,933 - Train: 36.63% [696700/1902000] [366.3/1000.0] [batch_t 1.451 (1.456)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-08 16:55:35,599 - Train: 36.64% [696800/1902000] [366.4/1000.0] [batch_t 1.461 (1.457)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 16:58:01,293 - Train: 36.64% [696900/1902000] [366.4/1000.0] [batch_t 1.456 (1.457)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 17:00:26,952 - Train: 36.65% [697000/1902000] [366.5/1000.0] [batch_t 1.458 (1.456)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 17:02:52,623 - Train: 36.65% [697100/1902000] [366.5/1000.0] [batch_t 1.460 (1.457)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 17:05:18,202 - Train: 36.66% [697200/1902000] [366.6/1000.0] [batch_t 1.451 (1.456)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-08 17:07:43,807 - Train: 36.66% [697300/1902000] [366.6/1000.0] [batch_t 1.455 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 17:10:09,437 - Train: 36.67% [697400/1902000] [366.7/1000.0] [batch_t 1.455 (1.456)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 17:12:35,097 - Train: 36.67% [697500/1902000] [366.7/1000.0] [batch_t 1.456 (1.457)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 17:15:00,854 - Train: 36.68% [697600/1902000] [366.8/1000.0] [batch_t 1.461 (1.457)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 17:17:26,563 - Train: 36.68% [697700/1902000] [366.8/1000.0] [batch_t 1.459 (1.457)] [data_t 0.004] [optim_t 1.456] [lr 0.005000] +2024-04-08 17:19:52,238 - Train: 36.69% [697800/1902000] [366.9/1000.0] [batch_t 1.461 (1.457)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 17:22:17,884 - Train: 36.69% [697900/1902000] [366.9/1000.0] [batch_t 1.444 (1.456)] [data_t 0.003] [optim_t 1.441] [lr 0.005000] +2024-04-08 17:24:43,586 - Train: 36.70% [698000/1902000] [367.0/1000.0] [batch_t 1.458 (1.457)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 17:25:33,246 - ==> Total time: 6 days, 16:25:12 Eta: 11 days, 12:41:31 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-08 17:27:11,572 - Train: 36.70% [698100/1902000] [367.0/1000.0] [batch_t 1.462 (1.458)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 17:29:37,285 - Train: 36.71% [698200/1902000] [367.1/1000.0] [batch_t 1.458 (1.457)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 17:32:02,992 - Train: 36.71% [698300/1902000] [367.1/1000.0] [batch_t 1.462 (1.457)] [data_t 0.003] [optim_t 1.459] [lr 0.005000] +2024-04-08 17:34:28,668 - Train: 36.72% [698400/1902000] [367.2/1000.0] [batch_t 1.457 (1.457)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 17:36:54,339 - Train: 36.72% [698500/1902000] [367.2/1000.0] [batch_t 1.455 (1.457)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 17:39:20,075 - Train: 36.73% [698600/1902000] [367.3/1000.0] [batch_t 1.457 (1.457)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 17:41:49,110 - Train: 36.74% [698700/1902000] [367.4/1000.0] [batch_t 1.457 (1.490)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 17:44:14,753 - Train: 36.74% [698800/1902000] [367.4/1000.0] [batch_t 1.451 (1.456)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-08 17:46:42,755 - Train: 36.75% [698900/1902000] [367.5/1000.0] [batch_t 1.457 (1.480)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 17:49:08,371 - Train: 36.75% [699000/1902000] [367.5/1000.0] [batch_t 1.457 (1.456)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 17:51:34,026 - Train: 36.76% [699100/1902000] [367.6/1000.0] [batch_t 1.454 (1.456)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 17:54:00,883 - Train: 36.76% [699200/1902000] [367.6/1000.0] [batch_t 1.461 (1.468)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 17:56:26,671 - Train: 36.77% [699300/1902000] [367.7/1000.0] [batch_t 1.452 (1.458)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-08 17:58:53,290 - Train: 36.77% [699400/1902000] [367.7/1000.0] [batch_t 1.457 (1.466)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 18:01:18,901 - Train: 36.78% [699500/1902000] [367.8/1000.0] [batch_t 1.460 (1.456)] [data_t 0.003] [optim_t 1.456] [lr 0.005000] +2024-04-08 18:03:45,691 - Train: 36.78% [699600/1902000] [367.8/1000.0] [batch_t 1.457 (1.468)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 18:06:11,413 - Train: 36.79% [699700/1902000] [367.9/1000.0] [batch_t 1.455 (1.457)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 18:08:37,186 - Train: 36.79% [699800/1902000] [367.9/1000.0] [batch_t 1.460 (1.458)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 18:11:02,871 - Train: 36.80% [699900/1902000] [368.0/1000.0] [batch_t 1.455 (1.457)] [data_t 0.004] [optim_t 1.451] [lr 0.005000] +2024-04-08 18:11:55,328 - ==> Total time: 6 days, 17:11:34 Eta: 11 days, 12:49:53 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-08 18:13:31,655 - Train: 36.80% [700000/1902000] [368.0/1000.0] [batch_t 1.461 (1.465)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 18:15:57,322 - Train: 36.81% [700100/1902000] [368.1/1000.0] [batch_t 1.463 (1.457)] [data_t 0.003] [optim_t 1.460] [lr 0.005000] +2024-04-08 18:18:22,963 - Train: 36.81% [700200/1902000] [368.1/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 18:20:48,599 - Train: 36.82% [700300/1902000] [368.2/1000.0] [batch_t 1.454 (1.456)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 18:23:14,237 - Train: 36.82% [700400/1902000] [368.2/1000.0] [batch_t 1.462 (1.456)] [data_t 0.003] [optim_t 1.459] [lr 0.005000] +2024-04-08 18:25:39,924 - Train: 36.83% [700500/1902000] [368.3/1000.0] [batch_t 1.455 (1.457)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 18:28:05,594 - Train: 36.83% [700600/1902000] [368.3/1000.0] [batch_t 1.461 (1.457)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 18:30:31,210 - Train: 36.84% [700700/1902000] [368.4/1000.0] [batch_t 1.460 (1.456)] [data_t 0.003] [optim_t 1.456] [lr 0.005000] +2024-04-08 18:32:56,800 - Train: 36.85% [700800/1902000] [368.5/1000.0] [batch_t 1.451 (1.456)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-08 18:35:22,462 - Train: 36.85% [700900/1902000] [368.5/1000.0] [batch_t 1.451 (1.457)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-08 18:37:48,157 - Train: 36.86% [701000/1902000] [368.6/1000.0] [batch_t 1.460 (1.457)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 18:40:13,818 - Train: 36.86% [701100/1902000] [368.6/1000.0] [batch_t 1.461 (1.457)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 18:42:39,510 - Train: 36.87% [701200/1902000] [368.7/1000.0] [batch_t 1.454 (1.457)] [data_t 0.003] [optim_t 1.450] [lr 0.005000] +2024-04-08 18:45:05,120 - Train: 36.87% [701300/1902000] [368.7/1000.0] [batch_t 1.460 (1.456)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 18:47:30,776 - Train: 36.88% [701400/1902000] [368.8/1000.0] [batch_t 1.457 (1.456)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 18:49:56,374 - Train: 36.88% [701500/1902000] [368.8/1000.0] [batch_t 1.462 (1.456)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 18:52:21,959 - Train: 36.89% [701600/1902000] [368.9/1000.0] [batch_t 1.457 (1.456)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 18:54:47,544 - Train: 36.89% [701700/1902000] [368.9/1000.0] [batch_t 1.454 (1.456)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 18:57:13,173 - Train: 36.90% [701800/1902000] [369.0/1000.0] [batch_t 1.450 (1.456)] [data_t 0.003] [optim_t 1.446] [lr 0.005000] +2024-04-08 18:58:08,456 - ==> Total time: 6 days, 17:57:47 Eta: 11 days, 12:57:42 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-08 18:59:41,520 - Train: 36.90% [701900/1902000] [369.0/1000.0] [batch_t 1.468 (1.458)] [data_t 0.003] [optim_t 1.465] [lr 0.005000] +2024-04-08 19:02:07,163 - Train: 36.91% [702000/1902000] [369.1/1000.0] [batch_t 1.461 (1.456)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 19:04:32,801 - Train: 36.91% [702100/1902000] [369.1/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 19:06:58,430 - Train: 36.92% [702200/1902000] [369.2/1000.0] [batch_t 1.462 (1.456)] [data_t 0.003] [optim_t 1.459] [lr 0.005000] +2024-04-08 19:09:23,953 - Train: 36.92% [702300/1902000] [369.2/1000.0] [batch_t 1.460 (1.455)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 19:11:49,521 - Train: 36.93% [702400/1902000] [369.3/1000.0] [batch_t 1.457 (1.456)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 19:14:15,098 - Train: 36.93% [702500/1902000] [369.3/1000.0] [batch_t 1.461 (1.456)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 19:16:40,757 - Train: 36.94% [702600/1902000] [369.4/1000.0] [batch_t 1.455 (1.456)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 19:19:06,387 - Train: 36.95% [702700/1902000] [369.5/1000.0] [batch_t 1.447 (1.456)] [data_t 0.003] [optim_t 1.445] [lr 0.005000] +2024-04-08 19:21:31,995 - Train: 36.95% [702800/1902000] [369.5/1000.0] [batch_t 1.457 (1.456)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 19:23:57,671 - Train: 36.96% [702900/1902000] [369.6/1000.0] [batch_t 1.451 (1.457)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-08 19:26:23,297 - Train: 36.96% [703000/1902000] [369.6/1000.0] [batch_t 1.452 (1.456)] [data_t 0.003] [optim_t 1.449] [lr 0.005000] +2024-04-08 19:28:48,915 - Train: 36.97% [703100/1902000] [369.7/1000.0] [batch_t 1.457 (1.456)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 19:31:14,468 - Train: 36.97% [703200/1902000] [369.7/1000.0] [batch_t 1.461 (1.455)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 19:33:40,109 - Train: 36.98% [703300/1902000] [369.8/1000.0] [batch_t 1.453 (1.456)] [data_t 0.003] [optim_t 1.450] [lr 0.005000] +2024-04-08 19:36:05,707 - Train: 36.98% [703400/1902000] [369.8/1000.0] [batch_t 1.466 (1.456)] [data_t 0.003] [optim_t 1.463] [lr 0.005000] +2024-04-08 19:38:31,292 - Train: 36.99% [703500/1902000] [369.9/1000.0] [batch_t 1.458 (1.456)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 19:40:56,858 - Train: 36.99% [703600/1902000] [369.9/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 19:43:22,387 - Train: 37.00% [703700/1902000] [370.0/1000.0] [batch_t 1.460 (1.455)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 19:44:20,617 - ==> Total time: 6 days, 18:44:00 Eta: 11 days, 13:05:11 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-08 19:45:50,601 - Train: 37.00% [703800/1902000] [370.0/1000.0] [batch_t 1.456 (1.459)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 19:48:16,201 - Train: 37.01% [703900/1902000] [370.1/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 19:50:41,772 - Train: 37.01% [704000/1902000] [370.1/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 19:53:07,472 - Train: 37.02% [704100/1902000] [370.2/1000.0] [batch_t 1.462 (1.457)] [data_t 0.003] [optim_t 1.459] [lr 0.005000] +2024-04-08 19:55:33,130 - Train: 37.02% [704200/1902000] [370.2/1000.0] [batch_t 1.458 (1.456)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 19:57:58,808 - Train: 37.03% [704300/1902000] [370.3/1000.0] [batch_t 1.460 (1.457)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 20:00:24,439 - Train: 37.03% [704400/1902000] [370.3/1000.0] [batch_t 1.458 (1.456)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 20:02:50,056 - Train: 37.04% [704500/1902000] [370.4/1000.0] [batch_t 1.460 (1.456)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 20:05:15,594 - Train: 37.05% [704600/1902000] [370.5/1000.0] [batch_t 1.455 (1.455)] [data_t 0.003] [optim_t 1.452] [lr 0.005000] +2024-04-08 20:07:41,217 - Train: 37.05% [704700/1902000] [370.5/1000.0] [batch_t 1.457 (1.456)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 20:10:06,747 - Train: 37.06% [704800/1902000] [370.6/1000.0] [batch_t 1.452 (1.455)] [data_t 0.003] [optim_t 1.449] [lr 0.005000] +2024-04-08 20:12:32,390 - Train: 37.06% [704900/1902000] [370.6/1000.0] [batch_t 1.459 (1.456)] [data_t 0.003] [optim_t 1.456] [lr 0.005000] +2024-04-08 20:14:58,028 - Train: 37.07% [705000/1902000] [370.7/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 20:17:23,623 - Train: 37.07% [705100/1902000] [370.7/1000.0] [batch_t 1.454 (1.456)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 20:19:49,236 - Train: 37.08% [705200/1902000] [370.8/1000.0] [batch_t 1.467 (1.456)] [data_t 0.003] [optim_t 1.464] [lr 0.005000] +2024-04-08 20:22:14,863 - Train: 37.08% [705300/1902000] [370.8/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 20:24:40,455 - Train: 37.09% [705400/1902000] [370.9/1000.0] [batch_t 1.455 (1.456)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 20:27:06,004 - Train: 37.09% [705500/1902000] [370.9/1000.0] [batch_t 1.453 (1.455)] [data_t 0.003] [optim_t 1.450] [lr 0.005000] +2024-04-08 20:29:31,655 - Train: 37.10% [705600/1902000] [371.0/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 20:30:32,788 - ==> Total time: 6 days, 19:30:12 Eta: 11 days, 13:12:23 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-08 20:32:00,197 - Train: 37.10% [705700/1902000] [371.0/1000.0] [batch_t 1.448 (1.462)] [data_t 0.003] [optim_t 1.445] [lr 0.005000] +2024-04-08 20:34:25,802 - Train: 37.11% [705800/1902000] [371.1/1000.0] [batch_t 1.466 (1.456)] [data_t 0.004] [optim_t 1.462] [lr 0.005000] +2024-04-08 20:36:51,431 - Train: 37.11% [705900/1902000] [371.1/1000.0] [batch_t 1.452 (1.456)] [data_t 0.003] [optim_t 1.449] [lr 0.005000] +2024-04-08 20:39:17,050 - Train: 37.12% [706000/1902000] [371.2/1000.0] [batch_t 1.454 (1.456)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 20:41:42,667 - Train: 37.12% [706100/1902000] [371.2/1000.0] [batch_t 1.459 (1.456)] [data_t 0.003] [optim_t 1.456] [lr 0.005000] +2024-04-08 20:44:08,280 - Train: 37.13% [706200/1902000] [371.3/1000.0] [batch_t 1.450 (1.456)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-08 20:46:33,833 - Train: 37.13% [706300/1902000] [371.3/1000.0] [batch_t 1.451 (1.455)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-08 20:48:59,353 - Train: 37.14% [706400/1902000] [371.4/1000.0] [batch_t 1.462 (1.455)] [data_t 0.003] [optim_t 1.459] [lr 0.005000] +2024-04-08 20:51:25,038 - Train: 37.15% [706500/1902000] [371.5/1000.0] [batch_t 1.459 (1.457)] [data_t 0.003] [optim_t 1.455] [lr 0.005000] +2024-04-08 20:53:50,588 - Train: 37.15% [706600/1902000] [371.5/1000.0] [batch_t 1.447 (1.455)] [data_t 0.003] [optim_t 1.444] [lr 0.005000] +2024-04-08 20:56:16,259 - Train: 37.16% [706700/1902000] [371.6/1000.0] [batch_t 1.462 (1.457)] [data_t 0.003] [optim_t 1.459] [lr 0.005000] +2024-04-08 20:58:41,853 - Train: 37.16% [706800/1902000] [371.6/1000.0] [batch_t 1.457 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 21:01:07,420 - Train: 37.17% [706900/1902000] [371.7/1000.0] [batch_t 1.461 (1.456)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 21:03:32,957 - Train: 37.17% [707000/1902000] [371.7/1000.0] [batch_t 1.456 (1.455)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 21:05:58,554 - Train: 37.18% [707100/1902000] [371.8/1000.0] [batch_t 1.454 (1.456)] [data_t 0.003] [optim_t 1.451] [lr 0.005000] +2024-04-08 21:08:24,085 - Train: 37.18% [707200/1902000] [371.8/1000.0] [batch_t 1.458 (1.455)] [data_t 0.003] [optim_t 1.454] [lr 0.005000] +2024-04-08 21:10:49,664 - Train: 37.19% [707300/1902000] [371.9/1000.0] [batch_t 1.462 (1.456)] [data_t 0.003] [optim_t 1.458] [lr 0.005000] +2024-04-08 21:13:15,290 - Train: 37.19% [707400/1902000] [371.9/1000.0] [batch_t 1.450 (1.456)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-08 21:15:40,862 - Train: 37.20% [707500/1902000] [372.0/1000.0] [batch_t 1.440 (1.456)] [data_t 0.003] [optim_t 1.437] [lr 0.005000] +2024-04-08 21:16:44,951 - ==> Total time: 6 days, 20:16:24 Eta: 11 days, 13:19:18 Logged in 'runs/MAMBAADTrainer_configs_mambaad_mambaad_coco0_nhcs57c1_20240402-010020' +2024-04-08 21:18:08,585 - Train: 37.20% [707600/1902000] [372.0/1000.0] [batch_t 1.465 (1.455)] [data_t 0.003] [optim_t 1.462] [lr 0.005000] +2024-04-08 21:20:34,164 - Train: 37.21% [707700/1902000] [372.1/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 21:22:59,768 - Train: 37.21% [707800/1902000] [372.1/1000.0] [batch_t 1.451 (1.456)] [data_t 0.003] [optim_t 1.448] [lr 0.005000] +2024-04-08 21:25:25,307 - Train: 37.22% [707900/1902000] [372.2/1000.0] [batch_t 1.461 (1.455)] [data_t 0.003] [optim_t 1.457] [lr 0.005000] +2024-04-08 21:27:50,877 - Train: 37.22% [708000/1902000] [372.2/1000.0] [batch_t 1.462 (1.456)] [data_t 0.003] [optim_t 1.459] [lr 0.005000] +2024-04-08 21:30:16,510 - Train: 37.23% [708100/1902000] [372.3/1000.0] [batch_t 1.456 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000] +2024-04-08 21:32:42,145 - Train: 37.23% [708200/1902000] [372.3/1000.0] [batch_t 1.464 (1.456)] [data_t 0.003] [optim_t 1.461] [lr 0.005000] +2024-04-08 21:35:07,768 - Train: 37.24% [708300/1902000] [372.4/1000.0] [batch_t 1.451 (1.456)] [data_t 0.003] [optim_t 1.447] [lr 0.005000] +2024-04-08 21:37:33,371 - Train: 37.25% [708400/1902000] [372.5/1000.0] [batch_t 1.457 (1.456)] [data_t 0.003] [optim_t 1.453] [lr 0.005000]