KillerBoss commited on
Commit
c5f08b1
·
verified ·
1 Parent(s): 89d59a6

log: export v6 npu-fail-SM8750

Browse files
Files changed (1) hide show
  1. logs/export_v6.log +359 -0
logs/export_v6.log CHANGED
@@ -486,3 +486,362 @@ ValueError: apply_plugin failed to apply plugin. See /tmp/tmp97ocbbyw.error for
486
 
487
  litertlm-Dateien: ['/kaggle/working/export_npu/model.litertlm']
488
  ### NPU-Export mit SM8850 fehlgeschlagen (rc=1)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
486
 
487
  litertlm-Dateien: ['/kaggle/working/export_npu/model.litertlm']
488
  ### NPU-Export mit SM8850 fehlgeschlagen (rc=1)
489
+ >>> Log nach HF (npu-fail-SM8850)
490
+
491
+ ### NPU-Versuch mit SoC: SM8750
492
+ + litert-torch export_hf /kaggle/tmp/functiongemma-plantgame-de /kaggle/working/export_npu --task=text_generation --bundle_litert_lm=True --aot_backend=qualcomm --aot_soc_model=SM8750 --use_litert_lm_compiler=True --externalize_embedder=True --prefill_lengths=128 --cache_length=1280 --keep_temporary_files=True
493
+ h: 1280
494
+ }
495
+ state_buffers {
496
+ prefill_input_name: "kv_cache_v_3"
497
+ prefill_output_name: "kv_cache_v_3"
498
+ decode_input_name: "kv_cache_v_3"
499
+ decode_output_name: "kv_cache_v_3"
500
+ type: TYPE_GLOBAL_VALUE_CACHE
501
+ sequence_axis: 2
502
+ maximum_sequence_length: 1280
503
+ }
504
+ state_buffers {
505
+ prefill_input_name: "kv_cache_k_4"
506
+ prefill_output_name: "kv_cache_k_4"
507
+ decode_input_name: "kv_cache_k_4"
508
+ decode_output_name: "kv_cache_k_4"
509
+ type: TYPE_GLOBAL_KEY_CACHE
510
+ sequence_axis: 3
511
+ maximum_sequence_length: 1280
512
+ }
513
+ state_buffers {
514
+ prefill_input_name: "kv_cache_v_4"
515
+ prefill_output_name: "kv_cache_v_4"
516
+ decode_input_name: "kv_cache_v_4"
517
+ decode_output_name: "kv_cache_v_4"
518
+ type: TYPE_GLOBAL_VALUE_CACHE
519
+ sequence_axis: 2
520
+ maximum_sequence_length: 1280
521
+ }
522
+ state_buffers {
523
+ prefill_input_name: "kv_cache_k_5"
524
+ prefill_output_name: "kv_cache_k_5"
525
+ decode_input_name: "kv_cache_k_5"
526
+ decode_output_name: "kv_cache_k_5"
527
+ type: TYPE_GLOBAL_KEY_CACHE
528
+ sequence_axis: 3
529
+ maximum_sequence_length: 1280
530
+ }
531
+ state_buffers {
532
+ prefill_input_name: "kv_cache_v_5"
533
+ prefill_output_name: "kv_cache_v_5"
534
+ decode_input_name: "kv_cache_v_5"
535
+ decode_output_name: "kv_cache_v_5"
536
+ type: TYPE_GLOBAL_VALUE_CACHE
537
+ sequence_axis: 2
538
+ maximum_sequence_length: 1280
539
+ }
540
+ state_buffers {
541
+ prefill_input_name: "kv_cache_k_6"
542
+ prefill_output_name: "kv_cache_k_6"
543
+ decode_input_name: "kv_cache_k_6"
544
+ decode_output_name: "kv_cache_k_6"
545
+ type: TYPE_GLOBAL_KEY_CACHE
546
+ sequence_axis: 3
547
+ maximum_sequence_length: 1280
548
+ }
549
+ state_buffers {
550
+ prefill_input_name: "kv_cache_v_6"
551
+ prefill_output_name: "kv_cache_v_6"
552
+ decode_input_name: "kv_cache_v_6"
553
+ decode_output_name: "kv_cache_v_6"
554
+ type: TYPE_GLOBAL_VALUE_CACHE
555
+ sequence_axis: 2
556
+ maximum_sequence_length: 1280
557
+ }
558
+ state_buffers {
559
+ prefill_input_name: "kv_cache_k_7"
560
+ prefill_output_name: "kv_cache_k_7"
561
+ decode_input_name: "kv_cache_k_7"
562
+ decode_output_name: "kv_cache_k_7"
563
+ type: TYPE_GLOBAL_KEY_CACHE
564
+ sequence_axis: 3
565
+ maximum_sequence_length: 1280
566
+ }
567
+ state_buffers {
568
+ prefill_input_name: "kv_cache_v_7"
569
+ prefill_output_name: "kv_cache_v_7"
570
+ decode_input_name: "kv_cache_v_7"
571
+ decode_output_name: "kv_cache_v_7"
572
+ type: TYPE_GLOBAL_VALUE_CACHE
573
+ sequence_axis: 2
574
+ maximum_sequence_length: 1280
575
+ }
576
+ state_buffers {
577
+ prefill_input_name: "kv_cache_k_8"
578
+ prefill_output_name: "kv_cache_k_8"
579
+ decode_input_name: "kv_cache_k_8"
580
+ decode_output_name: "kv_cache_k_8"
581
+ type: TYPE_GLOBAL_KEY_CACHE
582
+ sequence_axis: 3
583
+ maximum_sequence_length: 1280
584
+ }
585
+ state_buffers {
586
+ prefill_input_name: "kv_cache_v_8"
587
+ prefill_output_name: "kv_cache_v_8"
588
+ decode_input_name: "kv_cache_v_8"
589
+ decode_output_name: "kv_cache_v_8"
590
+ type: TYPE_GLOBAL_VALUE_CACHE
591
+ sequence_axis: 2
592
+ maximum_sequence_length: 1280
593
+ }
594
+ state_buffers {
595
+ prefill_input_name: "kv_cache_k_9"
596
+ prefill_output_name: "kv_cache_k_9"
597
+ decode_input_name: "kv_cache_k_9"
598
+ decode_output_name: "kv_cache_k_9"
599
+ type: TYPE_GLOBAL_KEY_CACHE
600
+ sequence_axis: 3
601
+ maximum_sequence_length: 1280
602
+ }
603
+ state_buffers {
604
+ prefill_input_name: "kv_cache_v_9"
605
+ prefill_output_name: "kv_cache_v_9"
606
+ decode_input_name: "kv_cache_v_9"
607
+ decode_output_name: "kv_cache_v_9"
608
+ type: TYPE_GLOBAL_VALUE_CACHE
609
+ sequence_axis: 2
610
+ maximum_sequence_length: 1280
611
+ }
612
+ state_buffers {
613
+ prefill_input_name: "kv_cache_k_10"
614
+ prefill_output_name: "kv_cache_k_10"
615
+ decode_input_name: "kv_cache_k_10"
616
+ decode_output_name: "kv_cache_k_10"
617
+ type: TYPE_GLOBAL_KEY_CACHE
618
+ sequence_axis: 3
619
+ maximum_sequence_length: 1280
620
+ }
621
+ state_buffers {
622
+ prefill_input_name: "kv_cache_v_10"
623
+ prefill_output_name: "kv_cache_v_10"
624
+ decode_input_name: "kv_cache_v_10"
625
+ decode_output_name: "kv_cache_v_10"
626
+ type: TYPE_GLOBAL_VALUE_CACHE
627
+ sequence_axis: 2
628
+ maximum_sequence_length: 1280
629
+ }
630
+ state_buffers {
631
+ prefill_input_name: "kv_cache_k_11"
632
+ prefill_output_name: "kv_cache_k_11"
633
+ decode_input_name: "kv_cache_k_11"
634
+ decode_output_name: "kv_cache_k_11"
635
+ type: TYPE_GLOBAL_KEY_CACHE
636
+ sequence_axis: 3
637
+ maximum_sequence_length: 1280
638
+ }
639
+ state_buffers {
640
+ prefill_input_name: "kv_cache_v_11"
641
+ prefill_output_name: "kv_cache_v_11"
642
+ decode_input_name: "kv_cache_v_11"
643
+ decode_output_name: "kv_cache_v_11"
644
+ type: TYPE_GLOBAL_VALUE_CACHE
645
+ sequence_axis: 2
646
+ maximum_sequence_length: 1280
647
+ }
648
+ state_buffers {
649
+ prefill_input_name: "kv_cache_k_12"
650
+ prefill_output_name: "kv_cache_k_12"
651
+ decode_input_name: "kv_cache_k_12"
652
+ decode_output_name: "kv_cache_k_12"
653
+ type: TYPE_GLOBAL_KEY_CACHE
654
+ sequence_axis: 3
655
+ maximum_sequence_length: 1280
656
+ }
657
+ state_buffers {
658
+ prefill_input_name: "kv_cache_v_12"
659
+ prefill_output_name: "kv_cache_v_12"
660
+ decode_input_name: "kv_cache_v_12"
661
+ decode_output_name: "kv_cache_v_12"
662
+ type: TYPE_GLOBAL_VALUE_CACHE
663
+ sequence_axis: 2
664
+ maximum_sequence_length: 1280
665
+ }
666
+ state_buffers {
667
+ prefill_input_name: "kv_cache_k_13"
668
+ prefill_output_name: "kv_cache_k_13"
669
+ decode_input_name: "kv_cache_k_13"
670
+ decode_output_name: "kv_cache_k_13"
671
+ type: TYPE_GLOBAL_KEY_CACHE
672
+ sequence_axis: 3
673
+ maximum_sequence_length: 1280
674
+ }
675
+ state_buffers {
676
+ prefill_input_name: "kv_cache_v_13"
677
+ prefill_output_name: "kv_cache_v_13"
678
+ decode_input_name: "kv_cache_v_13"
679
+ decode_output_name: "kv_cache_v_13"
680
+ type: TYPE_GLOBAL_VALUE_CACHE
681
+ sequence_axis: 2
682
+ maximum_sequence_length: 1280
683
+ }
684
+ state_buffers {
685
+ prefill_input_name: "kv_cache_k_14"
686
+ prefill_output_name: "kv_cache_k_14"
687
+ decode_input_name: "kv_cache_k_14"
688
+ decode_output_name: "kv_cache_k_14"
689
+ type: TYPE_GLOBAL_KEY_CACHE
690
+ sequence_axis: 3
691
+ maximum_sequence_length: 1280
692
+ }
693
+ state_buffers {
694
+ prefill_input_name: "kv_cache_v_14"
695
+ prefill_output_name: "kv_cache_v_14"
696
+ decode_input_name: "kv_cache_v_14"
697
+ decode_output_name: "kv_cache_v_14"
698
+ type: TYPE_GLOBAL_VALUE_CACHE
699
+ sequence_axis: 2
700
+ maximum_sequence_length: 1280
701
+ }
702
+ state_buffers {
703
+ prefill_input_name: "kv_cache_k_15"
704
+ prefill_output_name: "kv_cache_k_15"
705
+ decode_input_name: "kv_cache_k_15"
706
+ decode_output_name: "kv_cache_k_15"
707
+ type: TYPE_GLOBAL_KEY_CACHE
708
+ sequence_axis: 3
709
+ maximum_sequence_length: 1280
710
+ }
711
+ state_buffers {
712
+ prefill_input_name: "kv_cache_v_15"
713
+ prefill_output_name: "kv_cache_v_15"
714
+ decode_input_name: "kv_cache_v_15"
715
+ decode_output_name: "kv_cache_v_15"
716
+ type: TYPE_GLOBAL_VALUE_CACHE
717
+ sequence_axis: 2
718
+ maximum_sequence_length: 1280
719
+ }
720
+ state_buffers {
721
+ prefill_input_name: "kv_cache_k_16"
722
+ prefill_output_name: "kv_cache_k_16"
723
+ decode_input_name: "kv_cache_k_16"
724
+ decode_output_name: "kv_cache_k_16"
725
+ type: TYPE_GLOBAL_KEY_CACHE
726
+ sequence_axis: 3
727
+ maximum_sequence_length: 1280
728
+ }
729
+ state_buffers {
730
+ prefill_input_name: "kv_cache_v_16"
731
+ prefill_output_name: "kv_cache_v_16"
732
+ decode_input_name: "kv_cache_v_16"
733
+ decode_output_name: "kv_cache_v_16"
734
+ type: TYPE_GLOBAL_VALUE_CACHE
735
+ sequence_axis: 2
736
+ maximum_sequence_length: 1280
737
+ }
738
+ state_buffers {
739
+ prefill_input_name: "kv_cache_k_17"
740
+ prefill_output_name: "kv_cache_k_17"
741
+ decode_input_name: "kv_cache_k_17"
742
+ decode_output_name: "kv_cache_k_17"
743
+ type: TYPE_GLOBAL_KEY_CACHE
744
+ sequence_axis: 3
745
+ maximum_sequence_length: 1280
746
+ }
747
+ state_buffers {
748
+ prefill_input_name: "kv_cache_v_17"
749
+ prefill_output_name: "kv_cache_v_17"
750
+ decode_input_name: "kv_cache_v_17"
751
+ decode_output_name: "kv_cache_v_17"
752
+ type: TYPE_GLOBAL_VALUE_CACHE
753
+ sequence_axis: 2
754
+ maximum_sequence_length: 1280
755
+ }
756
+ }
757
+ >>>>>>>> end of ExecutorMetadata
758
+ ExecutorMetadataProto.pbtext dumped to: /tmp/tmpsrxymt_f/ExecutorMetadataProto.pbtext
759
+
760
+
761
+ Section 2:
762
+ Items:
763
+ <None>
764
+ Begin Offset: 49152
765
+ End Offset: 5635536
766
+ Data Type: HF_Tokenizer_Zlib
767
+ Section2_HF_Tokenizer_Zlib dumped to: /tmp/tmpsrxymt_f/Section2_HF_Tokenizer_Zlib.zlib
768
+
769
+
770
+ Section 3:
771
+ Items:
772
+ Key: model_type, Value (String): tf_lite_prefill_decode
773
+ Begin Offset: 5636096
774
+ End Offset: 282505680
775
+ Data Type: TFLiteModel
776
+ Section3_TFLiteModel_tf_lite_prefill_decode.tflite dumped to: /tmp/tmpsrxymt_f/Section3_TFLiteModel_tf_lite_prefill_decode.tflite
777
+
778
+
779
+ Section 4:
780
+ Items:
781
+ Key: model_type, Value (String): tf_lite_embedder
782
+ Begin Offset: 282509312
783
+ End Offset: 456578352
784
+ Data Type: TFLiteModel
785
+ Section4_TFLiteModel_tf_lite_embedder.tflite dumped to: /tmp/tmpsrxymt_f/Section4_TFLiteModel_tf_lite_embedder.tflite
786
+
787
+
788
+ Section 5:
789
+ Items:
790
+ Key: model_type, Value (String): tf_lite_aux
791
+ Begin Offset: 456589312
792
+ End Offset: 456704036
793
+ Data Type: TFLiteModel
794
+ Section5_TFLiteModel_tf_lite_aux.tflite dumped to: /tmp/tmpsrxymt_f/Section5_TFLiteModel_tf_lite_aux.tflite
795
+
796
+
797
+ RC=1
798
+ STDERR:
799
+ Skipping import of cpp extensions due to incompatible torch version. Please upgrade to torch >= 2.11.0 (found 2.10.0+cpu).
800
+
801
+ Loading weights: 0%| | 0/236 [00:00<?, ?it/s]
802
+ Loading weights: 100%|██████████| 236/236 [00:00<00:00, 5467.58it/s]
803
+ /usr/lib/python3.12/contextlib.py:144: UserWarning: The tensor attributes self.model.model.rotary_emb.data['sliding_attention'][0], self.model.model.rotary_emb.data['sliding_attention'][1], self.model.model.rotary_emb.data['full_attention'][0], self.model.model.rotary_emb.data['full_attention'][1] were assigned during export. Such attributes must be registered as buffers using the `register_buffer` API (https://pytorch.org/docs/stable/generated/torch.nn.Module.html#torch.nn.Module.register_buffer).
804
+ next(self.gen)
805
+
806
+ Generating Quantization Parameters:: 0%| | 0/3585 [00:00<?, ?it/s]
807
+ Generating Quantization Parameters:: 7%|▋ | 255/3585 [00:00<00:01, 2472.70it/s]
808
+ Generating Quantization Parameters:: 14%|█▍ | 507/3585 [00:00<00:01, 2471.51it/s]
809
+ Generating Quantization Parameters:: 21%|██ | 755/3585 [00:00<00:01, 2461.65it/s]
810
+ Generating Quantization Parameters:: 42%|████▏ | 1510/3585 [00:00<00:00, 4446.10it/s]
811
+ Generating Quantization Parameters:: 55%|█████▍ | 1957/3585 [00:01<00:01, 1280.67it/s]
812
+ Generating Quantization Parameters:: 100%|██████████| 3585/3585 [00:01<00:00, 2799.89it/s]
813
+
814
+ Applying Transformations to tensors:: 0%| | 0/4706 [00:00<?, ?it/s]
815
+ Applying Transformations to tensors:: 100%|██████████| 4706/4706 [00:00<00:00, 162219.60it/s]
816
+ WARNING:root:The model /kaggle/working/export_npu/model_quantized.tflite already exists in the folder. Overwriting the model since overwrite=True.
817
+ WARNING:root:The model /kaggle/working/export_npu/embedder_quantized.tflite already exists in the folder. Overwriting the model since overwrite=True.
818
+ ERROR:absl:Model path /tmp/tmpsrxymt_f/Section3_TFLiteModel_tf_lite_prefill_decode.tflite size: 276869584
819
+ ERROR:absl:Failed to compile prefill_decode: apply_plugin failed to apply plugin. See /tmp/tmpgseixfm9.error for details.
820
+ Traceback (most recent call last):
821
+ File "/usr/local/bin/litert-torch", line 8, in <module>
822
+ sys.exit(main())
823
+ ^^^^^^
824
+ File "/usr/local/lib/python3.12/dist-packages/litert_torch/cli.py", line 30, in main
825
+ fire.Fire(CLI())
826
+ File "/usr/local/lib/python3.12/dist-packages/fire/core.py", line 135, in Fire
827
+ component_trace = _Fire(component, args, parsed_flag_args, context, name)
828
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
829
+ File "/usr/local/lib/python3.12/dist-packages/fire/core.py", line 468, in _Fire
830
+ component, remaining_args = _CallAndUpdateTrace(
831
+ ^^^^^^^^^^^^^^^^^^^^
832
+ File "/usr/local/lib/python3.12/dist-packages/fire/core.py", line 684, in _CallAndUpdateTrace
833
+ component = fn(*varargs, **kwargs)
834
+ ^^^^^^^^^^^^^^^^^^^^^^
835
+ File "/usr/local/lib/python3.12/dist-packages/litert_torch/generative/export_hf/export.py", line 361, in export
836
+ export_utils.run_npu_compilation_pipeline(
837
+ File "/usr/local/lib/python3.12/dist-packages/litert_torch/generative/export_hf/export_utils.py", line 147, in run_npu_compilation_pipeline
838
+ compile_func(
839
+ File "/usr/local/lib/python3.12/dist-packages/litert_torch/generative/export_hf/experimental/litert_lm_npu_compiler/litert_lm_npu_compiler.py", line 406, in compile_litertlm
840
+ compiler(
841
+ File "/usr/local/lib/python3.12/dist-packages/ai_edge_litert/aot/core/apply_plugin.py", line 146, in __call__
842
+ raise ValueError(
843
+ ValueError: apply_plugin failed to apply plugin. See /tmp/tmpgseixfm9.error for details.
844
+
845
+ litertlm-Dateien: ['/kaggle/working/export_npu/model.litertlm']
846
+ ### NPU-Export mit SM8750 fehlgeschlagen (rc=1)
847
+