Upload folder using huggingface_hub

checkpoints/asr_transformer_HierDecayv2_woBias_XavierInit_Qv7/100epoch.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0dd0daaefe2355e6b0151b39c8f53b350fc4f531973a9cff2cb5dfb6fffeaddf
+size 398012507

checkpoints/asr_transformer_HierDecayv2_woBias_XavierInit_Qv7/64epoch.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab1a15b413af7c42ce18c259b8fb17d33196e4195a9e2100ecd498a0626ef41a
+size 398011377

checkpoints/asr_transformer_HierDecayv2_woBias_XavierInit_Qv7/72epoch.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ecbaecace577d8cafd1663294454dc1a1f1e26e2367df846e34411c28514d79
+size 398011377

checkpoints/asr_transformer_HierDecayv2_woBias_XavierInit_Qv7/74epoch.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e219419dd596a9af9ca66d2c07f7313798312122a37a9a3ffd8e931e32b89117
+size 398011377

checkpoints/asr_transformer_HierDecayv2_woBias_XavierInit_Qv7/81epoch.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b377bef5802f2e9c3f869171cf63538eb13e2783838e159caaa363396b058288
+size 398011377

checkpoints/asr_transformer_HierDecayv2_woBias_XavierInit_Qv7/86epoch.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cac0e8ed3cb48a764075095dace05f187f8fc060c991231a115b2883ae35bd76
+size 398011377

checkpoints/asr_transformer_HierDecayv2_woBias_XavierInit_Qv7/92epoch.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f18d055595cfb7665eca64e5079be6c0e7ed2d06871df4544f43dc9b81abcac6
+size 398011377

checkpoints/asr_transformer_HierDecayv2_woBias_XavierInit_Qv7/95epoch.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88bb8537bd4761cf34777676b35a826f89ebefe561683f88ad9f7f31c453d310
+size 398011377

checkpoints/asr_transformer_HierDecayv2_woBias_XavierInit_Qv7/98epoch.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:323a4273ec9025bdbd076a7cb947c176d25d86ad8dd91c7922833adfcb7e55c0
+size 398011377

checkpoints/asr_transformer_HierDecayv2_woBias_XavierInit_Qv7/99epoch.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b060bf5418cbd9020192dbed4490a99a803e47389627057a78d497138b8aa75d
+size 398011377

checkpoints/asr_transformer_HierDecayv2_woBias_XavierInit_Qv7/RESULTS.md

@@ -0,0 +1,38 @@
+<!-- Generated by scripts/utils/show_asr_result.sh -->
+# RESULTS
+## Environments
+- date: `Thu Jul 18 18:36:12 HKT 2024`
+- python version: `3.8.19 (default, Mar 20 2024, 19:58:24)  [GCC 11.2.0]`
+- espnet version: `espnet 202402`
+- pytorch version: `pytorch 1.13.1+cu117`
+- Git hash: `19787b1793eda2b4007aa5b2c4d03adf6c18abfb`
+  - Commit date: `Fri Jun 14 19:27:35 2024 +0900`
+
+## exp/asr_transformer_HierDecayv2_woBias_XavierInit_Qv7
+### WER
+
+|dataset|Snt|Wrd|Corr|Sub|Del|Ins|Err|S.Err|
+|---|---|---|---|---|---|---|---|---|
+|decode_asr_asr_model_valid.acc.ave/dev_clean|2703|54402|97.2|2.5|0.3|0.3|3.1|34.9|
+|decode_asr_asr_model_valid.acc.ave/dev_other|2864|50948|92.6|6.7|0.7|0.9|8.3|56.5|
+|decode_asr_asr_model_valid.acc.ave/test_clean|2620|52576|97.0|2.6|0.4|0.4|3.4|36.1|
+|decode_asr_asr_model_valid.acc.ave/test_other|2939|52343|93.0|6.3|0.7|1.0|7.9|57.4|
+
+### CER
+
+|dataset|Snt|Wrd|Corr|Sub|Del|Ins|Err|S.Err|
+|---|---|---|---|---|---|---|---|---|
+|decode_asr_asr_model_valid.acc.ave/dev_clean|2703|288456|99.2|0.4|0.4|0.3|1.2|34.9|
+|decode_asr_asr_model_valid.acc.ave/dev_other|2864|265951|97.1|1.7|1.2|1.0|3.8|56.5|
+|decode_asr_asr_model_valid.acc.ave/test_clean|2620|281530|99.0|0.4|0.5|0.4|1.3|36.1|
+|decode_asr_asr_model_valid.acc.ave/test_other|2939|272758|97.5|1.5|1.0|1.0|3.5|57.4|
+
+### TER
+
+|dataset|Snt|Wrd|Corr|Sub|Del|Ins|Err|S.Err|
+|---|---|---|---|---|---|---|---|---|
+|decode_asr_asr_model_valid.acc.ave/dev_clean|2703|68010|96.7|2.4|0.9|0.5|3.9|34.9|
+|decode_asr_asr_model_valid.acc.ave/dev_other|2864|63110|91.3|6.5|2.3|1.4|10.1|56.5|
+|decode_asr_asr_model_valid.acc.ave/test_clean|2620|65818|96.3|2.5|1.2|0.5|4.2|36.1|
+|decode_asr_asr_model_valid.acc.ave/test_other|2939|65101|91.7|5.9|2.3|1.2|9.4|57.4|
+

checkpoints/asr_transformer_HierDecayv2_woBias_XavierInit_Qv7/checkpoint.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:105b0b5d675fbdb31e2909bf6e275e154fb42673d91052d0a1e331e1e00940a1
+size 1193162349

checkpoints/asr_transformer_HierDecayv2_woBias_XavierInit_Qv7/config.yaml

@@ -0,0 +1,5211 @@
+config: conf/tuning/SNN/train_asr_Q_transformer3_HierDecayv2.yaml
+print_config: false
+log_level: INFO
+drop_last_iter: false
+dry_run: false
+iterator_type: sequence
+valid_iterator_type: null
+output_dir: exp/asr_transformer_HierDecayv2_woBias_XavierInit_Qv7
+ngpu: 1
+seed: 0
+num_workers: 1
+num_att_plot: 3
+dist_backend: nccl
+dist_init_method: env://
+dist_world_size: 4
+dist_rank: 0
+local_rank: 0
+dist_master_addr: localhost
+dist_master_port: 38893
+dist_launcher: null
+multiprocessing_distributed: true
+unused_parameters: false
+sharded_ddp: false
+cudnn_enabled: true
+cudnn_benchmark: false
+cudnn_deterministic: true
+collect_stats: false
+write_collected_feats: false
+max_epoch: 100
+patience: null
+val_scheduler_criterion:
+- valid
+- loss
+early_stopping_criterion:
+- valid
+- loss
+- min
+best_model_criterion:
+-   - valid
+    - acc
+    - max
+keep_nbest_models: 10
+nbest_averaging_interval: 0
+grad_clip: 5.0
+grad_clip_type: 2.0
+grad_noise: false
+accum_grad: 2
+no_forward_run: false
+resume: true
+train_dtype: float32
+use_amp: true
+log_interval: null
+use_matplotlib: true
+use_tensorboard: true
+create_graph_in_tensorboard: false
+use_wandb: false
+wandb_project: null
+wandb_id: null
+wandb_entity: null
+wandb_name: null
+wandb_model_log_interval: -1
+detect_anomaly: false
+use_adapter: false
+adapter: lora
+save_strategy: all
+adapter_conf: {}
+pretrain_path: null
+init_param: []
+ignore_init_mismatch: false
+freeze_param: []
+num_iters_per_epoch: null
+batch_size: 20
+valid_batch_size: null
+batch_bins: 45000000
+valid_batch_bins: null
+train_shape_file:
+- exp/asr_stats_raw_en_bpe5000_sp/train/speech_shape
+- exp/asr_stats_raw_en_bpe5000_sp/train/text_shape.bpe
+valid_shape_file:
+- exp/asr_stats_raw_en_bpe5000_sp/valid/speech_shape
+- exp/asr_stats_raw_en_bpe5000_sp/valid/text_shape.bpe
+batch_type: numel
+valid_batch_type: null
+fold_length:
+- 80000
+- 150
+sort_in_batch: descending
+shuffle_within_batch: false
+sort_batch: descending
+multiple_iterator: false
+chunk_length: 500
+chunk_shift_ratio: 0.5
+num_cache_chunks: 1024
+chunk_excluded_key_prefixes: []
+chunk_default_fs: null
+chunk_max_abs_length: null
+chunk_discard_short_samples: true
+train_data_path_and_name_and_type:
+-   - dump/raw/train_960_sp/wav.scp
+    - speech
+    - sound
+-   - dump/raw/train_960_sp/text
+    - text
+    - text
+valid_data_path_and_name_and_type:
+-   - dump/raw/dev/wav.scp
+    - speech
+    - sound
+-   - dump/raw/dev/text
+    - text
+    - text
+allow_variable_data_keys: false
+max_cache_size: 0.0
+max_cache_fd: 32
+allow_multi_rates: false
+valid_max_cache_size: null
+exclude_weight_decay: false
+exclude_weight_decay_conf: {}
+optim: adam
+optim_conf:
+    lr: 0.002
+scheduler: warmuplr
+scheduler_conf:
+    warmup_steps: 25000
+token_list:
+- <blank>
+- <unk>
+- ▁THE
+- S
+- ▁AND
+- ▁OF
+- ▁TO
+- ▁A
+- ▁IN
+- ▁I
+- ▁HE
+- ▁THAT
+- ▁WAS
+- ED
+- ▁IT
+- ''''
+- ▁HIS
+- ING
+- ▁YOU
+- ▁WITH
+- ▁FOR
+- ▁HAD
+- T
+- ▁AS
+- ▁HER
+- ▁IS
+- ▁BE
+- ▁BUT
+- ▁NOT
+- ▁SHE
+- D
+- ▁AT
+- ▁ON
+- LY
+- ▁HIM
+- ▁THEY
+- ▁ALL
+- ▁HAVE
+- ▁BY
+- ▁SO
+- ▁THIS
+- ▁MY
+- ▁WHICH
+- ▁ME
+- ▁SAID
+- ▁FROM
+- ▁ONE
+- Y
+- E
+- ▁WERE
+- ▁WE
+- ▁NO
+- N
+- ▁THERE
+- ▁OR
+- ER
+- ▁AN
+- ▁WHEN
+- ▁ARE
+- ▁THEIR
+- ▁WOULD
+- ▁IF
+- ▁WHAT
+- ▁THEM
+- ▁WHO
+- ▁OUT
+- M
+- ▁DO
+- ▁WILL
+- ▁UP
+- ▁BEEN
+- P
+- R
+- ▁MAN
+- ▁THEN
+- ▁COULD
+- ▁MORE
+- C
+- ▁INTO
+- ▁NOW
+- ▁VERY
+- ▁YOUR
+- ▁SOME
+- ▁LITTLE
+- ES
+- ▁TIME
+- RE
+- ▁CAN
+- ▁LIKE
+- LL
+- ▁ABOUT
+- ▁HAS
+- ▁THAN
+- ▁DID
+- ▁UPON
+- ▁OVER
+- IN
+- ▁ANY
+- ▁WELL
+- ▁ONLY
+- B
+- ▁SEE
+- ▁GOOD
+- ▁OTHER
+- ▁TWO
+- L
+- ▁KNOW
+- ▁GO
+- ▁DOWN
+- ▁BEFORE
+- A
+- AL
+- ▁OUR
+- ▁OLD
+- ▁SHOULD
+- ▁MADE
+- ▁AFTER
+- ▁GREAT
+- ▁DAY
+- ▁MUST
+- ▁COME
+- ▁HOW
+- ▁SUCH
+- ▁CAME
+- LE
+- ▁WHERE
+- ▁US
+- ▁NEVER
+- ▁THESE
+- ▁MUCH
+- ▁DE
+- ▁MISTER
+- ▁WAY
+- G
+- ▁S
+- ▁MAY
+- ATION
+- ▁LONG
+- OR
+- ▁AM
+- ▁FIRST
+- ▁BACK
+- ▁OWN
+- ▁RE
+- ▁AGAIN
+- ▁SAY
+- ▁MEN
+- ▁WENT
+- ▁HIMSELF
+- ▁HERE
+- NESS
+- ▁THINK
+- V
+- IC
+- ▁EVEN
+- ▁THOUGHT
+- ▁HAND
+- ▁JUST
+- ▁O
+- ▁UN
+- VE
+- ION
+- ▁ITS
+- 'ON'
+- ▁MAKE
+- ▁MIGHT
+- ▁TOO
+- K
+- ▁AWAY
+- ▁LIFE
+- TH
+- ▁WITHOUT
+- ST
+- ▁THROUGH
+- ▁MOST
+- ▁TAKE
+- ▁DON
+- ▁EVERY
+- F
+- O
+- ▁SHALL
+- ▁THOSE
+- ▁EYES
+- AR
+- ▁STILL
+- ▁LAST
+- ▁HOUSE
+- ▁HEAD
+- ABLE
+- ▁NOTHING
+- ▁NIGHT
+- ITY
+- ▁LET
+- ▁MANY
+- ▁OFF
+- ▁BEING
+- ▁FOUND
+- ▁WHILE
+- EN
+- ▁SAW
+- ▁GET
+- ▁PEOPLE
+- ▁FACE
+- ▁YOUNG
+- CH
+- ▁UNDER
+- ▁ONCE
+- ▁TELL
+- AN
+- ▁THREE
+- ▁PLACE
+- ▁ROOM
+- ▁YET
+- ▁SAME
+- IL
+- US
+- U
+- ▁FATHER
+- ▁RIGHT
+- EL
+- ▁THOUGH
+- ▁ANOTHER
+- LI
+- RI
+- ▁HEART
+- IT
+- ▁PUT
+- ▁TOOK
+- ▁GIVE
+- ▁EVER
+- ▁E
+- ▁PART
+- ▁WORK
+- ERS
+- ▁LOOK
+- ▁NEW
+- ▁KING
+- ▁MISSUS
+- ▁SIR
+- ▁LOVE
+- ▁MIND
+- ▁LOOKED
+- W
+- RY
+- ▁ASKED
+- ▁LEFT
+- ET
+- ▁LIGHT
+- CK
+- ▁DOOR
+- ▁MOMENT
+- RO
+- ▁WORLD
+- ▁THINGS
+- ▁HOME
+- UL
+- ▁THING
+- LA
+- ▁WHY
+- ▁MOTHER
+- ▁ALWAYS
+- ▁FAR
+- FUL
+- ▁WATER
+- CE
+- IVE
+- UR
+- ▁HEARD
+- ▁SOMETHING
+- ▁SEEMED
+- I
+- LO
+- ▁BECAUSE
+- OL
+- ▁END
+- ▁TOLD
+- ▁CON
+- ▁YES
+- ▁GOING
+- ▁GOT
+- RA
+- IR
+- ▁WOMAN
+- ▁GOD
+- EST
+- TED
+- ▁FIND
+- ▁KNEW
+- ▁SOON
+- ▁EACH
+- ▁SIDE
+- H
+- TON
+- MENT
+- ▁OH
+- NE
+- Z
+- LING
+- ▁AGAINST
+- TER
+- ▁NAME
+- ▁MISS
+- ▁QUITE
+- ▁WANT
+- ▁YEARS
+- ▁FEW
+- ▁BETTER
+- ENT
+- ▁HALF
+- ▁DONE
+- ▁ALSO
+- ▁BEGAN
+- ▁HAVING
+- ▁ENOUGH
+- IS
+- ▁LADY
+- ▁WHOLE
+- LESS
+- ▁BOTH
+- ▁SEEN
+- ▁SET
+- ▁WHITE
+- ▁COURSE
+- IES
+- ▁VOICE
+- ▁CALLED
+- ▁D
+- ▁EX
+- ATE
+- ▁TURNED
+- ▁GAVE
+- ▁C
+- ▁POOR
+- MAN
+- UT
+- NA
+- ▁DEAR
+- ISH
+- ▁GIRL
+- ▁MORNING
+- ▁BETWEEN
+- LED
+- ▁NOR
+- IA
+- ▁AMONG
+- MA
+- ▁
+- ▁SMALL
+- ▁REST
+- ▁WHOM
+- ▁FELT
+- ▁HANDS
+- ▁MYSELF
+- ▁HIGH
+- ▁M
+- ▁HOWEVER
+- ▁HERSELF
+- ▁P
+- CO
+- ▁STOOD
+- ID
+- ▁KIND
+- ▁HUNDRED
+- AS
+- ▁ROUND
+- ▁ALMOST
+- TY
+- ▁SINCE
+- ▁G
+- AM
+- ▁LA
+- SE
+- ▁BOY
+- ▁MA
+- ▁PERHAPS
+- ▁WORDS
+- ATED
+- ▁HO
+- X
+- ▁MO
+- ▁SAT
+- ▁REPLIED
+- ▁FOUR
+- ▁ANYTHING
+- ▁TILL
+- ▁UNTIL
+- ▁BLACK
+- TION
+- ▁CRIED
+- RU
+- TE
+- ▁FACT
+- ▁HELP
+- ▁NEXT
+- ▁LOOKING
+- ▁DOES
+- ▁FRIEND
+- ▁LAY
+- ANCE
+- ▁POWER
+- ▁BROUGHT
+- VER
+- ▁FIRE
+- ▁KEEP
+- PO
+- FF
+- ▁COUNTRY
+- ▁SEA
+- ▁WORD
+- ▁CAR
+- ▁DAYS
+- ▁TOGETHER
+- ▁IMP
+- ▁REASON
+- KE
+- ▁INDEED
+- TING
+- ▁MATTER
+- ▁FULL
+- ▁TEN
+- TIC
+- ▁LAND
+- ▁RATHER
+- ▁AIR
+- ▁HOPE
+- ▁DA
+- ▁OPEN
+- ▁FEET
+- ▁EN
+- ▁FIVE
+- ▁POINT
+- ▁CO
+- OM
+- ▁LARGE
+- ▁B
+- ▁CL
+- ME
+- ▁GONE
+- ▁CHILD
+- INE
+- GG
+- ▁BEST
+- ▁DIS
+- UM
+- ▁HARD
+- ▁LORD
+- OUS
+- ▁WIFE
+- ▁SURE
+- ▁FORM
+- DE
+- ▁DEATH
+- ANT
+- ▁NATURE
+- ▁BA
+- ▁CARE
+- ▁BELIEVE
+- PP
+- ▁NEAR
+- ▁RO
+- ▁RED
+- ▁WAR
+- IE
+- ▁SPEAK
+- ▁FEAR
+- ▁CASE
+- ▁TAKEN
+- ▁ALONG
+- ▁CANNOT
+- ▁HEAR
+- ▁THEMSELVES
+- CI
+- ▁PRESENT
+- AD
+- ▁MASTER
+- ▁SON
+- ▁THUS
+- ▁LI
+- ▁LESS
+- ▁SUN
+- ▁TRUE
+- IM
+- IOUS
+- ▁THOUSAND
+- ▁MONEY
+- ▁W
+- ▁BEHIND
+- ▁CHILDREN
+- ▁DOCTOR
+- AC
+- ▁TWENTY
+- ▁WISH
+- ▁SOUND
+- ▁WHOSE
+- ▁LEAVE
+- ▁ANSWERED
+- ▁THOU
+- ▁DUR
+- ▁HA
+- ▁CERTAIN
+- ▁PO
+- ▁PASSED
+- GE
+- TO
+- ▁ARM
+- ▁LO
+- ▁STATE
+- ▁ALONE
+- TA
+- ▁SHOW
+- ▁NEED
+- ▁LIVE
+- ND
+- ▁DEAD
+- ENCE
+- ▁STRONG
+- ▁PRE
+- ▁TI
+- ▁GROUND
+- SH
+- TI
+- ▁SHORT
+- IAN
+- UN
+- ▁PRO
+- ▁HORSE
+- MI
+- ▁PRINCE
+- ARD
+- ▁FELL
+- ▁ORDER
+- ▁CALL
+- AT
+- ▁GIVEN
+- ▁DARK
+- ▁THEREFORE
+- ▁CLOSE
+- ▁BODY
+- ▁OTHERS
+- ▁SENT
+- ▁SECOND
+- ▁OFTEN
+- ▁CA
+- ▁MANNER
+- MO
+- NI
+- ▁BRING
+- ▁QUESTION
+- ▁HOUR
+- ▁BO
+- AGE
+- ▁ST
+- ▁TURN
+- ▁TABLE
+- ▁GENERAL
+- ▁EARTH
+- ▁BED
+- ▁REALLY
+- ▁SIX
+- 'NO'
+- IST
+- ▁BECOME
+- ▁USE
+- ▁READ
+- ▁SE
+- ▁VI
+- ▁COMING
+- ▁EVERYTHING
+- ▁EM
+- ▁ABOVE
+- ▁EVENING
+- ▁BEAUTIFUL
+- ▁FEEL
+- ▁RAN
+- ▁LEAST
+- ▁LAW
+- ▁ALREADY
+- ▁MEAN
+- ▁ROSE
+- WARD
+- ▁ITSELF
+- ▁SOUL
+- ▁SUDDENLY
+- ▁AROUND
+- RED
+- ▁ANSWER
+- ICAL
+- ▁RA
+- ▁WIND
+- ▁FINE
+- ▁WON
+- ▁WHETHER
+- ▁KNOWN
+- BER
+- NG
+- ▁TA
+- ▁CAPTAIN
+- ▁EYE
+- ▁PERSON
+- ▁WOMEN
+- ▁SORT
+- ▁ASK
+- ▁BROTHER
+- ▁USED
+- ▁HELD
+- ▁BIG
+- ▁RETURNED
+- ▁STRANGE
+- ▁BU
+- ▁PER
+- ▁FREE
+- ▁EITHER
+- ▁WITHIN
+- ▁DOUBT
+- ▁YEAR
+- ▁CLEAR
+- ▁SIGHT
+- ▁GRA
+- ▁LOST
+- ▁KEPT
+- ▁F
+- PE
+- ▁BAR
+- ▁TOWN
+- ▁SLEEP
+- ARY
+- ▁HAIR
+- ▁FRIENDS
+- ▁DREAM
+- ▁FELLOW
+- PER
+- ▁DEEP
+- QUE
+- ▁BECAME
+- ▁REAL
+- ▁PAST
+- ▁MAKING
+- RING
+- ▁COMP
+- ▁ACT
+- ▁BAD
+- HO
+- STER
+- ▁YE
+- ▁MEANS
+- ▁RUN
+- MEN
+- ▁DAUGHTER
+- ▁SENSE
+- ▁CITY
+- ▁SOMETIMES
+- ▁TOWARDS
+- ▁ROAD
+- ▁SP
+- ▁LU
+- ▁READY
+- ▁FOOT
+- ▁COLD
+- ▁SA
+- ▁LETTER
+- ▁ELSE
+- ▁MAR
+- ▁STA
+- BE
+- ▁TRUTH
+- ▁LE
+- BO
+- ▁BUSINESS
+- CHE
+- ▁JOHN
+- ▁SUBJECT
+- ▁COURT
+- ▁IDEA
+- ILY
+- ▁RIVER
+- ATING
+- ▁FAMILY
+- HE
+- ▁DIDN
+- ▁GLAD
+- ▁SEVERAL
+- IAL
+- ▁UNDERSTAND
+- ▁SC
+- ▁POSSIBLE
+- ▁DIFFERENT
+- ▁RETURN
+- ▁ARMS
+- ▁LOW
+- ▁HOLD
+- ▁TALK
+- ▁RU
+- ▁WINDOW
+- ▁INTEREST
+- ▁SISTER
+- SON
+- ▁SH
+- ▁BLOOD
+- ▁SAYS
+- ▁CAP
+- ▁DI
+- ▁HUMAN
+- ▁CAUSE
+- NCE
+- ▁THANK
+- ▁LATE
+- GO
+- ▁CUT
+- ▁ACROSS
+- ▁STORY
+- NT
+- ▁COUNT
+- ▁ABLE
+- DY
+- LEY
+- ▁NUMBER
+- ▁STAND
+- ▁CHURCH
+- ▁THY
+- ▁SUPPOSE
+- LES
+- BLE
+- OP
+- ▁EFFECT
+- BY
+- ▁K
+- ▁NA
+- ▁SPOKE
+- ▁MET
+- ▁GREEN
+- ▁HUSBAND
+- ▁RESPECT
+- ▁PA
+- ▁FOLLOWED
+- ▁REMEMBER
+- ▁LONGER
+- ▁AGE
+- ▁TAKING
+- ▁LINE
+- ▁SEEM
+- ▁HAPPY
+- LAND
+- EM
+- ▁STAY
+- ▁PLAY
+- ▁COMMON
+- ▁GA
+- ▁BOOK
+- ▁TIMES
+- ▁OBJECT
+- ▁SEVEN
+- QUI
+- DO
+- UND
+- ▁FL
+- ▁PRETTY
+- ▁FAIR
+- WAY
+- ▁WOOD
+- ▁REACHED
+- ▁APPEARED
+- ▁SWEET
+- ▁FALL
+- BA
+- ▁PASS
+- ▁SIGN
+- ▁TREE
+- IONS
+- ▁GARDEN
+- ▁ILL
+- ▁ART
+- ▁REMAIN
+- ▁OPENED
+- ▁BRIGHT
+- ▁STREET
+- ▁TROUBLE
+- ▁PAIN
+- ▁CONTINUED
+- ▁SCHOOL
+- OUR
+- ▁CARRIED
+- ▁SAYING
+- HA
+- ▁CHANGE
+- ▁FOLLOW
+- ▁GOLD
+- ▁SW
+- ▁FEELING
+- ▁COMMAND
+- ▁BEAR
+- ▁CERTAINLY
+- ▁BLUE
+- ▁NE
+- CA
+- ▁WILD
+- ▁ACCOUNT
+- ▁OUGHT
+- UD
+- ▁T
+- ▁BREATH
+- ▁WANTED
+- ▁RI
+- ▁HEAVEN
+- ▁PURPOSE
+- ▁CHARACTER
+- ▁RICH
+- ▁PE
+- ▁DRESS
+- OS
+- FA
+- ▁TH
+- ▁ENGLISH
+- ▁CHANCE
+- ▁SHIP
+- ▁VIEW
+- ▁TOWARD
+- AK
+- ▁JOY
+- ▁JA
+- ▁HAR
+- ▁NEITHER
+- ▁FORCE
+- ▁UNCLE
+- DER
+- ▁PLAN
+- ▁PRINCESS
+- DI
+- ▁CHIEF
+- ▁HAT
+- ▁LIVED
+- ▁AB
+- ▁VISIT
+- ▁MOR
+- TEN
+- ▁WALL
+- UC
+- ▁MINE
+- ▁PLEASURE
+- ▁SMILE
+- ▁FRONT
+- ▁HU
+- ▁DEAL
+- OW
+- ▁FURTHER
+- GED
+- ▁TRIED
+- DA
+- VA
+- ▁NONE
+- ▁ENTERED
+- ▁QUEEN
+- ▁PAY
+- ▁EL
+- ▁EXCEPT
+- ▁SHA
+- ▁FORWARD
+- ▁EIGHT
+- ▁ADDED
+- ▁PUBLIC
+- ▁EIGHTEEN
+- ▁STAR
+- ▁HAPPENED
+- ▁LED
+- ▁WALKED
+- ▁ALTHOUGH
+- ▁LATER
+- ▁SPIRIT
+- ▁WALK
+- ▁BIT
+- ▁MEET
+- LIN
+- ▁FI
+- LT
+- ▁MOUTH
+- ▁WAIT
+- ▁HOURS
+- ▁LIVING
+- ▁YOURSELF
+- ▁FAST
+- ▁CHA
+- ▁HALL
+- ▁BEYOND
+- ▁BOAT
+- ▁SECRET
+- ENS
+- ▁CHAIR
+- RN
+- ▁RECEIVED
+- ▁CAT
+- RESS
+- ▁DESIRE
+- ▁GENTLEMAN
+- UGH
+- ▁LAID
+- EVER
+- ▁OCCASION
+- ▁WONDER
+- ▁GU
+- ▁PARTY
+- DEN
+- ▁FISH
+- ▁SEND
+- ▁NEARLY
+- ▁TRY
+- CON
+- ▁SEEMS
+- RS
+- ▁BELL
+- ▁BRA
+- ▁SILENCE
+- IG
+- ▁GUARD
+- ▁DIE
+- ▁DOING
+- ▁TU
+- ▁COR
+- ▁EARLY
+- ▁BANK
+- ▁FIGURE
+- IF
+- ▁ENGLAND
+- ▁MARY
+- ▁AFRAID
+- LER
+- ▁FO
+- ▁WATCH
+- ▁FA
+- ▁VA
+- ▁GRE
+- ▁AUNT
+- PED
+- ▁SERVICE
+- ▁JE
+- ▁PEN
+- ▁MINUTES
+- ▁PAN
+- ▁TREES
+- NED
+- ▁GLASS
+- ▁TONE
+- ▁PLEASE
+- ▁FORTH
+- ▁CROSS
+- ▁EXCLAIMED
+- ▁DREW
+- ▁EAT
+- ▁AH
+- ▁GRAVE
+- ▁CUR
+- PA
+- URE
+- CENT
+- ▁MILES
+- ▁SOFT
+- ▁AGO
+- ▁POSITION
+- ▁WARM
+- ▁LENGTH
+- ▁NECESSARY
+- ▁THINKING
+- ▁PICTURE
+- ▁PI
+- SHIP
+- IBLE
+- ▁HEAVY
+- ▁ATTENTION
+- ▁DOG
+- ABLY
+- ▁STANDING
+- ▁NATURAL
+- ▁APPEAR
+- OV
+- ▁CAUGHT
+- VO
+- ISM
+- ▁SPRING
+- ▁EXPERIENCE
+- ▁PAT
+- OT
+- ▁STOPPED
+- ▁REGARD
+- ▁HARDLY
+- ▁SELF
+- ▁STRENGTH
+- ▁GREW
+- ▁KNIGHT
+- ▁OPINION
+- ▁WIDE
+- ▁INSTEAD
+- ▁SOUTH
+- ▁TRANS
+- ▁CORNER
+- ▁LEARN
+- ▁ISLAND
+- ▁MI
+- ▁THIRD
+- ▁STE
+- ▁STRAIGHT
+- ▁TEA
+- ▁BOUND
+- ▁SEEING
+- ▁JU
+- ▁DINNER
+- ▁BEAUTY
+- ▁PEACE
+- AH
+- ▁REP
+- ▁SILENT
+- ▁CRE
+- ALLY
+- RIC
+- ▁STEP
+- ▁VER
+- ▁JO
+- GER
+- ▁SITTING
+- ▁THIRTY
+- ▁SAVE
+- ENED
+- ▁GLANCE
+- ▁REACH
+- ▁ACTION
+- ▁SAL
+- ▁SAD
+- ▁STONE
+- ITIES
+- ▁FRENCH
+- ▁STRUCK
+- ▁PAPER
+- ▁WHATEVER
+- ▁SUB
+- ▁DISTANCE
+- ▁WRONG
+- ▁KNOWLEDGE
+- ▁SAFE
+- ▁SNOW
+- ▁MUSIC
+- ▁FIFTY
+- RON
+- ▁ATTEMPT
+- ▁GOVERNMENT
+- TU
+- ▁CROWD
+- ▁BESIDES
+- ▁LOVED
+- ▁BOX
+- ▁DIRECTION
+- ▁TRAIN
+- ▁NORTH
+- ▁THICK
+- ▁GETTING
+- AV
+- ▁FLOOR
+- ▁COMPANY
+- ▁BLOW
+- ▁PLAIN
+- TRO
+- ▁BESIDE
+- ▁ROCK
+- ▁IMMEDIATELY
+- FI
+- ▁SHADOW
+- ▁SIT
+- ORS
+- ILE
+- ▁DRINK
+- ▁SPOT
+- ▁DANGER
+- ▁AL
+- ▁SAINT
+- ▁SLOWLY
+- ▁PALACE
+- IER
+- ▁RESULT
+- ▁PETER
+- ▁FOREST
+- ▁BELONG
+- ▁SU
+- ▁PAR
+- RIS
+- ▁TEARS
+- ▁APPEARANCE
+- ▁GATE
+- BU
+- ITION
+- ▁QUICKLY
+- ▁QUIET
+- ▁LONDON
+- ▁START
+- ▁BROWN
+- TRA
+- KIN
+- ▁CONSIDER
+- ▁BATTLE
+- ▁ANNE
+- ▁PIECE
+- ▁DIED
+- ▁SUCCESS
+- ▁LIPS
+- ▁FILLED
+- ▁FORGET
+- ▁POST
+- IFIED
+- ▁MARGARET
+- ▁FOOD
+- HAM
+- ▁PLEASANT
+- ▁FE
+- ▁EXPRESSION
+- ▁POCKET
+- ▁FRESH
+- ▁WEAR
+- TRI
+- ▁BROKEN
+- ▁LAUGHED
+- GING
+- ▁FOLLOWING
+- WN
+- IP
+- ▁TOUCH
+- ▁YOUTH
+- ATIVE
+- ▁LEG
+- ▁WEEK
+- ▁REMAINED
+- ▁EASY
+- NER
+- RK
+- ▁ENTER
+- ▁FIGHT
+- ▁PLACED
+- ▁TRAVEL
+- ▁SIMPLE
+- ▁GIRLS
+- ▁WAITING
+- ▁STOP
+- ▁WAVE
+- AU
+- ▁WISE
+- ▁CAMP
+- TURE
+- UB
+- ▁VE
+- ▁OFFICE
+- ▁GRAND
+- ▁FIT
+- ▁JUDGE
+- UP
+- MENTS
+- ▁QUICK
+- HI
+- ▁FLO
+- RIES
+- VAL
+- ▁COMFORT
+- ▁PARTICULAR
+- ▁STARTED
+- ▁SUIT
+- ▁NI
+- ▁PALE
+- ▁IMPOSSIBLE
+- ▁HOT
+- ▁CONVERSATION
+- ▁SCENE
+- ▁BOYS
+- ▁WIN
+- ▁BRE
+- ▁SOCIETY
+- ▁OUTSIDE
+- ▁WRITE
+- ▁EFFORT
+- ▁TALKING
+- ▁FORTUNE
+- ▁NINE
+- ▁WA
+- ▁SINGLE
+- ▁RULE
+- ▁PORT
+- ▁WINTER
+- ▁CAST
+- ▁CRA
+- ▁HAPPEN
+- ▁CRO
+- ▁SHUT
+- NING
+- ▁GUN
+- ▁NOBLE
+- ▁BEGIN
+- ▁PATH
+- ▁SKY
+- ▁WONDERFUL
+- ���SUDDEN
+- ▁ARMY
+- ▁CHE
+- ▁WORTH
+- ▁MOUNTAIN
+- ▁MIN
+- AG
+- ▁FLU
+- ▁GRACE
+- ▁CHAPTER
+- ▁BELOW
+- ▁RING
+- ▁TURNING
+- ▁IRON
+- ▁TOP
+- ▁AFTERNOON
+- ORY
+- ▁EVIL
+- ▁TRUST
+- ▁BOW
+- ▁TRI
+- ▁SAIL
+- ▁CONTENT
+- ▁HORSES
+- ITE
+- ▁SILVER
+- AP
+- ▁LAD
+- ▁RUNNING
+- ▁HILL
+- ▁BEGINNING
+- ▁MAD
+- ▁HABIT
+- GRA
+- ▁CLOTHES
+- ▁MORROW
+- ▁CRY
+- ▁FASHION
+- ▁PRESENCE
+- ▁Z
+- FE
+- ▁ARRIVED
+- ▁QUARTER
+- ▁PERFECT
+- ▁WO
+- ▁TRA
+- ▁USUAL
+- ▁NECK
+- ▁MARRIED
+- ▁SEAT
+- ▁WI
+- ▁GAR
+- ▁SAND
+- ▁SHORE
+- ▁GIVING
+- NY
+- ▁PROBABLY
+- ▁MINUTE
+- ▁EXPECT
+- ▁DU
+- ▁SHOT
+- ▁INSTANT
+- ▁DEGREE
+- ▁COLOR
+- ▁WEST
+- RT
+- ▁MARCH
+- ▁BIRD
+- ▁SHOWED
+- ▁GREATER
+- ▁SERIOUS
+- ▁CARRY
+- ▁COVERED
+- ▁FORMER
+- ▁LOUD
+- ▁MOVED
+- ▁MASS
+- ▁SEEK
+- ▁CHO
+- GEN
+- ▁ROMAN
+- IB
+- ▁MOON
+- ▁BOARD
+- ▁STREAM
+- ▁EASILY
+- ▁WISHED
+- ▁SEARCH
+- ▁COULDN
+- ▁MONTHS
+- ▁SICK
+- LIE
+- ▁DUTY
+- ▁TWELVE
+- ▁FAINT
+- ▁STRANGER
+- ▁SURPRISE
+- ▁KILL
+- ▁LEAVING
+- ▁JOURNEY
+- ▁SCARCELY
+- ▁RAISED
+- ▁SPEAKING
+- ▁TERRIBLE
+- ▁TOM
+- ▁FIELD
+- ▁GAME
+- ▁QUA
+- ▁PROMISE
+- ▁LIE
+- ▁CONDITION
+- ▁TRO
+- ▁PERSONAL
+- ▁TALL
+- ▁STICK
+- ▁THREW
+- ▁MARRY
+- ▁VAN
+- ▁BURN
+- ▁ACCORDING
+- ▁RISE
+- ▁ATTACK
+- ▁SWORD
+- ▁GUESS
+- ▁THOUGHTS
+- ▁THIN
+- ▁THROW
+- ▁CALM
+- SIDE
+- ▁VILLAGE
+- ▁DEN
+- ▁ANXIOUS
+- ▁MER
+- GI
+- ▁EXPECTED
+- ▁BALL
+- ▁ESPECIALLY
+- ▁CHARGE
+- ▁MEASURE
+- ISE
+- ▁NICE
+- ▁TRYING
+- ▁ALLOW
+- ▁SHARP
+- ▁BREAD
+- ▁HONOUR
+- ▁HONOR
+- ▁ENTIRELY
+- ▁BILL
+- ▁BRI
+- ▁WRITTEN
+- ▁AR
+- ▁BROKE
+- ▁KILLED
+- ▁MARK
+- ▁VEN
+- ▁LADIES
+- ▁LEARNED
+- ▁FLOWERS
+- PLE
+- ▁FORTY
+- ▁OFFER
+- ▁HAPPINESS
+- ▁PRAY
+- ▁CLASS
+- ▁FER
+- ▁PRINCIPLE
+- GU
+- ▁BOOKS
+- ▁SHAPE
+- ▁SUMMER
+- ▁JACK
+- ▁DRAW
+- ▁GOLDEN
+- ▁DECIDED
+- ▁LEAD
+- ▁UNLESS
+- ▁HARM
+- ▁LISTEN
+- HER
+- ▁SHOOK
+- ▁INFLUENCE
+- ▁PERFECTLY
+- ▁MARRIAGE
+- ▁BROAD
+- ▁ESCAPE
+- ▁STATES
+- ▁MIDDLE
+- ▁PLANT
+- ▁MIL
+- ▁MOVEMENT
+- ▁NOISE
+- ▁ENEMY
+- ▁HISTORY
+- ▁BREAK
+- ROUS
+- ▁UNDERSTOOD
+- ▁LATTER
+- FER
+- ▁COMES
+- ▁MERELY
+- ▁SIMPLY
+- WI
+- ▁IMAGINE
+- ▁LOWER
+- ▁CONDUCT
+- ▁BORN
+- WA
+- ▁YARD
+- ▁KA
+- ▁CLOSED
+- ▁NOTE
+- GA
+- ▁STRA
+- RAN
+- ▁EXIST
+- EV
+- ▁SPEECH
+- ▁BITTER
+- JO
+- ▁MAKES
+- ▁GRASS
+- ▁REPLY
+- ▁CHANGED
+- ▁MON
+- ▁LYING
+- ▁DANCE
+- ▁FINALLY
+- ▁AMERICAN
+- ▁ENJOY
+- ▁CONTAIN
+- ▁MEANT
+- USE
+- ▁OBSERVED
+- THER
+- ▁LAUGH
+- ▁AFTERWARDS
+- ▁BEAT
+- ▁RACE
+- ▁EQUAL
+- ▁RAIN
+- PS
+- ▁STEPS
+- ▁BENEATH
+- ▁TAIL
+- ▁TASTE
+- IO
+- EY
+- ▁CHAR
+- ▁GE
+- GN
+- TIN
+- ▁GROW
+- ▁TE
+- IANS
+- ▁MOVE
+- ▁REPEATED
+- ▁DRIVE
+- TUR
+- ▁SI
+- CLOCK
+- ▁BRAVE
+- ▁MADAME
+- ▁LOT
+- ▁CASTLE
+- ▁HI
+- AND
+- ▁FUTURE
+- ▁RELATION
+- ▁SORRY
+- ▁HEALTH
+- ▁DICK
+- ▁R
+- ▁BUILDING
+- ▁EDGE
+- ▁BLESS
+- ▁SPITE
+- WE
+- ▁MIS
+- ▁PRISONER
+- ▁ALLOWED
+- ▁PH
+- ▁CATCH
+- MER
+- ETH
+- ▁COAT
+- ▁COMPLETE
+- ▁WOULDN
+- ▁CREATURE
+- ▁YELLOW
+- ▁IMPORTANT
+- ▁ADD
+- ▁PASSING
+- ▁DARKNESS
+- ▁CARRIAGE
+- ▁MILL
+- ▁FIFTEEN
+- NCY
+- ▁HUNG
+- ▁OB
+- ▁PLEASED
+- ▁SPREAD
+- ▁CURIOUS
+- ▁WORSE
+- ▁CIRCUMSTANCES
+- ▁GI
+- LAR
+- ▁CAL
+- ▁HY
+- ▁MERE
+- ▁JANE
+- ▁EAST
+- BI
+- ▁CUP
+- ▁BLIND
+- ▁PASSION
+- ▁DISCOVERED
+- ▁NOTICE
+- ▁REPORT
+- ▁SPACE
+- ▁PRESENTLY
+- ▁SORROW
+- ▁PACK
+- ▁DIN
+- CY
+- ▁DRY
+- ▁ANCIENT
+- ▁DRESSED
+- ▁COVER
+- ▁VO
+- ▁EXISTENCE
+- ▁EXACTLY
+- ▁BEAST
+- ▁PROPER
+- ▁DROPPED
+- ▁CLEAN
+- ▁COLOUR
+- ▁HOST
+- ▁CHAMBER
+- ▁FAITH
+- LET
+- ▁DETERMINED
+- ▁PRIEST
+- ▁STORM
+- ▁SKIN
+- ▁DARE
+- ▁PERSONS
+- ▁PICK
+- ▁NARROW
+- ▁SUPPORT
+- ▁PRIVATE
+- ▁SMILED
+- ▁COUSIN
+- ▁DRAWING
+- ▁ATTEND
+- ▁COOK
+- ▁PREVENT
+- ▁VARIOUS
+- ▁BLA
+- ▁FIXED
+- ▁WEAK
+- THE
+- ▁HOLE
+- ▁BOTTOM
+- ▁NOBODY
+- ADE
+- ▁LEGS
+- ITCH
+- ▁INDIVIDUAL
+- ▁EARS
+- LIKE
+- ▁ADVANTAGE
+- ▁FRANCE
+- ▁BON
+- ▁WINE
+- ▁LIVES
+- OD
+- ▁WALLS
+- ▁TIRED
+- ▁SHOP
+- ▁ANIMAL
+- ▁CRU
+- ▁WROTE
+- ▁ROYAL
+- ▁CONSIDERED
+- ▁MORAL
+- ▁COMPANION
+- ▁LOSE
+- ▁ISN
+- ▁BAG
+- ▁LAKE
+- ▁INTER
+- ▁COM
+- ▁LETTERS
+- ▁LUCK
+- ▁EAR
+- ▁GERMAN
+- ▁PET
+- ▁SAKE
+- ▁DROP
+- ▁PAID
+- ▁BREAKFAST
+- ▁LABOR
+- ▁DESERT
+- ▁DECLARED
+- ▁HUM
+- ▁STUDY
+- ▁INSTANCE
+- ONE
+- ▁SOMEWHAT
+- ▁CLOTH
+- ▁SPECIAL
+- ▁COLONEL
+- ▁SONG
+- ▁MAIN
+- ▁VALUE
+- ▁PROUD
+- ▁EXPRESS
+- ▁NATION
+- ▁HANDSOME
+- ▁CONFESS
+- ▁PU
+- ▁PASSAGE
+- ▁PERIOD
+- ▁CUSTOM
+- ▁HURT
+- ▁SHOULDER
+- ▁CHRIST
+- ZA
+- ▁RECEIVE
+- ▁DIFFICULT
+- ▁DEPEND
+- ▁MEETING
+- ▁CHI
+- ▁GEN
+- LIGHT
+- ▁BELIEVED
+- ▁SOCIAL
+- ▁DIFFICULTY
+- ▁GREATEST
+- ▁DRAWN
+- ▁GRANT
+- ▁BIRDS
+- ▁ANGRY
+- ▁HEAT
+- UFF
+- ▁DUE
+- ▁PLACES
+- ▁SIN
+- ▁COURAGE
+- ▁EVIDENTLY
+- ▁GENTLE
+- ▁CRUEL
+- ▁GEORGE
+- ▁GRI
+- ▁SERVANT
+- ▁U
+- ▁PURE
+- OOK
+- ▁KNOWS
+- ▁KNOWING
+- LF
+- ▁WRITING
+- ▁REMEMBERED
+- ▁CU
+- ▁HOLDING
+- ▁TENDER
+- ▁QUI
+- ▁BURST
+- ▁SURELY
+- IGN
+- ▁VALLEY
+- ▁FU
+- ▁BUTTER
+- ▁SPOKEN
+- ▁STORE
+- ▁DISC
+- ▁CHRISTIAN
+- ▁PARIS
+- ▁HENRY
+- ▁FINISHED
+- ▁PROVE
+- ▁FOOL
+- ▁SOLDIERS
+- ▁LANGUAGE
+- ▁INSIDE
+- ▁BAN
+- ▁FALLEN
+- ROW
+- ▁MAL
+- ▁BABY
+- ▁SITUATION
+- ▁WATCHED
+- ANS
+- ▁RUIN
+- ▁GENTLEMEN
+- ▁FRO
+- ▁FANCY
+- ▁ACCEPT
+- ▁SEASON
+- ▁OURSELVES
+- ▁SAN
+- ▁SPEED
+- IZED
+- ▁COOL
+- ▁SERVE
+- ▁VESSEL
+- ▁WILLIAM
+- ▁OBLIGED
+- ▁GROUP
+- FORM
+- ▁GOES
+- UOUS
+- ▁LEAVES
+- ▁PECULIAR
+- ▁NEWS
+- ▁VAIN
+- ▁EVERYBODY
+- ▁PIN
+- UG
+- ▁FORGOTTEN
+- ▁FRA
+- GAN
+- ▁CAREFULLY
+- ▁FLASH
+- UCH
+- ▁FUR
+- ▁MURDER
+- ▁DELIGHT
+- ▁WAITED
+- ▁RENDER
+- ▁PROPERTY
+- ▁NOTICED
+- ▁ROLL
+- ▁KNOCK
+- ▁EARNEST
+- KI
+- ▁HONEST
+- ▁PROMISED
+- ▁BAL
+- AW
+- ▁WALKING
+- ANG
+- ▁SQUARE
+- ▁QUIETLY
+- ▁CLOUD
+- WOOD
+- ▁FORMED
+- ▁HIGHER
+- ▁BUILT
+- ▁FATE
+- ▁TEACH
+- MY
+- ▁FALSE
+- ▁YORK
+- ▁DUST
+- ▁CLIMB
+- ▁FOND
+- ▁GROWN
+- ▁DESCEND
+- ▁RAG
+- ▁FRUIT
+- ▁GENERALLY
+- ▁OFFERED
+- ▁ER
+- ▁NURSE
+- POSE
+- ▁SPENT
+- ▁JOIN
+- ▁STATION
+- ▁MEANING
+- ▁SMOKE
+- HOOD
+- ▁ROUGH
+- JU
+- ▁LIKELY
+- ▁SURFACE
+- ▁KE
+- ▁MONTH
+- ▁POSSESSION
+- ▁TONGUE
+- ▁DUKE
+- ▁NOSE
+- ▁LAUGHING
+- ▁WEATHER
+- ▁WHISPERED
+- ▁SYSTEM
+- ▁LAWS
+- DDLE
+- ▁TOUCHED
+- ▁TRADE
+- LD
+- ▁SURPRISED
+- RIN
+- ▁ARCH
+- ▁WEALTH
+- FOR
+- ▁TEMPER
+- ▁FRANK
+- ▁GAL
+- ▁BARE
+- ▁OPPORTUNITY
+- ▁CLAIM
+- ▁ANIMALS
+- ▁REV
+- ▁COST
+- ▁WASH
+- ZE
+- ▁CORN
+- ▁OPPOSITE
+- ▁POLICE
+- ▁IDEAS
+- LON
+- ▁KEY
+- ▁READING
+- ▁COLLECT
+- CHED
+- ▁H
+- ▁CROWN
+- ▁TAR
+- ▁SWIFT
+- ▁SHOULDERS
+- ▁ICE
+- ▁GRAY
+- ▁SHARE
+- ▁PREPARED
+- ▁GRO
+- ▁UND
+- ▁TER
+- ▁EMPTY
+- CING
+- ▁SMILING
+- ▁AVOID
+- ▁DIFFERENCE
+- ▁EXPLAIN
+- ▁POUR
+- ▁ATTRACT
+- ▁OPENING
+- ▁WHEEL
+- ▁MATERIAL
+- ▁BREAST
+- ▁SUFFERING
+- ▁DISTINCT
+- ▁BOOT
+- ▁ROW
+- ▁FINGERS
+- HAN
+- ▁ALTOGETHER
+- ▁FAT
+- ▁PAPA
+- ▁BRAIN
+- ▁ASLEEP
+- ▁GREY
+- ▁SUM
+- ▁GAS
+- ▁WINDOWS
+- ▁ALIVE
+- ▁PROCEED
+- ▁FLOWER
+- ▁LEAP
+- ▁PUR
+- ▁PIECES
+- ▁ALTER
+- ▁MEMORY
+- IENT
+- ▁FILL
+- ▁CLO
+- ▁THROWN
+- ▁KINGDOM
+- ▁RODE
+- IUS
+- ▁MAID
+- ▁DIM
+- ▁BAND
+- ▁VIRTUE
+- ▁DISH
+- ▁GUEST
+- ▁LOSS
+- ▁CAUSED
+- ▁MOTION
+- ▁POT
+- ▁MILLION
+- ▁FAULT
+- ▁LOVELY
+- ▁HERO
+- PPING
+- ▁UNITED
+- ▁SPI
+- SOME
+- BRA
+- ▁MOUNTAINS
+- ▁NU
+- ▁SATISFIED
+- ▁DOLLARS
+- ▁LOVER
+- ▁CONCEAL
+- ▁VAST
+- ▁PULL
+- ▁HATH
+- ▁RUSH
+- ▁J
+- ▁DESPAIR
+- EX
+- ▁HEIGHT
+- ▁CE
+- ▁BENT
+- ▁PITY
+- ▁RISING
+- ATH
+- ▁PRIDE
+- ▁HURRY
+- KA
+- ▁SETTLED
+- ▁JUSTICE
+- ▁LIFTED
+- PEN
+- ▁SOLDIER
+- ▁FINDING
+- ▁REMARK
+- ▁REGULAR
+- ▁STRUGGLE
+- ▁MACHINE
+- ▁SING
+- ▁HURRIED
+- ▁SUFFICIENT
+- ▁REPRESENT
+- ▁DOUBLE
+- ▁ALARM
+- ▁SUPPER
+- ▁DREADFUL
+- ▁FORE
+- ATOR
+- ▁STOCK
+- ▁TIN
+- ▁EXAMPLE
+- ▁ROOF
+- ▁FLOW
+- ▁SUPPOSED
+- ▁PRESERV
+- ▁L
+- ▁LISTENED
+- OC
+- ▁STO
+- ▁SECURE
+- ▁FRIGHTENED
+- ▁DISTURB
+- ▁EMOTION
+- ▁SERVANTS
+- ▁YO
+- ▁BUY
+- ▁FORCED
+- ▁KITCHEN
+- ▁TERROR
+- ▁STAIRS
+- ▁SIXTY
+- KER
+- ▁ORDINARY
+- ▁DIRECTLY
+- ▁HEADS
+- ▁METHOD
+- ▁FORGIVE
+- ▁AWFUL
+- ▁REFLECT
+- ▁GREATLY
+- ▁TALKED
+- ▁RIDE
+- STONE
+- ▁FAVOUR
+- ▁WELCOME
+- ▁SEIZED
+- OU
+- ▁CONTROL
+- ▁ORDERED
+- ▁ANGEL
+- ▁USUALLY
+- ▁POET
+- ▁BOLD
+- LINE
+- ▁ADVENTURE
+- ▁WATCHING
+- ▁FOLK
+- ▁MISTRESS
+- IZE
+- ▁GROWING
+- ▁CAVE
+- ▁EVIDENCE
+- ▁FINGER
+- ▁SEVENTEEN
+- ▁MOVING
+- EOUS
+- ▁DOESN
+- ▁COW
+- ▁TYPE
+- ▁BOIL
+- ▁TALE
+- ▁DELIVER
+- ▁FARM
+- ▁MONSIEUR
+- ▁GATHERED
+- ▁FEELINGS
+- ▁RATE
+- ▁REMARKED
+- ▁PUTTING
+- ▁MAT
+- ▁CONTRARY
+- ▁CRIME
+- ▁PLA
+- ▁COL
+- ▁NEARER
+- TES
+- ▁CIVIL
+- ▁SHAME
+- ▁LOOSE
+- ▁DISCOVER
+- ▁FLAT
+- ▁TWICE
+- ▁FAIL
+- VIS
+- ▁UNC
+- EA
+- ▁EUROPE
+- ▁PATIENT
+- ▁UNTO
+- ▁SUFFER
+- ▁PAIR
+- ▁TREASURE
+- OSE
+- ▁EAGER
+- ▁FLY
+- ▁N
+- ▁VAL
+- ▁DAN
+- ▁SALT
+- ▁BORE
+- BBE
+- ▁ARTHUR
+- ▁AFFAIRS
+- ▁SLOW
+- ▁CONSIST
+- ▁DEVIL
+- LAN
+- ▁AFFECTION
+- ▁ENGAGED
+- ▁KISS
+- ▁YA
+- ▁OFFICER
+- IFICATION
+- ▁LAMP
+- ▁PARTS
+- HEN
+- ▁MILK
+- ▁PROCESS
+- ▁GIFT
+- ▁PULLED
+- ▁HID
+- ▁RAY
+- ▁EXCELLENT
+- ▁IMPRESSION
+- ▁AUTHORITY
+- ▁PROVED
+- ▁TELLING
+- TTE
+- ▁TOWER
+- ▁CONSEQUENCE
+- ▁FAVOR
+- ▁FLEW
+- ▁CHARLES
+- ISTS
+- ▁ADDRESS
+- ▁FAMILIAR
+- ▁LIMIT
+- ▁CONFIDENCE
+- ▁RARE
+- ▁WEEKS
+- ▁WOODS
+- ▁INTENTION
+- ▁DIRECT
+- ▁PERFORM
+- ▁SOLEMN
+- ▁DISTANT
+- ▁IMAGE
+- ▁PRESIDENT
+- ▁FIRM
+- ▁INDIAN
+- ▁RANK
+- ▁LIKED
+- ▁AGREE
+- ▁HOUSES
+- ▁WIL
+- ▁MATTERS
+- ▁PRISON
+- ▁MODE
+- ▁MAJOR
+- ▁WORKING
+- ▁SLIP
+- ▁WEIGHT
+- ▁AWARE
+- ▁BUSY
+- ▁LOOKS
+- ▁WOUND
+- ▁THOR
+- ▁BATH
+- ▁EXERCISE
+- ▁SIMILAR
+- ▁WORE
+- ▁AMOUNT
+- ▁QUESTIONS
+- ▁VIOLENT
+- ▁EXCUSE
+- ▁ASIDE
+- ▁TUR
+- ▁DULL
+- OF
+- ▁EMPEROR
+- ▁NEVERTHELESS
+- ▁SHOUT
+- ▁EXPLAINED
+- ▁SIZE
+- ▁ACCOMPLISH
+- FORD
+- CAN
+- ▁MISTAKE
+- ▁INSTANTLY
+- ▁SMOOTH
+- ▁STRIKE
+- ▁BOB
+- ISED
+- ▁HORROR
+- ▁SCIENCE
+- ▁PROTEST
+- ▁MANAGE
+- ▁OBEY
+- ▁NECESSITY
+- ▁SPLENDID
+- ▁PRESS
+- ▁INTERESTING
+- ▁RELIGION
+- ▁UNKNOWN
+- ▁FIERCE
+- ▁DISAPPEARED
+- ▁HOLY
+- ▁HATE
+- ▁PLAYED
+- ▁LIN
+- ▁NATURALLY
+- ▁DROVE
+- ▁LOUIS
+- TIES
+- ▁BRAND
+- INESS
+- RIE
+- ▁SHOOT
+- ▁CONSENT
+- ▁SEATED
+- ▁LINES
+- GUE
+- ▁AGREED
+- ▁CIRCLE
+- ▁STIR
+- ▁STREETS
+- ▁TASK
+- ▁RID
+- ▁PRODUCED
+- ▁ACCIDENT
+- ▁WITNESS
+- ▁LIBERTY
+- ▁DETAIL
+- ▁MINISTER
+- ▁POWERFUL
+- ▁SAVAGE
+- ▁SIXTEEN
+- ▁PRETEND
+- ▁COAST
+- ▁SQU
+- ▁UTTER
+- ▁NAMED
+- ▁CLEVER
+- ▁ADMIT
+- ▁COUPLE
+- ▁WICKED
+- ▁MESSAGE
+- ▁TEMPLE
+- ▁STONES
+- ▁YESTERDAY
+- ▁HILLS
+- DAY
+- ▁SLIGHT
+- ▁DIAMOND
+- ▁POSSIBLY
+- ▁AFFAIR
+- ▁ORIGINAL
+- ▁HEARING
+- ▁WORTHY
+- ▁SELL
+- NEY
+- ICK
+- ▁COTTAGE
+- ▁SACRIFICE
+- ▁PROGRESS
+- ▁SHOCK
+- ▁DESIGN
+- ▁SOUGHT
+- ▁PIT
+- ▁SUNDAY
+- ▁OTHERWISE
+- ▁CABIN
+- ▁PRAYER
+- ▁DWELL
+- ▁GAIN
+- ▁BRIDGE
+- ▁PARTICULARLY
+- ▁YIELD
+- ▁TREAT
+- RIGHT
+- ▁OAK
+- ▁ROPE
+- WIN
+- ▁ORDERS
+- ▁SUSPECT
+- ▁EDWARD
+- AB
+- ▁ELEVEN
+- ▁TEETH
+- ▁OCCURRED
+- DDING
+- ▁AMERICA
+- ▁FALLING
+- ▁LION
+- ▁DEPART
+- ▁KEEPING
+- ▁DEMAND
+- ▁PAUSED
+- ▁CEASED
+- INA
+- ▁FUN
+- ▁CHEER
+- ▁PARDON
+- ▁NATIVE
+- LUS
+- LOW
+- ▁DOGS
+- ▁REQUIRED
+- ILITY
+- ▁ELECT
+- ▁ENTERTAIN
+- ITUDE
+- ▁HUGE
+- ▁CARRYING
+- ▁BLU
+- ▁INSIST
+- ▁SATISFACTION
+- ▁HUNT
+- ▁COUNTENANCE
+- ▁UPPER
+- ▁MAIDEN
+- ▁FAILED
+- ▁JAMES
+- ▁FOREIGN
+- ▁GATHER
+- ▁TEST
+- BOARD
+- ▁TERMS
+- ▁SILK
+- ▁BEG
+- ▁BROTHERS
+- ▁PAGE
+- ▁KNEES
+- ▁SHOWN
+- ▁PROFESSOR
+- ▁MIGHTY
+- ▁DEFI
+- ▁CHARM
+- ▁REQUIRE
+- ▁LOG
+- MORE
+- ▁PROOF
+- ▁POSSESSED
+- ▁SOFTLY
+- ▁UNFORTUNATE
+- ▁PRICE
+- ▁SEVERE
+- ▁SINGING
+- ▁STAGE
+- ▁FREEDOM
+- ▁SHOUTED
+- ▁FARTHER
+- ▁MAJESTY
+- ▁PREVIOUS
+- ▁GUIDE
+- ▁MATCH
+- ▁CHEST
+- ▁INTENDED
+- ▁BI
+- ▁EXCITEMENT
+- ▁OFFICERS
+- ▁SUR
+- ▁SHAKE
+- ▁SENTIMENT
+- ▁GENTLY
+- ▁SUCCEEDED
+- ▁MENTION
+- ▁LOCK
+- ▁ACQUAINTANCE
+- ▁IMAGINATION
+- ▁PHYSICAL
+- ▁LEADING
+- ▁SLAVE
+- ▁CART
+- ▁POINTED
+- ▁STEAM
+- ▁SHADE
+- ▁PIPE
+- ▁BASE
+- ▁INVENT
+- ▁ALAS
+- ▁WORKED
+- ▁REGRET
+- ▁BUR
+- ▁FAITHFUL
+- ▁MENTIONED
+- ▁RECORD
+- ▁COMPLAIN
+- ▁SUPERIOR
+- ▁BAY
+- ▁PAL
+- EMENT
+- UE
+- ▁SEVENTY
+- ▁HOTEL
+- ▁SHEEP
+- ▁MEAL
+- ▁ADVICE
+- ▁HIDDEN
+- ▁DEMANDED
+- ▁CONSCIOUS
+- ▁BROW
+- ▁POSSESS
+- ▁FOURTH
+- ▁EVENTS
+- ▁FRI
+- ▁PRAISE
+- ▁ADVANCED
+- ▁RESOLVED
+- ▁STUFF
+- ▁CHEERFUL
+- ▁BIRTH
+- ▁GRIEF
+- ▁AFFORD
+- ▁FAIRY
+- ▁WAKE
+- ▁SIDES
+- ▁SUBSTANCE
+- ▁ARTICLE
+- ▁LEVEL
+- ▁MIST
+- ▁JOINED
+- ▁PRACTICAL
+- ▁CLEARLY
+- ▁TRACE
+- ▁AWAKE
+- ▁OBSERVE
+- ▁BASKET
+- ▁LACK
+- VILLE
+- ▁SPIRITS
+- ▁EXCITED
+- ▁ABANDON
+- ▁SHINING
+- ▁FULLY
+- ▁CALLING
+- ▁CONSIDERABLE
+- ▁SPRANG
+- ▁MILE
+- ▁DOZEN
+- ▁PEA
+- ▁DANGEROUS
+- ▁WIT
+- ▁JEW
+- ▁POUNDS
+- ▁FOX
+- ▁INFORMATION
+- ▁LIES
+- ▁DECK
+- NNY
+- ▁PAUL
+- ▁STARS
+- ▁ANGER
+- ▁SETTLE
+- ▁WILLING
+- ▁ADAM
+- ▁FACES
+- ▁SMITH
+- ▁IMPORTANCE
+- ▁STRAIN
+- WAR
+- ▁SAM
+- ▁FEATHER
+- ▁SERVED
+- ▁AUTHOR
+- ▁PERCEIVED
+- ▁FLAME
+- ▁DIVINE
+- ▁TRAIL
+- ▁ANYBODY
+- ▁SIGH
+- ▁DELICATE
+- KY
+- ▁FOLD
+- ▁HAVEN
+- ▁DESIRED
+- ▁CURIOSITY
+- ▁PRACTICE
+- ▁CONSIDERATION
+- ▁ABSOLUTELY
+- ▁CITIZEN
+- ▁BOTTLE
+- ▁INTERESTED
+- ▁MEAT
+- ▁OCCUPIED
+- ▁CHOOSE
+- ▁THROAT
+- ETTE
+- ▁CANDLE
+- ▁DAWN
+- ▁PROTECT
+- ▁SENTENCE
+- IED
+- ▁ROCKS
+- ▁PORTION
+- ▁APPARENTLY
+- ▁PRESENTED
+- ▁TIGHT
+- ▁ACTUALLY
+- ▁DYING
+- ▁HAM
+- ▁DAILY
+- ▁SUFFERED
+- ▁POLITICAL
+- ▁BODIES
+- ▁MODERN
+- ▁COMPLETELY
+- ▁SOONER
+- TAN
+- ▁PROP
+- ▁ADVANCE
+- ▁REFUSED
+- ▁FARMER
+- ▁POLITE
+- ▁THUNDER
+- ▁BRIEF
+- ▁ELSIE
+- ▁SAILOR
+- ▁SUGGESTED
+- ▁PLATE
+- ▁AID
+- ▁FLESH
+- ▁WEEP
+- ▁BUCK
+- ▁ANTI
+- ▁OCEAN
+- ▁SPEND
+- WELL
+- ▁ODD
+- ▁GOVERNOR
+- ▁ENTRANCE
+- ▁SUSPICION
+- ▁STEPPED
+- ▁RAPIDLY
+- ▁CHECK
+- ▁HIDE
+- ▁FLIGHT
+- ���CLUB
+- ▁ENTIRE
+- ▁INDIANS
+- ASH
+- ▁CAPITAL
+- ▁MAMMA
+- HAR
+- ▁CORRECT
+- ▁CRACK
+- ▁SENSATION
+- ▁WORST
+- ▁PACE
+- ▁MIDST
+- ▁AUGUST
+- ▁PROPORTION
+- ▁INNOCENT
+- LINESS
+- ▁REGARDED
+- ▁DRIVEN
+- ORD
+- ▁HASTE
+- ▁EDUCATION
+- ▁EMPLOY
+- ▁TRULY
+- ▁INSTRUMENT
+- ▁MAG
+- ▁FRAME
+- ▁FOOLISH
+- ▁TAUGHT
+- ▁HANG
+- ▁ARGUMENT
+- ▁NINETEEN
+- ▁ELDER
+- ▁NAY
+- ▁NEEDED
+- ▁NEIGHBOR
+- ▁INSTRUCT
+- ▁PAPERS
+- ▁REWARD
+- ▁EQUALLY
+- ▁FIELDS
+- ▁DIG
+- HIN
+- ▁CONDITIONS
+- JA
+- ▁SPAR
+- ▁REQUEST
+- ▁WORN
+- ▁REMARKABLE
+- ▁LOAD
+- ▁WORSHIP
+- ▁PARK
+- ▁KI
+- ▁INTERRUPTED
+- ▁SKILL
+- ▁TERM
+- LAC
+- ▁CRITIC
+- ▁DISTRESS
+- ▁BELIEF
+- ▁STERN
+- IGHT
+- ▁TRACK
+- ▁HUNTING
+- ▁JEWEL
+- ▁GRADUALLY
+- ▁GLOW
+- ▁RUSHED
+- ▁MENTAL
+- ▁VISITOR
+- ▁PICKED
+- ▁BEHOLD
+- ▁EXPRESSED
+- ▁RUB
+- ▁SKI
+- ARTAGNAN
+- ▁MOREOVER
+- ▁OPERATION
+- ▁CAREFUL
+- ▁KEEN
+- ▁ASSERT
+- ▁WANDER
+- ▁ENEMIES
+- ▁MYSTERIOUS
+- ▁DEPTH
+- ▁PREFER
+- ▁CROSSED
+- ▁CHARMING
+- ▁DREAD
+- ▁FLOUR
+- ▁ROBIN
+- ▁TRE
+- ▁RELIEF
+- ▁INQUIRED
+- ▁APPLE
+- ▁HENCE
+- ▁WINGS
+- ▁CHOICE
+- ▁JUD
+- OO
+- ▁SPECIES
+- ▁DELIGHTED
+- IUM
+- ▁RAPID
+- ▁APPEAL
+- ▁FAMOUS
+- ▁USEFUL
+- ▁HELEN
+- ▁NEWSPAPER
+- ▁PLENTY
+- ▁BEARING
+- ▁NERVOUS
+- ▁PARA
+- ▁URGE
+- ▁ROAR
+- ▁WOUNDED
+- ▁CHAIN
+- ▁PRODUCE
+- ▁REFLECTION
+- ▁MERCHANT
+- ▁QUARREL
+- ▁GLORY
+- ▁BEGUN
+- ▁BARON
+- CUS
+- ▁QUEER
+- ▁MIX
+- ▁GAZE
+- ▁WHISPER
+- ▁BURIED
+- ▁DIV
+- ▁CARD
+- ▁FREQUENTLY
+- ▁TIP
+- ▁KNEE
+- ▁REGION
+- ▁ROOT
+- ▁LEST
+- ▁JEALOUS
+- CTOR
+- ▁SAVED
+- ▁ASKING
+- ▁TRIP
+- QUA
+- ▁UNION
+- HY
+- ▁COMPANIONS
+- ▁SHIPS
+- ▁HALE
+- ▁APPROACHED
+- ▁HARRY
+- ▁DRUNK
+- ▁ARRIVAL
+- ▁SLEPT
+- ▁FURNISH
+- HEAD
+- ▁PIG
+- ▁ABSENCE
+- ▁PHIL
+- ▁HEAP
+- ▁SHOES
+- ▁CONSCIOUSNESS
+- ▁KINDLY
+- ▁EVIDENT
+- ▁SCAR
+- ▁DETERMIN
+- ▁GRASP
+- ▁STEAL
+- ▁OWE
+- ▁KNIFE
+- ▁PRECIOUS
+- ▁ELEMENT
+- ▁PROCEEDED
+- ▁FEVER
+- ▁LEADER
+- ▁RISK
+- ▁EASE
+- ▁GRIM
+- ▁MOUNT
+- ▁MEANWHILE
+- ▁CENTURY
+- OON
+- ▁JUDGMENT
+- ▁AROSE
+- ▁VISION
+- ▁SPARE
+- ▁EXTREME
+- ▁CONSTANT
+- ▁OBSERVATION
+- ▁THRUST
+- ▁DELAY
+- ▁CENT
+- ▁INCLUD
+- ▁LIFT
+- ▁ADMIRE
+- ▁ISSUE
+- ▁FRIENDSHIP
+- ▁LESSON
+- ▁PRINCIPAL
+- ▁MOURN
+- ▁ACCEPTED
+- ▁BURNING
+- ▁CAPABLE
+- ▁EXTRAORDINARY
+- ▁SANG
+- ▁REMOVED
+- ▁HOPED
+- ▁HORN
+- ▁ALICE
+- ▁MUD
+- ▁APARTMENT
+- ▁FIGHTING
+- ▁BLAME
+- ▁TREMBLING
+- ▁SOMEBODY
+- ▁ANYONE
+- ▁BRIDE
+- ▁READER
+- ▁ROB
+- ▁EVERYWHERE
+- ▁LABOUR
+- ▁RECALL
+- ▁BULL
+- ▁HIT
+- ▁COUNCIL
+- ▁POPULAR
+- ▁CHAP
+- ▁TRIAL
+- ▁DUN
+- ▁WISHES
+- ▁BRILLIANT
+- ▁ASSURED
+- ▁FORGOT
+- ▁CONTINUE
+- ▁ACKNOWLEDG
+- ▁RETREAT
+- ▁INCREASED
+- ▁CONTEMPT
+- ▁GRANDFATHER
+- ▁SYMPATHY
+- ▁GHOST
+- ▁STRETCHED
+- ▁CREATURES
+- ▁CAB
+- ▁HIND
+- ▁PLAYING
+- ▁MISERABLE
+- ▁MEMBERS
+- ▁KINDNESS
+- ▁HIGHEST
+- ▁PRIM
+- ▁KISSED
+- ▁DESERVE
+- ▁HUT
+- ▁BEGGED
+- ▁EIGHTY
+- ▁CLOSELY
+- ▁WONDERED
+- ▁MILITARY
+- ▁REMIND
+- ▁ACCORDINGLY
+- ▁LARGER
+- ▁MAINTAIN
+- ▁ENGINE
+- ▁MOTIVE
+- ▁DESTROY
+- ▁STRIP
+- ▁HANS
+- ▁AHEAD
+- ▁INFINITE
+- ▁PROMPT
+- ▁INFORMED
+- TTLE
+- ▁PEER
+- ▁PRESSED
+- ▁TRAP
+- ▁SOMEWHERE
+- ▁BOUGHT
+- ▁VISIBLE
+- ▁ASHAMED
+- ▁TEAR
+- ▁NEIGHBOUR
+- ▁CONSTITUTION
+- ▁INTELLIGENCE
+- ▁PROFESSION
+- ▁HUNGRY
+- RIDGE
+- ▁SMELL
+- ▁STORIES
+- ▁LISTENING
+- ▁APPROACH
+- ▁STRING
+- ▁EXPLANATION
+- ▁IMMENSE
+- ▁RELIGIOUS
+- ▁THROUGHOUT
+- ▁HOLLOW
+- ▁AWAIT
+- ▁FLYING
+- ▁SCREAM
+- ▁ACTIVE
+- ▁RUM
+- ▁PRODUCT
+- ▁UNHAPPY
+- ▁VAGUE
+- ARIES
+- ▁ELIZABETH
+- ▁STUPID
+- ▁DIGNITY
+- ▁ISABEL
+- GAR
+- ▁BRO
+- ▁PITCH
+- ▁COMRADE
+- ▁STIFF
+- ▁RECKON
+- ▁SOLD
+- ▁SPARK
+- ▁STRO
+- ▁CRYING
+- ▁MAGIC
+- ▁REPEAT
+- PORT
+- ▁MARKED
+- ▁COMFORTABLE
+- ▁PROJECT
+- ▁BECOMING
+- ▁PARENTS
+- ▁SHELTER
+- ▁STOLE
+- ▁HINT
+- ▁NEST
+- ▁TRICK
+- ▁THOROUGHLY
+- ▁HOSPITAL
+- ▁WEAPON
+- ▁ROME
+- ▁STYLE
+- ▁ADMITTED
+- ▁SAFETY
+- FIELD
+- ▁UNDERSTANDING
+- ▁TREMBLE
+- ▁PRINT
+- ▁SLAVES
+- ▁WEARY
+- ▁ARTIST
+- ▁CREDIT
+- BURG
+- ▁CONCLUSION
+- ▁SELDOM
+- ▁UNUSUAL
+- ▁CLOUDS
+- ▁UNABLE
+- ▁GAY
+- ▁HANGING
+- ▁SCR
+- ▁BOWED
+- ▁DAVID
+- ▁VOL
+- ▁PUSHED
+- ▁ESCAPED
+- MOND
+- ▁WARN
+- ▁BETRAY
+- ▁EGGS
+- ▁PLAINLY
+- ▁EXHIBIT
+- ▁DISPLAY
+- ▁MEMBER
+- ▁GRIN
+- ▁PROSPECT
+- ▁BRUSH
+- ▁BID
+- ▁SUCCESSFUL
+- ▁EXTENT
+- ▁PERSUADE
+- ▁MID
+- ▁MOOD
+- ▁ARRANGED
+- ▁UNIVERSAL
+- ▁JIM
+- ▁SIGNAL
+- ▁WHILST
+- ▁PHILIP
+- ▁WOLF
+- RATE
+- ▁EAGERLY
+- ▁BILLY
+- ▁RETURNING
+- ▁CONSCIENCE
+- ▁FORTUNATE
+- ▁FEMALE
+- ▁GLEAM
+- ▁HASTILY
+- ▁PROVIDED
+- ▁OBTAIN
+- ��INSTINCT
+- ▁CONCERNED
+- ▁CONCERNING
+- ▁SOMEHOW
+- ▁PINK
+- ▁RAGE
+- ▁ACCUSTOMED
+- ▁UNCONSCIOUS
+- ▁ADVISE
+- ▁BRANCHES
+- ▁TINY
+- ▁REFUSE
+- ▁BISHOP
+- ▁SUPPLY
+- ▁PEASANT
+- ▁LAWYER
+- ▁WASTE
+- ▁CONNECTION
+- ▁DEVELOP
+- ▁CORRESPOND
+- ▁PLUM
+- ▁NODDED
+- ▁SLIPPED
+- ▁EU
+- ▁CONSTANTLY
+- CUM
+- MMED
+- ▁FAIRLY
+- HOUSE
+- ▁KIT
+- ▁RANG
+- ▁FEATURES
+- ▁PAUSE
+- ▁PAINFUL
+- ▁JOE
+- ▁WHENCE
+- ▁LAUGHTER
+- ▁COACH
+- ▁CHRISTMAS
+- ▁EATING
+- ▁WHOLLY
+- ▁APART
+- ▁SUPER
+- ▁REVOLUTION
+- ▁LONELY
+- ▁CHEEKS
+- ▁THRONE
+- ▁CREW
+- ▁ATTAIN
+- ▁ESTABLISHED
+- TIME
+- ▁DASH
+- ▁FRIENDLY
+- ▁OPERA
+- ▁EARL
+- ▁EXHAUST
+- ▁CLIFF
+- ▁REVEAL
+- ▁ADOPT
+- ▁CENTRE
+- ▁MERRY
+- ▁SYLVIA
+- ▁IDEAL
+- ▁MISFORTUNE
+- ▁FEAST
+- ▁ARAB
+- ▁NUT
+- ▁FETCH
+- ▁FOUGHT
+- ▁PILE
+- ▁SETTING
+- ▁SOURCE
+- ▁PERSIST
+- ▁MERCY
+- ▁BARK
+- ▁LUC
+- ▁DEEPLY
+- ▁COMPARE
+- ▁ATTITUDE
+- ▁ENDURE
+- ▁DELIGHTFUL
+- ▁BEARD
+- ▁PATIENCE
+- ▁LOCAL
+- ▁UTTERED
+- ▁VICTORY
+- ▁TREATED
+- ▁SEPARATE
+- ▁WAG
+- ▁DRAGG
+- ▁TITLE
+- ▁TROOPS
+- ▁TRIUMPH
+- ▁REAR
+- ▁GAINED
+- ▁SINK
+- ▁DEFEND
+- ▁TIED
+- ▁FLED
+- ▁DARED
+- ▁INCREASE
+- ▁POND
+- ▁CONQUER
+- ▁FOREHEAD
+- ▁FAN
+- ▁ANXIETY
+- ▁ENCOUNTER
+- ▁SEX
+- ▁HALT
+- ▁SANK
+- ▁CHEEK
+- ▁HUMBLE
+- ▁WRITER
+- ▁EMPLOYED
+- ▁DISTINGUISHED
+- ▁RAISE
+- ▁WHIP
+- ▁GIANT
+- ▁RANGE
+- ▁OBTAINED
+- ▁FLAG
+- ▁MAC
+- ▁JUMPED
+- ▁DISCOVERY
+- ▁NATIONAL
+- ▁COMMISSION
+- ▁POSITIVE
+- ▁LOVING
+- ▁EXACT
+- ▁MURMURED
+- ▁GAZED
+- ▁REFER
+- ▁COLLEGE
+- ▁ENCOURAGE
+- ▁NOVEL
+- ▁CLOCK
+- ▁MORTAL
+- ▁ROLLED
+- ▁RAT
+- IZING
+- ▁GUILTY
+- ▁VICTOR
+- WORTH
+- ▁PRA
+- ▁APPROACHING
+- ▁RELATIVE
+- ▁ESTATE
+- ▁UGLY
+- ▁METAL
+- ▁ROBERT
+- ▁TENT
+- ▁ADMIRATION
+- ▁FOURTEEN
+- ▁BARBAR
+- ▁WITCH
+- ELLA
+- ▁CAKE
+- ▁SHONE
+- ▁MANAGED
+- ▁VOLUME
+- ▁GREEK
+- ▁DANCING
+- ▁WRETCHED
+- ▁CONDEMN
+- ▁MAGNIFICENT
+- ▁CONSULT
+- J
+- ▁ORGAN
+- ▁FLEET
+- ▁ARRANGEMENT
+- ▁INCIDENT
+- ▁MISERY
+- ▁ARROW
+- ▁STROKE
+- ▁ASSIST
+- ▁BUILD
+- ▁SUCCEED
+- ▁DESPERATE
+- ▁WIDOW
+- UDE
+- ▁MARKET
+- ▁WISDOM
+- ▁PRECISE
+- ▁CURRENT
+- ▁SPOIL
+- ▁BADE
+- ▁WOODEN
+- ▁RESIST
+- ▁OBVIOUS
+- ▁SENSIBLE
+- FALL
+- ▁ADDRESSED
+- ▁GIL
+- ▁COUNSEL
+- ▁PURCHASE
+- ▁SELECT
+- ▁USELESS
+- ▁STARED
+- ▁ARREST
+- ▁POISON
+- ▁FIN
+- ▁SWALLOW
+- ▁BLOCK
+- ▁SLID
+- ▁NINETY
+- ▁SPORT
+- ▁PROVIDE
+- ▁ANNA
+- ▁LAMB
+- ▁INTERVAL
+- ▁JUMP
+- ▁DESCRIBED
+- ▁STRIKING
+- ▁PROVISION
+- ▁PROPOSED
+- ▁MELANCHOLY
+- ▁WARRIOR
+- ▁SUGGEST
+- ▁DEPARTURE
+- ▁BURDEN
+- ▁LIMB
+- ▁TROUBLED
+- ▁MEADOW
+- ▁SACRED
+- ▁SOLID
+- ▁TRU
+- ▁LUCY
+- ▁RECOVER
+- ▁ENERGY
+- ▁POWDER
+- ▁RESUMED
+- ▁INTENSE
+- ▁BRITISH
+- ▁STRAW
+- ▁AGREEABLE
+- ▁EVERYONE
+- ▁CONCERN
+- ▁VOYAGE
+- ▁SOUTHERN
+- ▁BOSOM
+- ▁UTTERLY
+- ▁FEED
+- ▁ESSENTIAL
+- ▁CONFINE
+- ▁HOUSEHOLD
+- ▁EXTREMELY
+- ▁WONDERING
+- ▁LIST
+- ▁PINE
+- PHA
+- ▁EXPERIMENT
+- ▁JOSEPH
+- ▁MYSTERY
+- ▁RESTORE
+- ▁BLUSH
+- FOLD
+- ▁CHOSEN
+- ▁INTELLECT
+- ▁CURTAIN
+- OLOGY
+- ▁MOUNTED
+- ▁LAP
+- ▁EPI
+- ▁PUNISH
+- ▁WEDDING
+- ▁RECOGNIZED
+- ▁DRIFT
+- ▁PREPARATION
+- ▁RESOLUTION
+- ▁OPPRESS
+- ▁FIX
+- ▁VICTIM
+- OGRAPH
+- ▁SUMMON
+- ▁JULIA
+- ▁FLOOD
+- ▁WAL
+- ULATION
+- ▁SLIGHTLY
+- ▁LODGE
+- ▁WIRE
+- ▁CONFUSION
+- ▁UNEXPECTED
+- ▁CONCEIVE
+- ▁PRIZE
+- ▁JESUS
+- ▁ADDITION
+- ▁RUDE
+- ▁FATAL
+- ▁CARELESS
+- ▁PATCH
+- ▁KO
+- ▁CATHERINE
+- ▁PARLIAMENT
+- ▁PROFOUND
+- ▁ALOUD
+- ▁RELIEVE
+- ▁PUSH
+- ABILITY
+- ▁ACCOMPANIED
+- ▁SOVEREIGN
+- ▁SINGULAR
+- ▁ECHO
+- ▁COMPOSED
+- ▁SHAKING
+- ATORY
+- ▁ASSISTANCE
+- ▁TEACHER
+- ▁HORRIBLE
+- ▁STRICT
+- ▁VERSE
+- ▁PUNISHMENT
+- ▁GOWN
+- ▁MISTAKEN
+- ▁VARI
+- ▁SWEPT
+- ▁GESTURE
+- ▁BUSH
+- ▁STEEL
+- ▁AFFECTED
+- ▁DIRECTED
+- ▁SURROUNDED
+- ▁ABSURD
+- ▁SUGAR
+- ▁SCRAP
+- ▁IMMEDIATE
+- ▁SADDLE
+- ▁TY
+- ▁ARISE
+- ▁SIGHED
+- ▁EXCHANGE
+- ▁IMPATIENT
+- ▁SNAP
+- ▁EMBRACE
+- ▁DISEASE
+- ▁PROFIT
+- ▁RIDING
+- ▁RECOVERED
+- ▁GOVERN
+- ▁STRETCH
+- ▁CONVINCED
+- ▁LEANING
+- ▁DOMESTIC
+- ▁COMPLEX
+- ▁MANIFEST
+- ▁INDULGE
+- ▁GENIUS
+- ▁AGENT
+- ▁VEIL
+- ▁DESCRIPTION
+- ▁INCLINED
+- ▁DECEIVE
+- ▁DARLING
+- ▁REIGN
+- HU
+- ▁ENORMOUS
+- ▁RESTRAIN
+- ▁DUTIES
+- BURY
+- TTERED
+- ▁POLE
+- ▁ENABLE
+- ▁EXCEPTION
+- ▁INTIMATE
+- ▁COUNTESS
+- ▁TRIBE
+- ▁HANDKERCHIEF
+- ▁MIDNIGHT
+- ▁PROBLEM
+- ▁TRAMP
+- ▁OIL
+- CAST
+- ▁CRUSH
+- ▁DISCUSS
+- ▁RAM
+- ▁TROT
+- ▁UNRE
+- ▁WHIRL
+- ▁LOCKED
+- ▁HORIZON
+- ▁OFFICIAL
+- ▁SCHEME
+- ▁DROWN
+- ▁PIERRE
+- ▁PERMITTED
+- ▁CONNECTED
+- ▁ASSURE
+- ▁COCK
+- ▁UTMOST
+- ▁DEVOTED
+- ▁RELI
+- ▁SUFFICIENTLY
+- ▁INTELLECTUAL
+- ▁CARPET
+- ▁OBJECTION
+- ▁AFTERWARD
+- ▁REALITY
+- ▁NEGRO
+- ▁RETAIN
+- ▁ASCEND
+- ▁CEASE
+- ▁KATE
+- ▁MARVEL
+- KO
+- ▁BOND
+- MOST
+- ▁COAL
+- GATE
+- ▁IGNORANT
+- ▁BREAKING
+- ▁TWIN
+- ▁ASTONISHMENT
+- ▁COFFEE
+- ▁JAR
+- ▁CITIES
+- ▁ORIGIN
+- ▁EXECUT
+- ▁FINAL
+- ▁INHABITANTS
+- ▁STABLE
+- ▁CHIN
+- ▁PARTIES
+- ▁PLUNGE
+- ▁GENEROUS
+- ▁DESCRIBE
+- ▁ANNOUNCED
+- ▁MERIT
+- ▁REVERE
+- ▁ERE
+- ACIOUS
+- ZI
+- ▁DISAPPOINT
+- ▁SUGGESTION
+- ▁DOUBTLESS
+- ▁TRUNK
+- ▁STAMP
+- ▁JOB
+- ▁APPOINTED
+- ▁DIVIDED
+- ▁ACQUAINTED
+- CHI
+- ▁ABSOLUTE
+- ▁FEARFUL
+- ▁PRIVILEGE
+- ▁CRAFT
+- ▁STEEP
+- ▁HUNTER
+- ▁FORBID
+- ▁MODEST
+- ▁ENDEAVOUR
+- ▁SWEEP
+- ▁BEHELD
+- ▁ABSORB
+- ▁CONSTRUCT
+- ▁EMPIRE
+- ▁EXPEDITION
+- ▁ERECT
+- ▁OFFEND
+- ▁INTEND
+- ▁PERMIT
+- ▁DESTROYED
+- ▁CONTRACT
+- ▁THIRST
+- ▁WAGON
+- ▁EVA
+- ▁GLOOM
+- ▁ATMOSPHERE
+- ▁RESERVE
+- ▁VOTE
+- ▁GER
+- ▁NONSENSE
+- ▁PREVAIL
+- ▁QUALITY
+- ▁CLASP
+- ▁CONCLUDED
+- ▁RAP
+- ▁KATY
+- ▁ETERNAL
+- ▁MUTTERED
+- ▁NEGLECT
+- ▁SQUIRE
+- ▁CREEP
+- LOCK
+- ▁ELECTRIC
+- ▁HAY
+- ▁EXPENSE
+- ▁SCORN
+- ▁RETIRED
+- ▁STOUT
+- ▁MURMUR
+- ▁SHARPLY
+- ▁DISTRICT
+- ▁LEAF
+- ▁FAILURE
+- WICK
+- ▁JEAN
+- ▁NUMEROUS
+- ▁INFANT
+- ▁REALIZED
+- ▁TRAVELLER
+- ▁HUNGER
+- ▁JUNE
+- ▁MUN
+- ▁RECOMMEND
+- ▁CREP
+- ZZLE
+- ▁RICHARD
+- WORK
+- ▁MONTE
+- ▁PREACH
+- ▁PALM
+- AVI
+- ▁ANYWHERE
+- ▁DISPOSITION
+- ▁MIRROR
+- ▁VENTURE
+- ▁POUND
+- ▁CIGAR
+- ▁INVITED
+- ▁BENCH
+- ▁PROTECTION
+- ▁BENEFIT
+- ▁THOMAS
+- ▁CLERK
+- ▁REPROACH
+- ▁UNIFORM
+- ▁GENERATION
+- ▁SEAL
+- ▁COMPASS
+- ▁WARNING
+- ▁EXTENDED
+- ▁DIFFICULTIES
+- ▁MAYBE
+- ▁GROAN
+- ▁AFFECT
+- ▁COMB
+- ▁EARN
+- ▁WESTERN
+- ▁IDLE
+- ▁SCORE
+- ▁TAP
+- ▁ASTONISHED
+- ▁INTRODUCED
+- ▁LEISURE
+- ▁LIEUTENANT
+- ▁VIOLENCE
+- ▁FIRMLY
+- ▁MONSTER
+- ▁UR
+- ▁PROPERLY
+- ▁TWIST
+- ▁PIRATE
+- ▁ROBBER
+- ▁BATTER
+- ▁WEPT
+- ▁LEANED
+- ▁FOG
+- ▁ORNAMENT
+- ▁ANDREW
+- ▁BUSHES
+- ▁REPUBLIC
+- ▁CONFIDENT
+- ▁LEAN
+- ▁DART
+- ▁STOOP
+- ▁CURL
+- ▁COUNTER
+- ▁NORTHERN
+- ▁PEARL
+- ▁NEAREST
+- ▁FRANCIS
+- ▁WANDERING
+- ▁FREQUENT
+- ▁STARTLED
+- ▁STATEMENT
+- ▁OCCUR
+- ▁BLOOM
+- ▁NERVE
+- ▁INSPECT
+- ▁INDUCE
+- ▁FLATTER
+- ▁DATE
+- ▁AMBITION
+- ▁SLOPE
+- ▁MALE
+- ▁MADAM
+- ▁MONK
+- ▁RENT
+- ▁CONFIRM
+- ▁INVESTIGAT
+- ▁RABBIT
+- ▁REGIMENT
+- ▁SUBMIT
+- ▁SPELL
+- ▁FURIOUS
+- ▁RAIL
+- ▁BESTOW
+- ▁RALPH
+- ▁SCATTERED
+- ▁COMPELLED
+- ▁THREAD
+- ▁CHILL
+- ▁DENY
+- ▁PRONOUNC
+- ▁MANKIND
+- ▁CATTLE
+- ▁EXECUTION
+- ▁REBEL
+- ▁SUPREME
+- ▁VALUABLE
+- ▁LIKEWISE
+- ▁CONVEY
+- ▁TIDE
+- ▁GLOOMY
+- ▁COIN
+- ▁ACTUAL
+- ▁TAX
+- ▁PROVINCE
+- ▁GRATEFUL
+- ▁SPIRITUAL
+- ▁VANISHED
+- ▁DIANA
+- ▁HAUNT
+- ▁DRAGON
+- ▁CRAWL
+- ▁CHINA
+- ▁GRATITUDE
+- ▁NEAT
+- ▁FINISH
+- ▁INTENT
+- ▁FRIGHT
+- ▁EMBARRASS
+- ▁THIRTEEN
+- ▁RUTH
+- ▁SLIGHTEST
+- ▁DEVELOPMENT
+- ▁INTERVIEW
+- ▁SPECTACLE
+- ▁BROOK
+- VIE
+- ▁WEAKNESS
+- ▁AUDIENCE
+- ▁CONSEQUENTLY
+- ▁ABROAD
+- ▁ASPECT
+- ▁PAINTED
+- ▁RELEASE
+- ▁INSULT
+- ▁SOOTH
+- ▁DISAPPOINTMENT
+- ▁EMERG
+- ▁BRIG
+- ▁ESTEEM
+- ▁INVITATION
+- ▁PASSENGER
+- ▁PUBLISH
+- ▁PIANO
+- ▁IRISH
+- ▁DESK
+- ▁BEATEN
+- ▁FIFTH
+- ▁IMPULSE
+- ▁SWEAR
+- ▁EATEN
+- ▁PURPLE
+- ▁COMMITTED
+- ▁COUNTRIES
+- ▁PERCEIVE
+- ISON
+- ▁CELEBRAT
+- ▁GRANDMOTHER
+- ▁SHUDDER
+- ▁SUNSHINE
+- ▁SPANISH
+- ▁HITHERTO
+- ▁MARILLA
+- ▁SNAKE
+- ▁MOCK
+- ▁INTERFERE
+- ▁WALTER
+- ▁AMID
+- ▁MARBLE
+- ▁MISSION
+- TERIOR
+- ▁DRIVING
+- ▁FURNITURE
+- ▁STEADY
+- ▁CIRCUMSTANCE
+- ▁INTERPRET
+- ▁ENCHANT
+- ▁ERROR
+- ▁CONVICTION
+- ▁HELPLESS
+- ▁MEDICINE
+- ▁QUALITIES
+- ▁ITALIAN
+- ▁HASTENED
+- ▁OCCASIONALLY
+- ▁PURSUED
+- ▁HESITATED
+- ▁INDEPENDENT
+- ▁OLIVER
+- ▁LINGER
+- UX
+- ▁EXAMINED
+- ▁REPENT
+- ▁PHYSICIAN
+- ▁CHASE
+- ▁BELOVED
+- ▁ATTACHED
+- ▁FLORENCE
+- ▁HONEY
+- ▁MOUSE
+- ▁CRIES
+- ▁BAKE
+- ▁POEM
+- ▁DESTRUCTION
+- ▁FULFIL
+- ▁MESSENGER
+- ▁TRISTRAM
+- ▁FANCIED
+- ▁EXCESS
+- ▁CURSE
+- ▁CHU
+- ▁QUANTITY
+- ▁THORNTON
+- ▁CREATED
+- ▁CONTINUALLY
+- ▁LIGHTNING
+- ▁BORNE
+- ▁TOTAL
+- ▁DISPOSED
+- ▁RIFLE
+- ▁POLLY
+- ▁GOAT
+- ▁BACKWARD
+- ▁VIRGINIA
+- ▁KICK
+- ▁PERIL
+- ▁QUO
+- ▁GLORIOUS
+- ▁MULTITUDE
+- ▁LEATHER
+- ▁ABSENT
+- ▁DEMON
+- ▁DEBT
+- ▁TORTURE
+- ▁ACCORD
+- ▁MATE
+- ▁CATHOLIC
+- ▁PILL
+- ▁LIBRARY
+- ▁PURSUIT
+- ▁SHIRT
+- ▁DEAREST
+- ▁COLLAR
+- ▁BEACH
+- ▁ROBE
+- ▁DECLARE
+- ▁BRANCH
+- ▁TEMPT
+- ▁STEADILY
+- ▁DISGUST
+- ▁SILLY
+- ▁ARRIVE
+- ▁DRANK
+- ▁LEVI
+- ▁COMMUNICAT
+- ▁RACHEL
+- ▁WASHINGTON
+- ▁RESIGN
+- ▁MEANTIME
+- ▁LACE
+- ▁ENGAGEMENT
+- ▁QUIVER
+- ▁SEPARATED
+- ▁DISCUSSION
+- ▁VENTURED
+- ▁SURROUNDING
+- ▁POLISH
+- ▁NAIL
+- ▁SWELL
+- ▁JOKE
+- ▁LINCOLN
+- ▁STUDENT
+- ▁GLITTER
+- ▁RUSSIAN
+- ▁READILY
+- ▁CHRIS
+- ▁POVERTY
+- ▁DISGRACE
+- ▁CHEESE
+- ▁HEAVILY
+- ▁SCALE
+- ▁STAFF
+- ▁ENTREAT
+- ▁FAREWELL
+- ▁LUNCH
+- ▁PEEP
+- ▁MULE
+- ▁SOMEONE
+- ▁DISAPPEAR
+- ▁DECISION
+- ▁PISTOL
+- ▁PUN
+- ▁SPUR
+- ▁ASSUMED
+- ▁EXTEND
+- ▁ENTHUSIASM
+- ▁DEFINITE
+- ▁UNDERTAKE
+- ▁COMMITTEE
+- ▁SIMON
+- ▁FENCE
+- ▁APPLIED
+- ▁RELATED
+- ▁VICE
+- ▁UNPLEASANT
+- ▁PROBABLE
+- ▁PROCURE
+- ▁FROWN
+- ▁CLOAK
+- ▁HUMANITY
+- ▁FAMILIES
+- ▁PHILOSOPHER
+- ▁DWARF
+- ▁OVERCOME
+- ▁DEFEAT
+- ▁FASTENED
+- ▁MARSH
+- ▁CLASSES
+- ▁TOMB
+- ▁GRACIOUS
+- ▁REMOTE
+- ▁CELL
+- ▁SHRIEK
+- ▁RESCUE
+- ▁POOL
+- ▁ORGANIZ
+- ▁CHOSE
+- ▁CUTTING
+- ▁COWARD
+- ▁BORDER
+- ▁DIRTY
+- ▁MONKEY
+- ▁HOOK
+- ▁CHUCK
+- ▁EMILY
+- ▁JEST
+- ▁PLAC
+- ▁WEIGH
+- ▁ASSOCIATE
+- ▁GLIMPSE
+- ▁STUCK
+- ▁BOLT
+- ▁MURDERER
+- ▁PONY
+- ▁DISTINGUISH
+- ▁INSTITUTION
+- ▁CUNNING
+- ▁COMPLIMENT
+- ▁APPETITE
+- ▁REPUTATION
+- ▁FEEBLE
+- ▁KIN
+- ▁SERIES
+- ▁GRACEFUL
+- ▁PLATFORM
+- ▁BREEZE
+- ▁PHRASE
+- ▁CLAY
+- MONT
+- ▁RATTL
+- ▁OPPOSITION
+- ▁LANE
+- ▁BOAST
+- ▁GROWTH
+- ▁INCLINATION
+- ▁BEHAVE
+- ▁SUSAN
+- ▁DISTINCTION
+- ▁DISLIKE
+- ▁NICHOLAS
+- ▁SATISFY
+- ▁DRAMA
+- ▁ELBOW
+- ▁GAZING
+- ▁CONSUM
+- ▁SPIN
+- ▁OATH
+- ▁CHANNEL
+- ▁CHARACTERISTIC
+- ▁SPEAR
+- ▁SLAIN
+- ▁SAUCE
+- ▁FROG
+- ▁CONCEPTION
+- ▁TIMID
+- ▁ZEAL
+- ▁APPARENT
+- SHIRE
+- ▁CENTER
+- ▁VARIETY
+- ▁DUSK
+- ▁APT
+- ▁COLUMN
+- ▁REVENGE
+- ▁RIVAL
+- ▁IMITAT
+- ▁PASSIONATE
+- ▁SELFISH
+- ▁NORMAN
+- ▁REPAIR
+- ▁THRILL
+- ▁TREATMENT
+- ▁ROSA
+- ▁MARTIN
+- ▁INDIFFERENT
+- ▁THITHER
+- ▁GALLANT
+- ▁PEPPER
+- ▁RECOLLECT
+- ▁VINE
+- ▁SCARCE
+- ▁SHIELD
+- ▁MINGLED
+- CLOSE
+- ▁HARSH
+- ▁BRICK
+- ▁HUMOR
+- ▁MISCHIEF
+- ▁TREMENDOUS
+- ▁FUNCTION
+- ▁SMART
+- ▁SULTAN
+- ▁DISMISS
+- ▁THREATENED
+- ▁CHEAP
+- ▁FLOCK
+- ▁ENDEAVOR
+- ▁WHISK
+- ▁ITALY
+- ▁WAIST
+- ▁FLUTTER
+- ▁SMOKING
+- ▁MONARCH
+- ▁AFRICA
+- ▁ACCUSE
+- ▁HERBERT
+- ▁REFRESH
+- ▁REJOICE
+- ▁PILLOW
+- ▁EXPECTATION
+- ▁POETRY
+- ▁HOPELESS
+- ▁PERISH
+- ▁PHILOSOPHY
+- ▁WHISTLE
+- ▁BERNARD
+- ▁LAMENT
+- ▁IMPROVE
+- ▁SUP
+- ▁PERPLEX
+- ▁FOUNTAIN
+- ▁LEAGUE
+- ▁DESPISE
+- ▁IGNORANCE
+- ▁REFERENCE
+- ▁DUCK
+- ▁GROVE
+- ▁PURSE
+- ▁PARTNER
+- ▁PROPHET
+- ▁SHIVER
+- ▁NEIGHBOURHOOD
+- ▁REPRESENTATIVE
+- SAIL
+- ▁WIP
+- ▁ACQUIRED
+- ▁CHIMNEY
+- ▁DOCTRINE
+- ▁MAXIM
+- ▁ANGLE
+- ▁MAJORITY
+- ▁AUTUMN
+- ▁CONFUSED
+- ▁CRISTO
+- ▁ACHIEVE
+- ▁DISGUISE
+- ▁REDUCED
+- ▁EARLIER
+- ▁THEATRE
+- ▁DECIDE
+- MINATED
+- OLOGICAL
+- ▁OCCUPATION
+- ▁VIGOROUS
+- ▁CONTINENT
+- ▁DECLINE
+- ▁COMMUNITY
+- ▁MOTIONLESS
+- ▁HATRED
+- ▁COMMUNICATION
+- ▁BOWL
+- ▁COMMENT
+- ▁APPROVE
+- ▁CEREMONY
+- ▁CRIMINAL
+- ▁SCIENTIFIC
+- ▁DUCHESS
+- ▁VIVID
+- ▁SHIFT
+- ▁AVAIL
+- ▁DAMP
+- ▁JOHNSON
+- ▁SLENDER
+- ▁CONTRAST
+- ▁AMUSEMENT
+- ▁PLOT
+- ▁LYN
+- ▁ASSOCIATION
+- ▁SNATCH
+- ▁UNCERTAIN
+- ▁PRESSURE
+- ▁PERCH
+- ▁APPLY
+- ▁PLANET
+- ▁NOTWITHSTANDING
+- ▁SWUNG
+- ▁STIRRED
+- ▁ATTENDANT
+- ▁ENJOYMENT
+- ▁WORRY
+- ▁ALBERT
+- ▁NAKED
+- ▁TALENT
+- ▁MARIAN
+- ▁REFORM
+- ▁DELIBERATE
+- ▁INTELLIGENT
+- ▁SENSITIVE
+- ▁YONDER
+- ▁PUPIL
+- ▁FRIGHTFUL
+- ▁DOUBTFUL
+- ▁STANDARD
+- ▁MAGISTRATE
+- ▁SHEPHERD
+- ▁STOMACH
+- ▁DEPOSIT
+- ▁RENEW
+- ▁HEDGE
+- ▁FRANCS
+- ▁POSSIBILITY
+- ▁RESEMBLE
+- ▁FATIGUE
+- ▁PORTRAIT
+- ▁FAVORITE
+- ▁CREAM
+- ▁BURG
+- ▁SECRETARY
+- ▁DIVERS
+- ▁ACTIVITY
+- ▁SPECULAT
+- ▁HUMOUR
+- ▁FITTED
+- ▁EXTERNAL
+- ▁CETERA
+- ▁WRAPPED
+- ▁WHIT
+- ▁FRED
+- ▁EXAMINATION
+- ▁LODGING
+- ▁OWING
+- ▁JAW
+- ▁CROW
+- ▁BALANCE
+- ▁PUFF
+- ▁TENDERNESS
+- ▁PORTHOS
+- ▁ANCHOR
+- ▁INTERRUPT
+- ▁NECESSARILY
+- ▁PERPETUAL
+- ▁AGONY
+- ▁POPE
+- ▁SCHOLAR
+- ▁SCOTLAND
+- ▁SUPPRESS
+- ▁WRATH
+- ▁WRECK
+- ▁EXCEED
+- ▁PERFECTION
+- ▁INDIA
+- ▁TRADITION
+- ▁SECTION
+- ▁EASTERN
+- ▁DOORWAY
+- ▁WIVES
+- ▁CONVENTION
+- ▁ANNOUNC
+- ▁EGYPT
+- ▁CONTRADICT
+- ▁SCRATCH
+- ▁CENTRAL
+- ▁GLOVE
+- ▁WAX
+- ▁PREPARE
+- ▁ACCOMPANY
+- ▁INCREASING
+- ▁LIBERAL
+- ▁RAISING
+- ▁ORANGE
+- ▁SHOE
+- ▁ATTRIBUTE
+- ▁LITERATURE
+- ▁PUZZLED
+- ▁WITHDRAW
+- ▁WHITHER
+- ▁HAWK
+- ▁MOONLIGHT
+- ▁EXAMINE
+- ▁HAPPILY
+- ▁PRECEDE
+- ▁DETECTIVE
+- ▁INCHES
+- ▁SOLITARY
+- ▁DUTCH
+- ▁NAPOLEON
+- ▁UNEASY
+- ▁CARDINAL
+- ▁BLEW
+- ▁FOWL
+- ▁DECORAT
+- ▁CHILDHOOD
+- ▁TORMENT
+- ▁LOSING
+- ▁PERMISSION
+- ▁BLANK
+- ▁UPSTAIRS
+- ▁CAPACITY
+- ▁TRIFLE
+- ▁FOLLY
+- ▁RECOGNIZE
+- ▁REMOVE
+- ▁VENGEANCE
+- ▁ENTERPRISE
+- ▁BEDROOM
+- ▁ANYHOW
+- ▁INQUIRY
+- ▁ASHES
+- ▁DRAG
+- ▁HUSH
+- ▁AWKWARD
+- ▁SATURDAY
+- ▁GENUINE
+- ▁SURVIV
+- ▁SKIRT
+- ▁AFFECTIONATE
+- ▁TANG
+- ▁MUTUAL
+- ▁DISPUTE
+- ▁EAGLE
+- ▁INCOME
+- ▁BIND
+- ▁FAME
+- ▁IMPROVEMENT
+- ROVING
+- ▁DIFFER
+- ▁AWOKE
+- ▁SLEEVE
+- ▁SOLITUDE
+- ▁FAVOURITE
+- JI
+- ▁DETECT
+- ▁COMPREHEND
+- ▁PREPARING
+- ▁SERPENT
+- ▁SUMMIT
+- ▁KNOT
+- ▁KNIT
+- ▁COPY
+- ▁STOPPING
+- ▁FADED
+- ▁HIDEOUS
+- ▁JULIE
+- STEAD
+- ▁SHINE
+- ▁CONFLICT
+- ▁PROPOSITION
+- ▁REFUGE
+- ▁GALLERY
+- ▁BUNDLE
+- ▁AXE
+- ▁SLAVERY
+- ▁MASK
+- ▁ALYOSHA
+- ▁LADDER
+- ▁DEPARTMENT
+- ▁DISCHARGE
+- ▁DEPRESS
+- ▁GALLOP
+- ▁SCARLET
+- ▁KITTY
+- ▁RECEIVING
+- ▁SURRENDER
+- ▁SUSTAIN
+- ▁TWILIGHT
+- ▁CONGRESS
+- ▁IRELAND
+- ▁FUNNY
+- ▁LEND
+- ▁CONSTITUTE
+- ▁FUNERAL
+- ▁CRYSTAL
+- ▁SPAIN
+- ▁EXCEEDINGLY
+- ▁DAMN
+- ▁COMMUN
+- ▁CIVILIZATION
+- ▁PREJUDICE
+- ▁PORCH
+- ▁ASSISTANT
+- ▁INDUSTRY
+- ▁TUMBLE
+- ▁DEFENCE
+- ▁HITHER
+- ▁SMOT
+- ▁COLONI
+- ▁AMAZEMENT
+- ▁MARGUERITE
+- ▁MIRACLE
+- ▁INHERIT
+- ▁BEGGAR
+- ▁ENVELOPE
+- ▁INDIGNATION
+- ▁NATASHA
+- ▁PROPOSAL
+- ▁FRAGMENT
+- ▁ROUSED
+- ▁ROAST
+- ENCIES
+- ▁COMMENCED
+- ▁RESOURCE
+- ▁POPULATION
+- ▁QUOTH
+- ▁PURSUE
+- ▁EDUCAT
+- ▁AFFLICT
+- ▁CONTACT
+- ▁CRIMSON
+- ▁DIVISION
+- ▁DISORDER
+- ▁COPPER
+- ▁SOLICIT
+- ▁MODERATE
+- ▁DRUM
+- ▁SWIM
+- ▁SALUTE
+- ▁ASSUME
+- ▁MUSCLE
+- ▁OVERWHELM
+- ▁SHAKESPEARE
+- ▁STRUGGLING
+- ▁TRANQUIL
+- ▁CHICKEN
+- ▁TREAD
+- ▁CLAW
+- ▁BIBLE
+- ▁RIDGE
+- ▁THREAT
+- ▁VELVET
+- ▁EXPOSED
+- ▁IDIOT
+- ▁BARREL
+- ▁PENNY
+- ▁TEMPTATION
+- ▁DANGLARS
+- ▁CENTURIES
+- ▁DISTRIBUT
+- ▁REJECT
+- ▁RETORTED
+- ▁CONCENTRAT
+- ▁CORDIAL
+- ▁MOTOR
+- ▁CANNON
+- KEEP
+- ▁WRETCH
+- ▁ASSURANCE
+- ▁THIEF
+- ▁SURVEY
+- ▁VITAL
+- ▁RAILWAY
+- ▁JACKSON
+- ▁CRASH
+- ▁GROWL
+- ▁COMBAT
+- ▁RECOLLECTION
+- ▁SECURITY
+- ▁JACOB
+- ▁CLUTCH
+- ▁BLANKET
+- ▁NANCY
+- ▁CELLAR
+- ▁CONVENIENT
+- ▁INDIGNANT
+- ▁COARSE
+- ▁WORM
+- ▁SCREEN
+- ▁TRANSPORT
+- ▁BULLET
+- ▁APPRECIATE
+- ▁DEVOTION
+- ▁INVISIBLE
+- ▁DRIED
+- ▁MIXTURE
+- ▁CANDID
+- ▁PERFORMANCE
+- ▁RIPE
+- ▁EXQUISITE
+- ▁BARGAIN
+- ▁TOBACCO
+- ▁LOYAL
+- ▁MOULD
+- ▁ATTENTIVE
+- ▁DOROTHY
+- ▁BRUTE
+- ▁ESTABLISHMENT
+- ▁ABILITY
+- ▁INHABIT
+- ▁OBSCURE
+- ▁BORROW
+- ▁ESSENCE
+- ▁DISMAY
+- ▁FLEE
+- ▁BLADE
+- ▁PLUCK
+- ▁COFFIN
+- ▁SUNSET
+- ▁STEPHEN
+- ▁ECONOMIC
+- ▁HOLIDAY
+- ▁MECHANICAL
+- ▁COTTON
+- ▁AWAKENED
+- ▁SEIZE
+- ▁RIDICULOUS
+- ▁SANCHO
+- ▁HESITATION
+- ▁CORPSE
+- ▁SAVING
+- HOLD
+- FOOT
+- ▁ELDEST
+- ▁DESPITE
+- ▁EDITH
+- ▁CHERISH
+- ▁RESISTANCE
+- ▁WILSON
+- ▁ARGUE
+- ▁INQUIRE
+- ▁APPREHENSION
+- ▁AVENUE
+- ▁DRAKE
+- ▁PROPOSE
+- HURST
+- ▁INFERIOR
+- ▁STAIRCASE
+- ▁WHEREFORE
+- ▁CARLYLE
+- ▁COUCH
+- ▁ROUTE
+- ▁POLITICS
+- ▁TOMORROW
+- ▁THRONG
+- ▁NAUGHT
+- ▁SUNLIGHT
+- ▁INDIFFERENCE
+- ▁OBEDIENCE
+- ▁RECEPTION
+- ▁VEGETABLE
+- ▁IMPERFECT
+- ▁RESIDENCE
+- ▁TURKEY
+- ▁VIOLET
+- ▁SARAH
+- ▁ALTAR
+- ▁GRIEVE
+- ▁JERK
+- ▁ENSU
+- ▁MAGICIAN
+- ▁BLOSSOM
+- ▁LANTERN
+- ▁RESOLUTE
+- ▁THOUGHTFULLY
+- ▁FORTNIGHT
+- ▁TRUMPET
+- ▁VALJEAN
+- ▁UNWILLING
+- ▁LECTURE
+- ▁WHEREUPON
+- ▁HOLLAND
+- ▁CHANGING
+- ▁CREEK
+- ▁SLICE
+- ▁NORMAL
+- ▁ANNIE
+- ▁ACCENT
+- ▁FREDERICK
+- ▁DISAGREEABLE
+- ▁RUBBED
+- ▁DUMB
+- ▁ESTABLISH
+- ▁IMPORT
+- ▁AFFIRM
+- ▁MATTHEW
+- ▁BRISK
+- ▁CONVERT
+- ▁BENDING
+- ▁IVAN
+- ▁MADEMOISELLE
+- ▁MICHAEL
+- ▁EASIER
+- ▁JONES
+- ▁FACING
+- ▁EXCELLENCY
+- ▁LITERARY
+- ▁GOSSIP
+- ▁DEVOUR
+- ▁STAGGER
+- ▁PENCIL
+- ▁AVERAGE
+- ▁HAMMER
+- ▁TRIUMPHANT
+- ▁PREFERRED
+- ▁APPLICATION
+- ▁OCCUPY
+- ▁AUTHORITIES
+- BURN
+- ▁ASCERTAIN
+- ▁CORRIDOR
+- ▁DELICIOUS
+- ▁PRACTISE
+- ▁UNIVERSE
+- ▁SHILLING
+- ▁CONTEST
+- ▁ASHORE
+- ▁COMMIT
+- ▁ADMINISTRATION
+- ▁STUDIED
+- ▁RIGID
+- ▁ADORN
+- ▁ELSEWHERE
+- ▁INNOCENCE
+- ▁JOURNAL
+- ▁LANDSCAPE
+- ▁TELEGRAPH
+- ▁ANGRILY
+- ▁CAMPAIGN
+- ▁UNJUST
+- ▁CHALLENGE
+- ▁TORRENT
+- ▁RELATE
+- ▁ASSEMBLED
+- ▁IMPRESSED
+- ▁CANOE
+- ▁CONCLUD
+- ▁QUIXOTE
+- ▁SATISFACTORY
+- ▁NIECE
+- ▁DEAF
+- ▁RAFT
+- ▁JIMMY
+- ▁GLID
+- ▁REGULAT
+- ▁CHATTER
+- ▁GLACIER
+- ▁ENVY
+- ▁STATUE
+- ▁BOSTON
+- ▁RICHMOND
+- ▁DENIED
+- ▁FANNY
+- ▁SOLOMON
+- ▁VULGAR
+- ▁STALK
+- ▁REPLACE
+- ▁SPOON
+- ▁BASIN
+- ▁FEATURE
+- ▁CONVICT
+- ▁ARCHITECT
+- ▁ADMIRAL
+- ▁RIBBON
+- ▁PERMANENT
+- ▁APRIL
+- ▁JOLLY
+- ▁NEIGHBORHOOD
+- ▁IMPART
+- BOROUGH
+- CAMP
+- ▁HORRID
+- ▁IMMORTAL
+- ▁PRUDENCE
+- ▁SPANIARD
+- ▁SUPPOSING
+- ▁TELEPHONE
+- ▁TEMPERATURE
+- ▁PENETRATE
+- ▁OYSTER
+- ▁APPOINTMENT
+- ▁EGYPTIAN
+- ▁DWELT
+- ▁NEPHEW
+- ▁RAILROAD
+- ▁SEPTEMBER
+- ���DEVICE
+- ▁WHEAT
+- ▁GILBERT
+- ▁ELEGANT
+- ▁ADVERTISE
+- ▁RATIONAL
+- ▁TURTLE
+- ▁BROOD
+- ▁ASSEMBLY
+- ▁CULTIVATE
+- ▁EDITOR
+- ▁SPECIMEN
+- ▁UNDOUBTEDLY
+- ▁WHALE
+- ▁DROPPING
+- ▁BALLOON
+- ▁MEDICAL
+- COMB
+- ▁COMPOSITION
+- ▁FOOTSTEPS
+- ▁LAUNCELOT
+- ▁DISCOURSE
+- ▁ERRAND
+- ▁CONVERSE
+- ▁ADVANCING
+- ▁DOWNSTAIRS
+- ▁TUMULT
+- ▁CORRUPT
+- ▁SUFFICE
+- ▁ANGUISH
+- ▁SHAGGY
+- ▁RETIRE
+- ▁TIMBER
+- ▁BLAZE
+- ▁ABSTRACT
+- ▁EMBROIDER
+- ▁PHOTOGRAPH
+- ▁PROSPERITY
+- ▁TERRIBLY
+- ▁TERRITORY
+- ▁THRESHOLD
+- ▁PAVEMENT
+- ▁INJURED
+- ▁LIMP
+- ▁AGITATION
+- ▁RASCAL
+- ▁PRESUME
+- ▁OBSERVING
+- ▁OBSTACLE
+- ▁SIMPLICITY
+- ▁SLUMBER
+- ▁SUPPLIED
+- ▁COMBINATION
+- ▁DRAIN
+- ▁WILDERNESS
+- ▁BELIEVING
+- ▁VILLAIN
+- ▁RECKLESS
+- ▁INJURY
+- ▁CLAPP
+- ▁FRIDAY
+- ▁HERCULES
+- ▁KENNEDY
+- ▁SYMPTOM
+- ▁SLEDGE
+- ▁CEILING
+- ▁LEMON
+- ▁PLAGUE
+- ▁MONDAY
+- ▁CANVAS
+- ▁IMPATIENCE
+- ▁UNCOMFORTABLE
+- ▁ACCESS
+- ▁FROZEN
+- ▁SENATOR
+- ▁FRANZ
+- ▁SWIMMING
+- ▁BARRIER
+- ▁ADJUST
+- ▁COMPARISON
+- ▁PROCLAIM
+- ▁WRINKL
+- ▁OVERLOOK
+- ▁MITYA
+- ▁GUILT
+- ▁PERCEPTION
+- ▁PRECAUTION
+- ▁SPECTATOR
+- ▁SURPRISING
+- ▁DISTRACT
+- ▁DISDAIN
+- ▁BONNET
+- ▁MAGNET
+- ▁PROFESS
+- ▁CONFOUND
+- ▁NARRATIVE
+- ▁STRUCTURE
+- ▁SKETCH
+- ▁ULTIMATE
+- ▁GLOBE
+- ▁INSECT
+- FICIENCY
+- ▁ORCHARD
+- ▁AMIABLE
+- ▁DESCENT
+- ▁INDEPENDENCE
+- ▁MANUFACTURE
+- ▁SPRINKLE
+- ▁NIGHTINGALE
+- ▁CUSHION
+- ▁EMINENT
+- ▁SCOTT
+- ▁ARRAY
+- ▁COSETTE
+- ▁WAVING
+- ▁EXTRACT
+- ▁IRREGULAR
+- ▁PERSECUT
+- ▁DERIVED
+- ▁WITHDREW
+- ▁CAUTION
+- ▁SUSPICIOUS
+- ▁MEMORIES
+- ▁NOWHERE
+- ▁SUBTLE
+- ▁THOROUGH
+- Q
+- ▁APPROPRIATE
+- ▁SLAUGHTER
+- ▁YOURSELVES
+- ▁THUMB
+- ▁TWAS
+- ▁ABODE
+- ▁BIDDING
+- ▁CONSPICUOUS
+- ▁REBECCA
+- ▁SERGEANT
+- ▁APRON
+- ▁ANTICIPATE
+- ▁DISCIPLINE
+- ▁GLANCING
+- ▁PILGRIM
+- ▁SULLEN
+- ▁CONTRIBUTE
+- ▁PRAIRIE
+- ▁CARVED
+- ▁COMMERCE
+- ▁EXCLAMATION
+- ▁MUSCULAR
+- ▁NOVEMBER
+- ▁PHENOMENA
+- ▁SYMBOL
+- ▁UMBRELLA
+- ▁DIMINISH
+- ▁PARLOUR
+- ▁THREATENING
+- ▁STUMP
+- ▁EXTENSIVE
+- ▁PLEASING
+- ▁REMEMBRANCE
+- ▁COMBINED
+- ▁SHERIFF
+- ▁SHAFT
+- ▁LAURA
+- ▁INTERCOURSE
+- ▁STRICKEN
+- ▁SUPPLIES
+- ▁LANDLORD
+- ▁SHRINK
+- ▁PRICK
+- ▁CAESAR
+- ▁DRUG
+- ▁BEWILDERED
+- ▁NAUTILUS
+- ▁BRUTAL
+- ▁COMMERCIAL
+- ▁MAGGIE
+- ▁SPHERE
+- ▁VIRGIN
+- ▁BRETHREN
+- ▁DESTINY
+- ▁POLICY
+- ▁TERRIFIED
+- ▁HOUSEKEEPER
+- ▁CRAZY
+- ▁ARDENT
+- ▁DISCERN
+- ▁WRAP
+- ▁MARQUIS
+- ▁RUSSIA
+- MOUTH
+- ▁BRITAIN
+- ▁HARBOUR
+- ▁CONCERT
+- ▁DONKEY
+- ▁DAMAGE
+- ▁SLIM
+- ABOUT
+- ▁LUXURY
+- ▁MONSTROUS
+- ▁TENDENCY
+- ▁PARADISE
+- ▁CULTURE
+- ▁JULIUS
+- ▁RAOUL
+- ▁REMEDY
+- ▁DECAY
+- ▁SCOLD
+- ▁SPLIT
+- ▁ASSAULT
+- ▁DECEMBER
+- ▁MOSCOW
+- ▁EXPLORE
+- ▁TROUSERS
+- ▁WRIST
+- PIECE
+- ▁MUSKET
+- ▁VALENTINE
+- ▁TYRANT
+- ▁ABRAHAM
+- ▁MEDIUM
+- ▁ARTIFICIAL
+- ▁FACULTY
+- ▁OBLIGATION
+- ▁RESEMBLANCE
+- ▁INQUIRIES
+- ▁DETAIN
+- ▁SWARM
+- ▁PLEDGE
+- ▁ADMIRABLE
+- ▁DEFECT
+- ▁SUPERINTEND
+- ▁PATRIOT
+- ▁CLUNG
+- ▁DISMAL
+- ▁RECIT
+- ▁IGNOR
+- ▁AMELIA
+- ▁JUSTIFY
+- ▁ELEPHANT
+- ▁ESTIMATE
+- ▁KNELT
+- ▁SERVING
+- ▁WHIM
+- ▁SHRILL
+- ▁STUDIO
+- ▁TEXT
+- ▁ALEXANDER
+- ▁WROUGHT
+- ▁ABUNDANT
+- ▁SITUATED
+- ▁REGAIN
+- ▁FIERY
+- ▁SNEER
+- ▁SWEAT
+- ▁GLARE
+- ▁NIGH
+- ▁ESCORT
+- ▁INEVITABLE
+- ▁PSMITH
+- ▁RELUCTANT
+- ▁PRECEDING
+- ▁RESORT
+- ▁OUTRAGE
+- ▁AMBASSADOR
+- ▁CONSOLATION
+- ▁RECOGNITION
+- ▁REMORSE
+- ▁BEHALF
+- ▁FORMIDABLE
+- ▁GRAVITY
+- ▁DIVIDE
+- ▁CONFRONT
+- ▁GIGANTIC
+- ▁OCTOBER
+- ▁FLANK
+- ▁SLEW
+- ▁CLARA
+- ▁FILM
+- ▁BULK
+- ▁POMP
+- ▁ELEANOR
+- ▁EMPHASIS
+- ▁JAPANESE
+- ▁CAVALRY
+- ▁EXCLUSIVE
+- ▁PERFUME
+- ▁BRONZE
+- ▁FEDERAL
+- ▁LIQUID
+- ▁RUBBING
+- ▁OVEN
+- DOLPH
+- ▁CONVULS
+- ▁DEPRIVED
+- ▁RESPONSIBILITY
+- ▁SIGNIFICANT
+- ▁WAISTCOAT
+- ▁CLUSTER
+- ▁MARTHA
+- ▁REVERSE
+- ▁ATTORNEY
+- ▁DROOP
+- ▁SKILFUL
+- ▁HABITUAL
+- ▁PUMP
+- ▁INTERVEN
+- ▁OWL
+- ▁CONJECTURE
+- ▁FANTASTIC
+- ▁RESPONSIBLE
+- ▁DESTINED
+- ▁DOCUMENT
+- ▁THEREUPON
+- ▁GODDESS
+- ▁PACIFIC
+- ▁WARRANT
+- ▁COSTUME
+- ▁BRIDLE
+- ▁CALIFORNIA
+- ▁DEMOCRATIC
+- ▁EUSTACE
+- ▁SQUIRREL
+- ▁UNCOMMON
+- ▁MARVELLOUS
+- ▁PLOUGH
+- ▁TRAGEDY
+- ▁VAULT
+- ▁HESITATE
+- ▁REFRAIN
+- ▁ADMIRING
+- ▁CORPORAL
+- ▁ENTITLED
+- ▁SHREWD
+- ▁SQUEEZ
+- ▁ACCURATE
+- ▁TEMPEST
+- ▁MONUMENT
+- ▁SIEGE
+- ▁CHINESE
+- ▁RAVEN
+- ▁LOUNG
+- ▁ASSASSIN
+- ▁INFLICT
+- ▁AGITATED
+- ▁DESIRABLE
+- ▁EARLIEST
+- ▁LAUNCH
+- ▁PILOT
+- ▁PULSE
+- ▁MUTE
+- LEIGH
+- ▁LIQUOR
+- ▁SCARECROW
+- ▁SKULL
+- ▁DESOLATE
+- ▁SUBLIME
+- ▁SERENE
+- ▁RECESS
+- ▁WAKING
+- ▁CHARLOTTE
+- ▁CIRCULAR
+- ���INJUSTICE
+- ▁PINOCCHIO
+- ▁PRISCILLA
+- ▁THYSELF
+- ▁OCCURRENCE
+- ▁CASUAL
+- ▁FRANTIC
+- ▁LEGEND
+- ▁FERTIL
+- ▁BACKGROUND
+- ▁DELICACY
+- ▁ESTRALLA
+- ▁MANUSCRIPT
+- ▁RESPONSE
+- ▁UNIVERSITY
+- ▁WOLVES
+- ▁SCANDAL
+- ▁STUMBLE
+- ▁HOARSE
+- ▁BODILY
+- ▁CONVENT
+- ▁EXAMINING
+- ▁INCAPABLE
+- ▁PERCEIVING
+- ▁PHILADELPHIA
+- ▁SUBSEQUENT
+- ▁THIEVES
+- ▁ACCUMULAT
+- ▁DAMSEL
+- ▁SCOTCH
+- ▁UNDERNEATH
+- ▁NOBILITY
+- ▁SMASH
+- ▁REVOLT
+- ▁ENGAGE
+- ▁CATHEDRAL
+- ▁CHAMPION
+- ▁DESPATCH
+- ▁ETERNITY
+- ▁JANUARY
+- ▁PLEADED
+- ▁PROBABILITY
+- ▁JIMMIE
+- ▁PARALLEL
+- ▁FISHERMAN
+- ▁JERRY
+- ▁SWORE
+- ▁DRAUGHT
+- ▁OPPONENT
+- ▁PRIMITIVE
+- ▁SIGNIFICANCE
+- ▁SUBSTANTIAL
+- ▁AMAZED
+- ▁DUNBAR
+- ▁COMMEND
+- ▁CONTEMPLATE
+- ▁TESTIMONY
+- ▁IMPERIAL
+- ▁ADAPT
+- ▁JUICE
+- ▁CALAMIT
+- CULAR
+- ▁CHATEAU
+- ▁PHOENIX
+- ▁PRUDENT
+- ▁SOLUTION
+- ▁VILLEFORT
+- ▁REACTION
+- ▁RELAX
+- ▁YU
+- ▁PROHIBIT
+- ▁DISTRUST
+- ▁PLUNDER
+- ▁WELFARE
+- ▁NAVIGAT
+- ▁PARLOR
+- ▁LAZY
+- ▁DETACH
+- OMETER
+- ▁PRIV
+- ▁DISCOURAGE
+- ▁OBSTINATE
+- ▁REJOICING
+- ▁SERMON
+- ▁VEHICLE
+- ▁FANCIES
+- ▁ENLIGHTEN
+- ▁ACUTE
+- ▁ILLUSION
+- ▁ANTHEA
+- ▁MARTIAN
+- ▁EXCITE
+- ▁GENEROSITY
+- OLOGIST
+- ▁AMAZING
+- ▁UNWORTHY
+- ▁INTERNAL
+- ▁INCENSE
+- ▁VIBRAT
+- ▁ADHERE
+- ROACH
+- ▁FEBRUARY
+- ▁MEXICAN
+- ▁POTATOES
+- ▁INCESSANT
+- ▁INTERPOSED
+- ▁PARCEL
+- ▁VEXED
+- ▁PROMOTE
+- MIDST
+- ▁ARISTOCRAT
+- ▁CYRIL
+- ▁EMBARK
+- ▁ABUNDANCE
+- ▁LITERALLY
+- ▁SURGEON
+- ▁TERRACE
+- ▁ATLANTIC
+- ▁MARTYR
+- ▁SPECK
+- ▁SENATE
+- ▁LOAF
+- ▁ADMINISTER
+- ▁APPREHEND
+- ▁SUBDUED
+- ▁TEMPORARY
+- ▁DOMINION
+- ▁ELABORATE
+- ▁DIGNIFIED
+- ▁ELIZA
+- ▁SPLASH
+- ▁CONSEIL
+- ▁DEXTER
+- ▁UNSEEN
+- ▁TRAGIC
+- VOCATION
+- ▁GRATIFY
+- ▁BACHELOR
+- ▁DEFENSE
+- ▁EXCURSION
+- ▁FACULTIES
+- ▁PROPRIETOR
+- ▁SYMPATHETIC
+- ▁UNNECESSARY
+- ▁RADIANT
+- ▁VACANT
+- ▁OUNCE
+- ▁SCREW
+- ▁PHENOMENON
+- ▁PROMINENT
+- ▁WORRIED
+- ▁STUDIES
+- ▁CLIMATE
+- ▁KEITH
+- ▁ARAMIS
+- ▁BLISS
+- ▁CONTINUAL
+- ▁SURPASS
+- ▁HEBREW
+- ▁IDENTITY
+- ▁PROVOKE
+- ▁TEMPERAMENT
+- ▁CHARIOT
+- ▁HARBOR
+- ▁NINTH
+- ▁PRIOR
+- ▁DESIROUS
+- ▁JERUSALEM
+- ▁UNDERTAKING
+- ▁EDISON
+- ▁MIRTH
+- ▁SCOUT
+- ▁APPARATUS
+- ▁ILLUSTRATION
+- ▁INTELLIGIBLE
+- ▁INVARIABLY
+- ▁PIERCED
+- ▁REVIEW
+- ▁FLICKER
+- ▁HAZARD
+- ▁REVELATION
+- ▁DIXON
+- ▁EXCITING
+- ▁GOSPEL
+- ▁CONSTANCE
+- ▁OVERTAKE
+- ▁GUINEA
+- ▁ALADDIN
+- ▁CHICAGO
+- ▁TULLIVER
+- ▁HAMILTON
+- ▁GARRISON
+- ▁DISCIPLE
+- ▁INTENSITY
+- ▁TRAITOR
+- ▁CHANCELLOR
+- ▁PROVERB
+- ▁DAGGER
+- ▁FORESEE
+- ▁CONFIDE
+- ▁GLIMMER
+- ▁CHAUVELIN
+- ▁ILLUSTRATE
+- ▁VOLUNTEER
+- ▁JUNGLE
+- ▁STREAK
+- ▁SUNRISE
+- ▁DISSOLV
+- ▁QUEST
+- ▁AWHILE
+- ▁FELICITY
+- ▁LEGISLATURE
+- ▁LEONORA
+- ▁MAGAZINE
+- ▁PITIFUL
+- ▁COLONY
+- ▁SHAWL
+- ▁ARRIVING
+- ▁FUNDAMENTAL
+- ▁CARPENTER
+- ▁OVERFLOW
+- ▁EXPAND
+- ▁HARVEST
+- ▁FEMININE
+- ▁INNUMERABLE
+- ▁SCRAMBLE
+- ▁TWENTIETH
+- ▁TRIFLING
+- ▁GHASTL
+- ▁CONQUEST
+- ▁DANIEL
+- ▁FACILIT
+- ▁FORSAKE
+- ▁BEHAVIOUR
+- ▁GORGEOUS
+- ▁PRODUCING
+- ▁HAPPIER
+- ▁PROMISING
+- ▁RAINBOW
+- ▁INSTINCTIVELY
+- ▁DECREE
+- ▁EYEBROWS
+- ▁IRRESISTIBLE
+- ▁PHARAOH
+- ▁SCROOGE
+- ▁UNNATURAL
+- ▁CRUMBS
+- ▁REFINED
+- ▁DREARY
+- ▁TRENCH
+- ▁CONVINCE
+- ▁FRINGE
+- ▁EXTREMITY
+- ▁INTIMACY
+- ▁SCOUNDREL
+- ▁SUFFRAGE
+- ▁UNEASINESS
+- ▁BARRICADE
+- ▁CIRCULAT
+- ▁SAMUEL
+- ▁BRUCE
+- ▁DARCY
+- <sos/eos>
+init: xavier_uniform
+input_size: null
+ctc_conf:
+    dropout_rate: 0.0
+    ctc_type: builtin
+    reduce: true
+    ignore_nan_grad: null
+    zero_infinity: true
+    brctc_risk_strategy: exp
+    brctc_group_strategy: end
+    brctc_risk_factor: 0.0
+joint_net_conf: null
+use_preprocessor: true
+use_lang_prompt: false
+use_nlp_prompt: false
+token_type: bpe
+bpemodel: data/en_token_list/bpe_unigram5000/bpe.model
+non_linguistic_symbols: null
+cleaner: null
+g2p: null
+speech_volume_normalize: null
+rir_scp: null
+rir_apply_prob: 1.0
+noise_scp: null
+noise_apply_prob: 1.0
+noise_db_range: '13_15'
+short_noise_thres: 0.5
+aux_ctc_tasks: []
+frontend: default
+frontend_conf:
+    fs: 16k
+specaug: specaug
+specaug_conf:
+    apply_time_warp: true
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 30
+    num_freq_mask: 2
+    apply_time_mask: true
+    time_mask_width_range:
+    - 0
+    - 40
+    num_time_mask: 2
+normalize: global_mvn
+normalize_conf:
+    stats_file: exp/asr_stats_raw_en_bpe5000_sp/train/feats_stats.npz
+model: espnet
+model_conf:
+    ctc_weight: 0.3
+    lsm_weight: 0.1
+    length_normalized_loss: false
+preencoder: null
+preencoder_conf: {}
+encoder: q_transformer
+encoder_conf:
+    output_size: 512
+    attention_heads: 8
+    attention_layer_type: HierDecayv2
+    linear_units: 2048
+    num_blocks: 18
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: conv2d6
+    normalize_before: true
+postencoder: null
+postencoder_conf: {}
+decoder: transformer
+decoder_conf:
+    attention_heads: 8
+    linear_units: 2048
+    num_blocks: 6
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    self_attention_dropout_rate: 0.1
+    src_attention_dropout_rate: 0.1
+preprocessor: default
+preprocessor_conf: {}
+required:
+- output_dir
+- token_list
+version: '202402'
+distributed: true

checkpoints/asr_transformer_HierDecayv2_woBias_XavierInit_Qv7/latest.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0dd0daaefe2355e6b0151b39c8f53b350fc4f531973a9cff2cb5dfb6fffeaddf
+size 398012507

checkpoints/asr_transformer_HierDecayv2_woBias_XavierInit_Qv7/run.sh

@@ -0,0 +1 @@
+./asr.sh --skip_data_prep true --stage 11 --lang en --ngpu 4 --use_lm false --nbpe 5000 --max_wav_duration 30 --asr_tag transformer_HierDecayv2_woBias_XavierInit_Qv7 --speed_perturb_factors '0.9 1.0 1.1' --asr_config conf/tuning/SNN/train_asr_Q_transformer3_HierDecayv2.yaml --lm_config conf/tuning/train_lm_transformer2.yaml --inference_config conf/decode_asr.yaml --train_set train_960 --valid_set dev --test_sets 'test_clean test_other dev_clean dev_other' --lm_train_text 'data/train_960/text data/local/other_text/text' --bpe_train_text data/train_960/text --stage 11 --stage 11 --stage 11 "$@"; exit $?

checkpoints/asr_transformer_HierDecayv2_woBias_XavierInit_Qv7/tensorboard/train/events.out.tfevents.1720237326.bmi2.3524061.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e389810df809c7f2dae4fa896397bb161c71f7d0a80d0f562b62cc88f0291f0
+size 140505218

checkpoints/asr_transformer_HierDecayv2_woBias_XavierInit_Qv7/tensorboard/train/events.out.tfevents.1721133711.bmi2.1781101.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c7227f05b7abc16967fc1a41727ea41d048a5d8cae4eec3ab31c9431854db80
+size 723

checkpoints/asr_transformer_HierDecayv2_woBias_XavierInit_Qv7/tensorboard/train/events.out.tfevents.1721134148.bmi2.1856035.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f12dae579bb931487ce29814644f11cbc964734e8f3cebf67052bc1c3dd8557
+size 352256376

checkpoints/asr_transformer_HierDecayv2_woBias_XavierInit_Qv7/tensorboard/valid/events.out.tfevents.1720237326.bmi2.3524061.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7df5e3f820a7014cd8b7729120f0335c00b06ff2ca558dd210df3b42c229c556
+size 27601

checkpoints/asr_transformer_HierDecayv2_woBias_XavierInit_Qv7/tensorboard/valid/events.out.tfevents.1721133711.bmi2.1781101.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d01cc65e5c3d9a8578da8dc9cca430c31d30ab82661c01f2986e23e0db71f6b1
+size 88

checkpoints/asr_transformer_HierDecayv2_woBias_XavierInit_Qv7/tensorboard/valid/events.out.tfevents.1721134148.bmi2.1856035.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f4a8940d85c376e7ea7afcb132a358fc632019cb8d9ae4f3dc960e92d1fe152
+size 10267

checkpoints/asr_transformer_HierDecayv2_woBias_XavierInit_Qv7/valid.acc.ave.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1454a34a2d05ee8c95bfe0335072731efe31608febed0aad6b2b80e633b9bc0d
+size 398011411

checkpoints/asr_transformer_HierDecayv2_woBias_XavierInit_Qv7/valid.acc.ave_10best.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1454a34a2d05ee8c95bfe0335072731efe31608febed0aad6b2b80e633b9bc0d
+size 398011411

checkpoints/asr_transformer_HierDecayv2_woBias_XavierInit_Qv7/valid.acc.best.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0dd0daaefe2355e6b0151b39c8f53b350fc4f531973a9cff2cb5dfb6fffeaddf
+size 398012507

src/Spike_driven/Q_transformer_encoder.py

@@ -0,0 +1,397 @@
+# Copyright 2019 Shigeki Karita
+#  Apache 2.0  (http://www.apache.org/licenses/LICENSE-2.0)
+
+"""Transformer encoder definition."""
+
+from typing import List, Optional, Tuple
+
+import torch
+import torch.nn as nn
+from typeguard import typechecked
+
+from espnet2.asr.ctc import CTC
+from espnet2.asr.encoder.abs_encoder import AbsEncoder
+from espnet.nets.pytorch_backend.nets_utils import make_pad_mask
+from espnet2.asr.encoder.Spike_driven.Spike_driven_modules.Q_attention import *
+
+from espnet.nets.pytorch_backend.transformer.embedding import PositionalEncoding
+from espnet.nets.pytorch_backend.transformer.layer_norm import LayerNorm
+# from espnet2.asr_transducer.normalization import RMSNorm
+from espnet.nets.pytorch_backend.transformer.multi_layer_conv import (
+    Conv1dLinear,
+    MultiLayeredConv1d,
+)
+from espnet2.asr.encoder.Spike_driven.Spike_driven_modules.Q_positionwise_feed_forward import Q_PositionwiseFeedForward, Q_GLU
+from espnet.nets.pytorch_backend.transformer.repeat import repeat
+from espnet.nets.pytorch_backend.transformer.subsampling import (
+    Conv1dSubsampling2,
+    Conv2dSubsampling,
+    Conv2dSubsampling1,
+    Conv2dSubsampling2,
+    Conv2dSubsampling6,
+    Conv2dSubsampling8,
+    TooShortUttError,
+    check_short_utt,
+)
+from espnet2.asr.encoder.Spike_driven.Q_trick import MultiSpike
+
+class Q_Transformer_EncoderLayer(nn.Module):
+    """Encoder layer module.
+
+    Args:
+        size (int): Input dimension.
+        self_attn (torch.nn.Module): Self-attention module instance.
+            `MultiHeadedAttention` or `RelPositionMultiHeadedAttention` instance
+            can be used as the argument.
+        feed_forward (torch.nn.Module): Feed-forward module instance.
+            `PositionwiseFeedForward`, `MultiLayeredConv1d`, or `Conv1dLinear` instance
+            can be used as the argument.
+        dropout_rate (float): Dropout rate.
+        normalize_before (bool): Whether to use layer_norm before the first block.
+        concat_after (bool): Whether to concat attention layer's input and output.
+            if True, additional linear will be applied.
+            i.e. x -> x + linear(concat(x, att(x)))
+            if False, no additional linear will be applied. i.e. x -> x + att(x)
+        stochastic_depth_rate (float): Proability to skip this layer.
+            During training, the layer may skip residual computation and return input
+            as-is with given probability.
+    """
+
+    def __init__(
+        self,
+        size,
+        self_attn,
+        feed_forward,
+        dropout_rate,
+        normalize_before=True,
+        concat_after=False,
+        stochastic_depth_rate=0.0,
+    ):
+        """Construct an EncoderLayer object."""
+        super(Q_Transformer_EncoderLayer, self).__init__()
+        self.self_attn = self_attn
+        self.feed_forward = feed_forward
+        self.norm1 = LayerNorm(size)
+        self.norm2 = LayerNorm(size)
+        self.dropout = nn.Dropout(dropout_rate)
+        self.size = size
+        self.normalize_before = normalize_before
+        self.concat_after = concat_after
+        if self.concat_after:
+            self.concat_linear = nn.Linear(size + size, size)
+        self.stochastic_depth_rate = stochastic_depth_rate
+        self.ATT_sn = MultiSpike(size)
+        self.FFN_sn = MultiSpike(size)
+
+    def forward(self, x, mask, iiter=None, cache=None):
+        """Compute encoded features.
+
+        Args:
+            x_input (torch.Tensor): Input tensor (#batch, time, size).
+            mask (torch.Tensor): Mask tensor for the input (#batch, 1, time).
+            cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size).
+
+        Returns:
+            torch.Tensor: Output tensor (#batch, time, size).
+            torch.Tensor: Mask tensor (#batch, 1, time).
+
+        """
+        skip_layer = False
+        # with stochastic depth, residual connection `x + f(x)` becomes
+        # `x <- x + 1 / (1 - p) * f(x)` at training time.
+        stoch_layer_coeff = 1.0
+        if self.training and self.stochastic_depth_rate > 0:
+            skip_layer = torch.rand(1).item() < self.stochastic_depth_rate
+            stoch_layer_coeff = 1.0 / (1 - self.stochastic_depth_rate)
+
+        if skip_layer:
+            if cache is not None:
+                x = torch.cat([cache, x], dim=1)
+            return x, mask
+
+        residual = x
+        if self.normalize_before:
+            x = self.norm1(x)
+
+        if cache is None:
+            x_q = x
+        else:
+            assert cache.shape == (x.shape[0], x.shape[1] - 1, self.size)
+            x_q = x[:, -1:, :]
+            residual = residual[:, -1:, :]
+            mask = None if mask is None else mask[:, -1:, :]
+
+        x_q = self.ATT_sn(x_q, iiter)
+        x = self.ATT_sn(x, iiter)
+        if self.concat_after:
+            x_concat = torch.cat((x, self.self_attn(x_q, x, x, mask, iiter)), dim=-1)
+            x = residual + stoch_layer_coeff * self.concat_linear(x_concat)
+        else:
+            x = residual + stoch_layer_coeff * self.dropout(
+                self.self_attn(x_q, x, x, mask, iiter)
+            )
+        if not self.normalize_before:
+            x = self.norm1(x)
+
+        residual = x
+        x = self.FFN_sn(x, iiter)
+        if self.normalize_before:
+            x = self.norm2(x)
+        x = residual + stoch_layer_coeff * self.dropout(self.feed_forward(x, iiter))
+        if not self.normalize_before:
+            x = self.norm2(x)
+
+        if cache is not None:
+            x = torch.cat([cache, x], dim=1)
+
+        return x, mask
+
+
+class Q_TransformerEncoder(AbsEncoder):
+    """Transformer encoder module.
+
+    Args:
+        input_size: input dim
+        output_size: dimension of attention
+        attention_heads: the number of heads of multi head attention
+        linear_units: the number of units of position-wise feed forward
+        num_blocks: the number of decoder blocks
+        dropout_rate: dropout rate
+        attention_dropout_rate: dropout rate in attention
+        positional_dropout_rate: dropout rate after adding positional encoding
+        input_layer: input layer type
+        pos_enc_class: PositionalEncoding or ScaledPositionalEncoding
+        normalize_before: whether to use layer_norm before the first block
+        concat_after: whether to concat attention layer's input and output
+            if True, additional linear will be applied.
+            i.e. x -> x + linear(concat(x, att(x)))
+            if False, no additional linear will be applied.
+            i.e. x -> x + att(x)
+        positionwise_layer_type: linear of conv1d
+        positionwise_conv_kernel_size: kernel size of positionwise conv1d layer
+        padding_idx: padding_idx for input_layer=embed
+    """
+
+    @typechecked
+    def __init__(
+        self,
+        input_size: int,
+        output_size: int = 256,
+        attention_heads: int = 4,
+        attention_layer_type: str = "selfattn",
+        linear_units: int = 2048,
+        num_blocks: int = 6,
+        dropout_rate: float = 0.1,
+        positional_dropout_rate: float = 0.1,
+        attention_dropout_rate: float = 0.0,
+        input_layer: Optional[str] = "conv2d",
+        pos_enc_class=PositionalEncoding,
+        normalize_before: bool = True,
+        concat_after: bool = False,
+        positionwise_layer_type: str = "FFN",
+        padding_idx: int = -1,
+        interctc_layer_idx: List[int] = [],
+        interctc_use_conditioning: bool = False,
+        layer_drop_rate: float = 0.0,
+    ):
+        super().__init__()
+        self._output_size = output_size
+
+        if input_layer == "linear":
+            self.embed = torch.nn.Sequential(
+                torch.nn.Linear(input_size, output_size),
+                torch.nn.LayerNorm(output_size),
+                torch.nn.Dropout(dropout_rate),
+                torch.nn.ReLU(),
+                pos_enc_class(output_size, positional_dropout_rate),
+            )
+        elif input_layer == "conv1d2":
+            self.embed = Conv1dSubsampling2(
+                input_size,
+                output_size,
+                dropout_rate,
+                pos_enc_class(output_size, positional_dropout_rate),
+            )
+        elif input_layer == "conv2d":
+            self.embed = Conv2dSubsampling(input_size, output_size, dropout_rate)
+        elif input_layer == "conv2d1":
+            self.embed = Conv2dSubsampling1(input_size, output_size, dropout_rate)
+        elif input_layer == "conv2d2":
+            self.embed = Conv2dSubsampling2(input_size, output_size, dropout_rate)
+        elif input_layer == "conv2d6":
+            self.embed = Conv2dSubsampling6(input_size, output_size, dropout_rate)
+        elif input_layer == "conv2d8":
+            self.embed = Conv2dSubsampling8(input_size, output_size, dropout_rate)
+        elif input_layer == "embed":
+            self.embed = torch.nn.Sequential(
+                torch.nn.Embedding(input_size, output_size, padding_idx=padding_idx),
+                pos_enc_class(output_size, positional_dropout_rate),
+            )
+        elif input_layer is None:
+            if input_size == output_size:
+                self.embed = None
+            else:
+                self.embed = torch.nn.Linear(input_size, output_size)
+        else:
+            raise ValueError("unknown input_layer: " + input_layer)
+        self.normalize_before = normalize_before
+        if attention_layer_type == "selfattn":
+            encoder_selfattn_layer = Q_MultiHeadedAttention
+            encoder_selfattn_layer_args = (
+                attention_heads, 
+                output_size, 
+                attention_dropout_rate
+            )
+        elif attention_layer_type == "selfattn_woSoftMax":
+            encoder_selfattn_layer = Q_MultiHeadedAttention_woSoftMax
+            encoder_selfattn_layer_args = (
+                attention_heads, 
+                output_size, 
+                attention_dropout_rate
+            ) 
+        elif attention_layer_type == "HierDecayv2":
+            encoder_selfattn_layer = Q_MultiHeadedAttention_HierDecay
+            encoder_selfattn_layer_args = (
+                attention_heads,
+                output_size,
+                attention_dropout_rate,
+            )
+        elif attention_layer_type == "HierDecay_woSoftMax":
+            encoder_selfattn_layer = Q_MultiHeadedAttention_HierDecay_woSoftMax
+            encoder_selfattn_layer_args = (
+                attention_heads,
+                output_size,
+                attention_dropout_rate,
+            )
+        
+        else:
+            raise ValueError("unknown encoder_attn_layer: " + attention_layer_type)
+
+        positionwise_layer = Q_PositionwiseFeedForward
+        positionwise_layer_args = (
+            output_size,
+            linear_units,
+            dropout_rate,
+        )
+        
+
+        if "HierDecay" in attention_layer_type:
+            self.encoders = repeat(
+                num_blocks,
+                lambda lnum: Q_Transformer_EncoderLayer(
+                    output_size,
+                    encoder_selfattn_layer(*encoder_selfattn_layer_args, lnum),
+                    positionwise_layer(*positionwise_layer_args),
+                    dropout_rate,
+                    normalize_before,
+                    concat_after,
+                ),
+                layer_drop_rate,
+            )
+        else:
+            self.encoders = repeat(
+                num_blocks,
+                lambda lnum: Q_Transformer_EncoderLayer(
+                    output_size,
+                    encoder_selfattn_layer(*encoder_selfattn_layer_args),
+                    positionwise_layer(*positionwise_layer_args),
+                    dropout_rate,
+                    normalize_before,
+                    concat_after,
+                ),
+                layer_drop_rate,
+            )
+        
+        if self.normalize_before:
+            self.after_norm = LayerNorm(output_size)
+
+        self.interctc_layer_idx = interctc_layer_idx
+        if len(interctc_layer_idx) > 0:
+            assert 0 < min(interctc_layer_idx) and max(interctc_layer_idx) < num_blocks
+        self.interctc_use_conditioning = interctc_use_conditioning
+        self.conditioning_layer = None
+
+    def output_size(self) -> int:
+        return self._output_size
+
+    def forward(
+        self,
+        xs_pad: torch.Tensor,
+        ilens: torch.Tensor,
+        iiter: int = 0,
+        prev_states: torch.Tensor = None,
+        ctc: CTC = None,
+        return_all_hs: bool = False,
+    ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
+        """Embed positions in tensor.
+
+        Args:
+            xs_pad: input tensor (B, L, D)
+            ilens: input length (B)
+            prev_states: Not to be used now.
+            ctc (CTC): ctc module for intermediate CTC loss
+            return_all_hs (bool): whether to return all hidden states
+
+        Returns:
+            position embedded tensor and mask
+        """
+        masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device)
+        # print('iiter:{}'.format(iiter))
+        if self.embed is None:
+            xs_pad = xs_pad
+        elif (
+            isinstance(self.embed, Conv2dSubsampling)
+            or isinstance(self.embed, Conv1dSubsampling2)
+            or isinstance(self.embed, Conv2dSubsampling1)
+            or isinstance(self.embed, Conv2dSubsampling2)
+            or isinstance(self.embed, Conv2dSubsampling6)
+            or isinstance(self.embed, Conv2dSubsampling8)
+        ):
+            short_status, limit_size = check_short_utt(self.embed, xs_pad.size(1))
+            if short_status:
+                raise TooShortUttError(
+                    f"has {xs_pad.size(1)} frames and is too short for subsampling "
+                    + f"(it needs more than {limit_size} frames), return empty results",
+                    xs_pad.size(1),
+                    limit_size,
+                )
+            xs_pad, masks = self.embed(xs_pad, masks)
+        else:
+            xs_pad = self.embed(xs_pad)
+        intermediate_outs = []
+        if len(self.interctc_layer_idx) == 0:
+            for layer_idx, encoder_layer in enumerate(self.encoders):
+                xs_pad, masks = encoder_layer(xs_pad, masks, iiter)
+                if return_all_hs:
+                    if isinstance(xs_pad, tuple):
+                        intermediate_outs.append(xs_pad[0])
+                    else:
+                        intermediate_outs.append(xs_pad)
+
+                
+        else:
+            for layer_idx, encoder_layer in enumerate(self.encoders):
+                xs_pad, masks = encoder_layer(xs_pad, masks, iiter)
+
+                if layer_idx + 1 in self.interctc_layer_idx:
+                    encoder_out = xs_pad
+
+                    # intermediate outputs are also normalized
+                    if self.normalize_before:
+                        encoder_out = self.after_norm(encoder_out)
+
+                    intermediate_outs.append((layer_idx + 1, encoder_out))
+
+                    if self.interctc_use_conditioning:
+                        ctc_out = ctc.softmax(encoder_out)
+                        xs_pad = xs_pad + self.conditioning_layer(ctc_out)
+
+        if self.normalize_before:
+            xs_pad = self.after_norm(xs_pad)
+
+        olens = masks.squeeze(1).sum(1)
+        # from IPython import embed; embed()
+        if len(intermediate_outs) > 0:
+            return (xs_pad, intermediate_outs), olens, None
+        
+        return xs_pad, olens, None

src/Spike_driven/Q_trick.py

@@ -0,0 +1,46 @@
+import torch
+import torch.nn as nn
+
+class quant(torch.autograd.Function):
+    @staticmethod
+    def forward(ctx, input, T):
+        ctx.save_for_backward(input)
+        ctx.T = T
+        return torch.round(torch.clamp(input, min=0, max=T))
+        
+    @staticmethod
+    def backward(ctx, grad_output):
+        input, = ctx.saved_tensors
+        grad_input = grad_output.clone()
+        grad_input[input < 0] = 0
+        grad_input[input > ctx.T] = 0
+        return grad_input, None
+
+
+
+class MultiSpike(torch.nn.Module):
+    def __init__(self, dim: int, T=4):
+        super().__init__()
+        self.T = T
+        self.spike = quant()
+        self.momentum = 0.1
+        self.eps = 1e-5
+        self.register_buffer("running_stats", torch.zeros(dim))
+    
+    def __repr__(self):
+        return f"MultiSpike(T={self.T})"    
+    
+    def forward(self, x, iiter=0):
+        #v7
+        # print('iiter:{}'.format(iiter))
+        if self.training:
+            Stats = x.max(dim=0).values.max(dim=0).values
+            # Stats = x.abs().mean(dim=[0,1])
+            with torch.no_grad():
+                self.running_stats = self.momentum * Stats + (1-self.momentum) * self.running_stats
+        else:
+            Stats = self.running_stats
+
+        scale = self.T / (Stats[None, None, :] + self.eps)
+
+        return self.spike.apply(scale* x, self.T) / scale

src/Spike_driven/Spike_driven_modules/Q_attention.py

@@ -0,0 +1,455 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# Copyright 2019 Shigeki Karita
+#  Apache 2.0  (http://www.apache.org/licenses/LICENSE-2.0)
+
+"""Multi-Head Attention layer definition."""
+
+import math
+import numpy
+import torch
+from torch import nn
+from espnet2.asr.encoder.Spike_driven.Q_trick import MultiSpike
+
+
+
+class Q_MultiHeadedAttention_HierDecay(nn.Module):
+    """Implementation of HD-RepSSA_S
+
+    Args:
+        n_head (int): The number of heads.
+        n_feat (int): The number of features.
+        dropout_rate (float): Dropout rate.
+        layer_id (int): Layer ID for decay calculation.
+
+    """
+
+    def __init__(self, n_head, n_feat, dropout_rate, layer_id):
+        """Construct an MultiHeadedAttention object."""
+        super(Q_MultiHeadedAttention_HierDecay_v2, self).__init__()
+        assert n_feat % n_head == 0
+        # We assume d_v always equals d_k
+        self.d_k = n_feat // n_head
+        self.h = n_head
+        self.linear_q = nn.Linear(n_feat, n_feat, bias=False)
+        self.linear_k = nn.Linear(n_feat, n_feat, bias=False)
+        self.linear_v = nn.Linear(n_feat, n_feat, bias=False)
+        self.v_sn = MultiSpike(n_feat)
+        self.output_sn = MultiSpike(n_feat)
+        self.linear_out = nn.Linear(n_feat, n_feat)
+        self.attn = None
+        self.dropout = nn.Dropout(p=dropout_rate)
+        
+        # Hierarchical decay calculation
+        layer_decay = 1 - 2 ** (-5 - layer_id)
+        decay = torch.log(torch.tensor(layer_decay).repeat(n_head))
+        self.register_buffer("decay", decay)
+
+    def forward_qkv(self, query, key, value, iiter):
+        """Transform query, key and value.
+
+        Args:
+            query (torch.Tensor): Query tensor (#batch, time1, size).
+            key (torch.Tensor): Key tensor (#batch, time2, size).
+            value (torch.Tensor): Value tensor (#batch, time2, size).
+
+        Returns:
+            torch.Tensor: Transformed query tensor (#batch, n_head, time1, d_k).
+            torch.Tensor: Transformed key tensor (#batch, n_head, time2, d_k).
+            torch.Tensor: Transformed value tensor (#batch, n_head, time2, d_k).
+
+        """
+        n_batch = query.size(0)
+        q = self.linear_q(query).view(n_batch, -1, self.h, self.d_k)
+        k = self.linear_k(key).view(n_batch, -1, self.h, self.d_k)
+        v = self.v_sn(self.linear_v(value)).view(n_batch, -1, self.h, self.d_k)
+        q = q.transpose(1, 2)  # (batch, head, time1, d_k)
+        k = k.transpose(1, 2)  # (batch, head, time2, d_k)
+        v = v.transpose(1, 2)  # (batch, head, time2, d_k)
+        
+        return q, k, v
+
+    def forward_attention(self, value, scores, mask, inner_mask, iiter):
+        """Compute attention context vector.
+
+        Args:
+            value (torch.Tensor): Transformed value (#batch, n_head, time2, d_k).
+            scores (torch.Tensor): Attention score (#batch, n_head, time1, time2).
+            mask (torch.Tensor): Mask (#batch, 1, time2) or (#batch, time1, time2).
+            inner_mask (torch.Tensor): Inner mask for hierarchical decay.
+
+        Returns:
+            torch.Tensor: Transformed value (#batch, time1, d_model)
+                weighted by the attention score (#batch, time1, time2).
+
+        """
+        n_batch = value.size(0)
+        scores += inner_mask
+        if mask is not None:
+            mask = mask.unsqueeze(1).eq(0)  # (batch, 1, *, time2)
+            min_value = torch.finfo(scores.dtype).min
+            scores = scores.masked_fill(mask, min_value)
+            self.attn = torch.softmax(scores, dim=-1).masked_fill(
+                mask, 0.0
+            )  # (batch, head, time1, time2)
+        else:
+            self.attn = torch.softmax(scores, dim=-1)  # (batch, head, time1, time2)
+        p_attn = self.dropout(self.attn)
+
+        x = torch.matmul(p_attn, value)  # (batch, head, time1, d_k)
+        x = (
+            x.transpose(1, 2).contiguous().view(n_batch, -1, self.h * self.d_k)
+        )  # (batch, time1, d_model)
+        x = self.output_sn(x)
+        return self.linear_out(x)  # (batch, time1, d_model)
+    
+    def forward(self, query, key, value, mask, iiter):
+        """Compute scaled dot product attention.
+
+        Args:
+            query (torch.Tensor): Query tensor (#batch, time1, size).
+            key (torch.Tensor): Key tensor (#batch, time2, size).
+            value (torch.Tensor): Value tensor (#batch, time2, size).
+            mask (torch.Tensor): Mask tensor (#batch, 1, time2) or
+                (#batch, time1, time2).
+
+        Returns:
+            torch.Tensor: Output tensor (#batch, time1, d_model).
+
+        """
+        slen = query.shape[1] if query.shape[1]==key.shape[1] else mask.shape[1]
+        index = torch.arange(slen).to(self.decay)
+        inner_mask = torch.abs(index.view(slen,1) - index.view(1, slen))
+        inner_mask = inner_mask * self.decay[:, None, None]
+
+        q, k, v = self.forward_qkv(query, key, value, iiter)
+        scores = torch.matmul(q, k.transpose(-2, -1))/ math.sqrt(self.d_k)
+        return self.forward_attention(v, scores, mask, inner_mask, iiter)
+
+class Q_MultiHeadedAttention_HierDecay_woSoftMax(Q_MultiHeadedAttention):
+    """Implementation of HD-RepSSA_S
+
+    Args:
+        n_head (int): The number of heads.
+        n_feat (int): The number of features.
+        dropout_rate (float): Dropout rate.
+        layer_id (int): Layer ID for decay calculation.
+
+    """
+
+    def __init__(self, n_head, n_feat, dropout_rate, layer_id):
+        """Construct an MultiHeadedAttention object."""
+        
+        super().__init__(n_head, n_feat, dropout_rate)
+        assert n_feat % n_head == 0
+        # We assume d_v always equals d_k
+        self.d_k = n_feat // n_head
+        self.h = n_head
+        self.linear_q = nn.Linear(n_feat, n_feat, bias=False)
+        self.linear_k = nn.Linear(n_feat, n_feat, bias=False)
+        self.linear_v = nn.Linear(n_feat, n_feat, bias=False)
+        self.q_sn = MultiSpike(n_feat)
+        self.k_sn = MultiSpike(n_feat)
+        self.v_sn = MultiSpike(n_feat)
+        self.output_sn = MultiSpike(n_feat)
+        self.linear_out = nn.Linear(n_feat, n_feat)
+        self.attn = None
+        self.dropout = nn.Dropout(p=dropout_rate)
+        
+        layer_decay = 1 - 2 ** (-5 - layer_id)
+        
+        decay = torch.log(torch.tensor(layer_decay).repeat(n_head))
+        self.register_buffer("decay", decay)
+
+        self.ln = torch.nn.LayerNorm(self.d_k)
+
+
+    def forward_qkv(self, query, key, value, iiter):
+        """Transform query, key and value.
+
+        Args:
+            query (torch.Tensor): Query tensor (#batch, time1, size).
+            key (torch.Tensor): Key tensor (#batch, time2, size).
+            value (torch.Tensor): Value tensor (#batch, time2, size).
+
+        Returns:
+            torch.Tensor: Transformed query tensor (#batch, n_head, time1, d_k).
+            torch.Tensor: Transformed key tensor (#batch, n_head, time2, d_k).
+            torch.Tensor: Transformed value tensor (#batch, n_head, time2, d_k).
+
+        """
+        n_batch = query.size(0)
+        q = self.q_sn(self.linear_q(query)).view(n_batch, -1, self.h, self.d_k)
+        k = self.k_sn(self.linear_k(key)).view(n_batch, -1, self.h, self.d_k)
+        v = self.v_sn(self.linear_v(value)).view(n_batch, -1, self.h, self.d_k)
+        q = q.transpose(1, 2)  # (batch, head, time1, d_k)
+        k = k.transpose(1, 2)  # (batch, head, time2, d_k)
+        v = v.transpose(1, 2)  # (batch, head, time2, d_k)
+        
+        return q, k, v
+
+    def forward_attention(self, value, scores, mask, inner_mask, iiter):
+        """Compute attention context vector.
+
+        Args:
+            value (torch.Tensor): Transformed value (#batch, n_head, time2, d_k).
+            scores (torch.Tensor): Attention score (#batch, n_head, time1, time2).
+            mask (torch.Tensor): Mask (#batch, 1, time2) or (#batch, time1, time2).
+
+        Returns:
+            torch.Tensor: Transformed value (#batch, time1, d_model)
+                weighted by the attention score (#batch, time1, time2).
+
+        """
+        n_batch = value.size(0)
+        scores = scores / scores.detach().abs().sum(dim=-1, keepdim=True).clamp(min=1, max=5e4)
+        if mask is not None:
+            mask = mask.unsqueeze(1).eq(0)  # (batch, 1, *, time2)
+            min_value = torch.finfo(scores.dtype).min
+            scores = scores.masked_fill(mask, min_value)
+            self.attn = scores.masked_fill(
+                mask, 0.0
+            )  # (batch, head, time1, time2)
+        else:
+            self.attn = scores  # (batch, head, time1, time2)
+        self.attn = inner_mask * self.attn
+        p_attn = self.dropout(self.attn)
+
+        x = self.ln(torch.matmul(p_attn, value))  # (batch, head, time1, d_k)
+        x = (
+            x.transpose(1, 2).contiguous().view(n_batch, -1, self.h * self.d_k)
+        )  # (batch, time1, d_model)
+        x = self.output_sn(x)
+        return self.linear_out(x)  # (batch, time1, d_model)
+
+    def forward(self, query, key, value, mask, iiter):
+        """Compute scaled dot product attention.
+
+        Args:
+            query (torch.Tensor): Query tensor (#batch, time1, size).
+            key (torch.Tensor): Key tensor (#batch, time2, size).
+            value (torch.Tensor): Value tensor (#batch, time2, size).
+            mask (torch.Tensor): Mask tensor (#batch, 1, time2) or
+                (#batch, time1, time2).
+
+        Returns:
+            torch.Tensor: Output tensor (#batch, time1, d_model).
+
+        """
+        slen = query.shape[1] if query.shape[1]==key.shape[1] else mask.shape[1]
+        index = torch.arange(slen).to(self.decay)
+        inner_mask = torch.abs(index.view(slen,1) - index.view(1, slen))
+        inner_mask = torch.exp(inner_mask * self.decay[:, None, None])
+
+        q, k, v = self.forward_qkv(query, key, value, iiter)
+        scores = torch.matmul(q, k.transpose(-2, -1))
+        return self.forward_attention(v, scores, mask, inner_mask, iiter)
+
+class Q_MultiHeadedAttention(nn.Module):
+    """Multi-Head Attention layer.
+
+    Args:
+        n_head (int): The number of heads.
+        n_feat (int): The number of features.
+        dropout_rate (float): Dropout rate.
+
+    """
+
+    def __init__(self, n_head, n_feat, dropout_rate):
+        """Construct an MultiHeadedAttention object."""
+        super(Q_MultiHeadedAttention, self).__init__()
+        assert n_feat % n_head == 0
+        # We assume d_v always equals d_k
+        self.d_k = n_feat // n_head
+        self.h = n_head
+        self.linear_q = nn.Linear(n_feat, n_feat, bias=False)
+        self.linear_k = nn.Linear(n_feat, n_feat, bias=False)
+        self.linear_v = nn.Linear(n_feat, n_feat, bias=False)
+        self.v_sn = MultiSpike(n_feat)
+        self.output_sn = MultiSpike(n_feat)
+        self.linear_out = nn.Linear(n_feat, n_feat)
+        self.attn = None
+        self.dropout = nn.Dropout(p=dropout_rate)
+
+    def forward_qkv(self, query, key, value, iiter):
+        """Transform query, key and value.
+
+        Args:
+            query (torch.Tensor): Query tensor (#batch, time1, size).
+            key (torch.Tensor): Key tensor (#batch, time2, size).
+            value (torch.Tensor): Value tensor (#batch, time2, size).
+
+        Returns:
+            torch.Tensor: Transformed query tensor (#batch, n_head, time1, d_k).
+            torch.Tensor: Transformed key tensor (#batch, n_head, time2, d_k).
+            torch.Tensor: Transformed value tensor (#batch, n_head, time2, d_k).
+
+        """
+        n_batch = query.size(0)
+        q = self.linear_q(query).view(n_batch, -1, self.h, self.d_k)
+        k = self.linear_k(key).view(n_batch, -1, self.h, self.d_k)
+        v = self.v_sn(self.linear_v(value)).view(n_batch, -1, self.h, self.d_k)
+        q = q.transpose(1, 2)  # (batch, head, time1, d_k)
+        k = k.transpose(1, 2)  # (batch, head, time2, d_k)
+        v = v.transpose(1, 2)  # (batch, head, time2, d_k)
+
+        return q, k, v
+
+    def forward_attention(self, value, scores, mask, iiter):
+        """Compute attention context vector.
+
+        Args:
+            value (torch.Tensor): Transformed value (#batch, n_head, time2, d_k).
+            scores (torch.Tensor): Attention score (#batch, n_head, time1, time2).
+            mask (torch.Tensor): Mask (#batch, 1, time2) or (#batch, time1, time2).
+
+        Returns:
+            torch.Tensor: Transformed value (#batch, time1, d_model)
+                weighted by the attention score (#batch, time1, time2).
+
+        """
+        n_batch = value.size(0)
+        if mask is not None:
+            mask = mask.unsqueeze(1).eq(0)  # (batch, 1, *, time2)
+            min_value = torch.finfo(scores.dtype).min
+            scores = scores.masked_fill(mask, min_value)
+            self.attn = torch.softmax(scores, dim=-1).masked_fill(
+                mask, 0.0
+            )  # (batch, head, time1, time2)
+        else:
+            self.attn = torch.softmax(scores, dim=-1)  # (batch, head, time1, time2)
+
+        p_attn = self.dropout(self.attn)
+        x = torch.matmul(p_attn, value)  # (batch, head, time1, d_k)
+        
+        x = (
+            x.transpose(1, 2).contiguous().view(n_batch, -1, self.h * self.d_k)
+        )  # (batch, time1, d_model)
+        x = self.output_sn(x)
+        return self.linear_out(x)  # (batch, time1, d_model)
+
+    def forward(self, query, key, value, mask, iiter):
+        """Compute scaled dot product attention.
+
+        Args:
+            query (torch.Tensor): Query tensor (#batch, time1, size).
+            key (torch.Tensor): Key tensor (#batch, time2, size).
+            value (torch.Tensor): Value tensor (#batch, time2, size).
+            mask (torch.Tensor): Mask tensor (#batch, 1, time2) or
+                (#batch, time1, time2).
+
+        Returns:
+            torch.Tensor: Output tensor (#batch, time1, d_model).
+
+        """
+        q, k, v = self.forward_qkv(query, key, value, iiter)
+        scores = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(self.d_k)
+        return self.forward_attention(v, scores, mask, iiter)
+
+
+
+class Q_MultiHeadedAttention_woSoftMax(Q_MultiHeadedAttention):
+    """Multi-Head Attention layer without SoftMax.
+
+    Args:
+        n_head (int): The number of heads.
+        n_feat (int): The number of features.
+        dropout_rate (float): Dropout rate.
+
+    """
+
+    def __init__(self, n_head, n_feat, dropout_rate):
+        """Construct an MultiHeadedAttention object."""
+        super().__init__(n_head, n_feat, dropout_rate)
+        assert n_feat % n_head == 0
+        # We assume d_v always equals d_k
+        self.d_k = n_feat // n_head
+        self.h = n_head
+        self.linear_q = nn.Linear(n_feat, n_feat, bias=False)
+        self.linear_k = nn.Linear(n_feat, n_feat, bias=False)
+        self.linear_v = nn.Linear(n_feat, n_feat, bias=False)
+        self.q_sn = MultiSpike(n_feat)
+        self.k_sn = MultiSpike(n_feat)
+        self.v_sn = MultiSpike(n_feat)
+        self.output_sn = MultiSpike(n_feat)
+        self.linear_out = nn.Linear(n_feat, n_feat)
+        self.attn = None
+        # self.dropout = nn.Dropout(p=dropout_rate)
+        self.ln = torch.nn.LayerNorm(self.d_k)
+        # self.scale = self.d_k ** -0.5
+    def forward_qkv(self, query, key, value, iiter):
+        """Transform query, key and value.
+
+        Args:
+            query (torch.Tensor): Query tensor (#batch, time1, size).
+            key (torch.Tensor): Key tensor (#batch, time2, size).
+            value (torch.Tensor): Value tensor (#batch, time2, size).
+
+        Returns:
+            torch.Tensor: Transformed query tensor (#batch, n_head, time1, d_k).
+            torch.Tensor: Transformed key tensor (#batch, n_head, time2, d_k).
+            torch.Tensor: Transformed value tensor (#batch, n_head, time2, d_k).
+
+        """
+        n_batch = query.size(0)
+        k = self.linear_k(key).view(n_batch, -1, self.h, self.d_k)
+        v = self.linear_v(value).view(n_batch, -1, self.h, self.d_k)
+        q = self.q_sn(self.linear_q(query)).view(n_batch, -1, self.h, self.d_k)
+        q = q.transpose(1, 2)  # (batch, head, time1, d_k)
+        k = k.transpose(1, 2)  # (batch, head, time2, d_k)
+        v = v.transpose(1, 2)  # (batch, head, time2, d_k)
+
+        return q, k, v
+
+    def forward_attention(self, value, scores, mask, iiter):
+        """Compute attention context vector.
+
+        Args:
+            value (torch.Tensor): Transformed value (#batch, n_head, time2, d_k).
+            scores (torch.Tensor): Attention score (#batch, n_head, time1, time2).
+            mask (torch.Tensor): Mask (#batch, 1, time2) or (#batch, time1, time2).
+
+        Returns:
+            torch.Tensor: Transformed value (#batch, time1, d_model)
+                weighted by the attention score (#batch, time1, time2).
+
+        """
+        n_batch = value.size(0)
+        scores = scores / scores.detach().abs().sum(dim=-1, keepdim=True).clamp(min=1, max=5e4)
+        if mask is not None:
+            mask = mask.unsqueeze(1).eq(0)  # (batch, 1, *, time2)
+            min_value = torch.finfo(scores.dtype).min
+            scores = scores.masked_fill(mask, min_value)
+            self.attn = scores.masked_fill(
+                mask, 0.0
+            )  # (batch, head, time1, time2)
+        else:
+            self.attn = scores  # (batch, head, time1, time2)
+        p_attn = self.dropout(self.attn)
+        x = self.ln(torch.matmul(p_attn, value))# (batch, head, time1, d_k)
+        # x = torch.matmul(p_attn, value) * self.scale# (batch, head, time1, d_k)
+        x = (
+            x.transpose(1, 2).contiguous().view(n_batch, -1, self.h * self.d_k)
+        )  # (batch, time1, d_model)
+        x = self.output_sn(x)
+        return self.linear_out(x)  # (batch, time1, d_model)
+
+    def forward(self, query, key, value, mask, iiter):
+        """Compute scaled dot product attention.
+
+        Args:
+            query (torch.Tensor): Query tensor (#batch, time1, size).
+            key (torch.Tensor): Key tensor (#batch, time2, size).
+            value (torch.Tensor): Value tensor (#batch, time2, size).
+            mask (torch.Tensor): Mask tensor (#batch, 1, time2) or
+                (#batch, time1, time2).
+
+        Returns:
+            torch.Tensor: Output tensor (#batch, time1, d_model).
+
+        """
+        q, k, v = self.forward_qkv(query, key, value, iiter)
+        scores = torch.matmul(q, k.transpose(-2, -1))
+        return self.forward_attention(v, scores, mask, iiter)
+

src/Spike_driven/Spike_driven_modules/Q_positionwise_feed_forward.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# Copyright 2019 Shigeki Karita
+#  Apache 2.0  (http://www.apache.org/licenses/LICENSE-2.0)
+
+"""Positionwise feed forward layer definition."""
+
+import torch
+from torch import nn
+from espnet2.asr.encoder.Spike_driven.Q_trick import MultiSpike,
+
+class Q_PositionwiseFeedForward(torch.nn.Module):
+    """Positionwise feed forward layer.
+
+    Args:
+        idim (int): Input dimenstion.
+        hidden_units (int): The number of hidden units.
+        dropout_rate (float): Dropout rate.
+
+    """
+
+    def __init__(self, idim, hidden_units, dropout_rate):
+        """Construct an PositionwiseFeedForward object."""
+        super(Q_PositionwiseFeedForward, self).__init__()
+        self.w_1 = torch.nn.Linear(idim, hidden_units)
+        self.w_2 = torch.nn.Linear(hidden_units, idim)
+        self.activation = MultiSpike(hidden_units)
+
+    def forward(self, x, iiter):
+        """Forward function."""
+        x = self.w_1(x)
+        x = self.activation(x)
+        x = self.w_2(x)
+        return x

src/asr.py

@@ -0,0 +1,635 @@
+import argparse
+import logging
+from typing import Callable, Collection, Dict, List, Optional, Tuple
+
+import numpy as np
+import torch
+from typeguard import typechecked
+
+from espnet2.asr.ctc import CTC
+from espnet2.asr.decoder.abs_decoder import AbsDecoder
+from espnet2.asr.decoder.hugging_face_transformers_decoder import (  # noqa: H301
+    HuggingFaceTransformersDecoder,
+)
+from espnet2.asr.decoder.mlm_decoder import MLMDecoder
+from espnet2.asr.decoder.rnn_decoder import RNNDecoder
+from espnet2.asr.decoder.s4_decoder import S4Decoder
+from espnet2.asr.decoder.transducer_decoder import TransducerDecoder
+from espnet2.asr.decoder.transformer_decoder import (
+    DynamicConvolution2DTransformerDecoder,
+    DynamicConvolutionTransformerDecoder,
+    LightweightConvolution2DTransformerDecoder,
+    LightweightConvolutionTransformerDecoder,
+    TransformerDecoder,
+)
+from espnet2.asr.decoder.whisper_decoder import OpenAIWhisperDecoder
+from espnet2.asr.encoder.abs_encoder import AbsEncoder
+from espnet2.asr.encoder.avhubert_encoder import FairseqAVHubertEncoder
+from espnet2.asr.encoder.branchformer_encoder import BranchformerEncoder
+from espnet2.asr.encoder.conformer_encoder import ConformerEncoder
+from espnet2.asr.encoder.contextual_block_conformer_encoder import (
+    ContextualBlockConformerEncoder,
+)
+from espnet2.asr.encoder.contextual_block_transformer_encoder import (
+    ContextualBlockTransformerEncoder,
+)
+from espnet2.asr.encoder.e_branchformer_encoder import EBranchformerEncoder
+from espnet2.asr.encoder.hubert_encoder import (
+    FairseqHubertEncoder,
+    FairseqHubertPretrainEncoder,
+    TorchAudioHuBERTPretrainEncoder,
+)
+from espnet2.asr.encoder.longformer_encoder import LongformerEncoder
+from espnet2.asr.encoder.rnn_encoder import RNNEncoder
+from espnet2.asr.encoder.transformer_encoder import TransformerEncoder
+from espnet2.asr.encoder.Spike_driven.Q_transformer_encoder import Q_TransformerEncoder
+from espnet2.asr.encoder.transformer_encoder_multispkr import (
+    TransformerEncoder as TransformerEncoderMultiSpkr,
+)
+from espnet2.asr.encoder.vgg_rnn_encoder import VGGRNNEncoder
+from espnet2.asr.encoder.wav2vec2_encoder import FairSeqWav2Vec2Encoder
+from espnet2.asr.encoder.whisper_encoder import OpenAIWhisperEncoder
+from espnet2.asr.espnet_model import ESPnetASRModel
+from espnet2.asr.frontend.abs_frontend import AbsFrontend
+from espnet2.asr.frontend.default import DefaultFrontend
+from espnet2.asr.frontend.fused import FusedFrontends
+from espnet2.asr.frontend.s3prl import S3prlFrontend
+from espnet2.asr.frontend.whisper import WhisperFrontend
+from espnet2.asr.frontend.windowing import SlidingWindow
+from espnet2.asr.maskctc_model import MaskCTCModel
+from espnet2.asr.pit_espnet_model import ESPnetASRModel as PITESPnetModel
+from espnet2.asr.postencoder.abs_postencoder import AbsPostEncoder
+from espnet2.asr.postencoder.hugging_face_transformers_postencoder import (
+    HuggingFaceTransformersPostEncoder,
+)
+from espnet2.asr.postencoder.length_adaptor_postencoder import LengthAdaptorPostEncoder
+from espnet2.asr.preencoder.abs_preencoder import AbsPreEncoder
+from espnet2.asr.preencoder.linear import LinearProjection
+from espnet2.asr.preencoder.sinc import LightweightSincConvs
+from espnet2.asr.specaug.abs_specaug import AbsSpecAug
+from espnet2.asr.specaug.specaug import SpecAug
+from espnet2.asr_transducer.joint_network import JointNetwork
+from espnet2.layers.abs_normalize import AbsNormalize
+from espnet2.layers.global_mvn import GlobalMVN
+from espnet2.layers.utterance_mvn import UtteranceMVN
+from espnet2.tasks.abs_task import AbsTask
+from espnet2.text.phoneme_tokenizer import g2p_choices
+from espnet2.torch_utils.initialize import initialize
+from espnet2.train.abs_espnet_model import AbsESPnetModel
+from espnet2.train.class_choices import ClassChoices
+from espnet2.train.collate_fn import CommonCollateFn
+from espnet2.train.preprocessor import (
+    AbsPreprocessor,
+    CommonPreprocessor,
+    CommonPreprocessor_multi,
+)
+from espnet2.train.trainer import Trainer
+from espnet2.utils.get_default_kwargs import get_default_kwargs
+from espnet2.utils.nested_dict_action import NestedDictAction
+from espnet2.utils.types import float_or_none, int_or_none, str2bool, str_or_none
+
+frontend_choices = ClassChoices(
+    name="frontend",
+    classes=dict(
+        default=DefaultFrontend,
+        sliding_window=SlidingWindow,
+        s3prl=S3prlFrontend,
+        fused=FusedFrontends,
+        whisper=WhisperFrontend,
+    ),
+    type_check=AbsFrontend,
+    default="default",
+)
+specaug_choices = ClassChoices(
+    name="specaug",
+    classes=dict(
+        specaug=SpecAug,
+    ),
+    type_check=AbsSpecAug,
+    default=None,
+    optional=True,
+)
+normalize_choices = ClassChoices(
+    "normalize",
+    classes=dict(
+        global_mvn=GlobalMVN,
+        utterance_mvn=UtteranceMVN,
+    ),
+    type_check=AbsNormalize,
+    default="utterance_mvn",
+    optional=True,
+)
+model_choices = ClassChoices(
+    "model",
+    classes=dict(
+        espnet=ESPnetASRModel,
+        maskctc=MaskCTCModel,
+        pit_espnet=PITESPnetModel,
+    ),
+    type_check=AbsESPnetModel,
+    default="espnet",
+)
+preencoder_choices = ClassChoices(
+    name="preencoder",
+    classes=dict(
+        sinc=LightweightSincConvs,
+        linear=LinearProjection,
+    ),
+    type_check=AbsPreEncoder,
+    default=None,
+    optional=True,
+)
+encoder_choices = ClassChoices(
+    "encoder",
+    classes=dict(
+        conformer=ConformerEncoder,
+        transformer=TransformerEncoder,
+        Q_transformer=Q_TransformerEncoder,
+        transformer_multispkr=TransformerEncoderMultiSpkr,
+        contextual_block_transformer=ContextualBlockTransformerEncoder,
+        contextual_block_conformer=ContextualBlockConformerEncoder,
+        vgg_rnn=VGGRNNEncoder,
+        rnn=RNNEncoder,
+        wav2vec2=FairSeqWav2Vec2Encoder,
+        hubert=FairseqHubertEncoder,
+        hubert_pretrain=FairseqHubertPretrainEncoder,
+        torchaudiohubert=TorchAudioHuBERTPretrainEncoder,
+        longformer=LongformerEncoder,
+        branchformer=BranchformerEncoder,
+        whisper=OpenAIWhisperEncoder,
+        e_branchformer=EBranchformerEncoder,
+        avhubert=FairseqAVHubertEncoder,
+    ),
+    type_check=AbsEncoder,
+    default="rnn",
+)
+postencoder_choices = ClassChoices(
+    name="postencoder",
+    classes=dict(
+        hugging_face_transformers=HuggingFaceTransformersPostEncoder,
+        length_adaptor=LengthAdaptorPostEncoder,
+    ),
+    type_check=AbsPostEncoder,
+    default=None,
+    optional=True,
+)
+decoder_choices = ClassChoices(
+    "decoder",
+    classes=dict(
+        transformer=TransformerDecoder,
+        lightweight_conv=LightweightConvolutionTransformerDecoder,
+        lightweight_conv2d=LightweightConvolution2DTransformerDecoder,
+        dynamic_conv=DynamicConvolutionTransformerDecoder,
+        dynamic_conv2d=DynamicConvolution2DTransformerDecoder,
+        rnn=RNNDecoder,
+        transducer=TransducerDecoder,
+        mlm=MLMDecoder,
+        whisper=OpenAIWhisperDecoder,
+        hugging_face_transformers=HuggingFaceTransformersDecoder,
+        s4=S4Decoder,
+    ),
+    type_check=AbsDecoder,
+    default=None,
+    optional=True,
+)
+preprocessor_choices = ClassChoices(
+    "preprocessor",
+    classes=dict(
+        default=CommonPreprocessor,
+        multi=CommonPreprocessor_multi,
+    ),
+    type_check=AbsPreprocessor,
+    default="default",
+)
+
+
+class ASRTask(AbsTask):
+    # If you need more than one optimizers, change this value
+    num_optimizers: int = 1
+
+    # Add variable objects configurations
+    class_choices_list = [
+        # --frontend and --frontend_conf
+        frontend_choices,
+        # --specaug and --specaug_conf
+        specaug_choices,
+        # --normalize and --normalize_conf
+        normalize_choices,
+        # --model and --model_conf
+        model_choices,
+        # --preencoder and --preencoder_conf
+        preencoder_choices,
+        # --encoder and --encoder_conf
+        encoder_choices,
+        # --postencoder and --postencoder_conf
+        postencoder_choices,
+        # --decoder and --decoder_conf
+        decoder_choices,
+        # --preprocessor and --preprocessor_conf
+        preprocessor_choices,
+    ]
+
+    # If you need to modify train() or eval() procedures, change Trainer class here
+    trainer = Trainer
+
+    @classmethod
+    def add_task_arguments(cls, parser: argparse.ArgumentParser):
+        group = parser.add_argument_group(description="Task related")
+
+        # NOTE(kamo): add_arguments(..., required=True) can't be used
+        # to provide --print_config mode. Instead of it, do as
+        required = parser.get_default("required")
+        required += ["token_list"]
+
+        group.add_argument(
+            "--token_list",
+            type=str_or_none,
+            default=None,
+            help="A text mapping int-id to token",
+        )
+        group.add_argument(
+            "--init",
+            type=lambda x: str_or_none(x.lower()),
+            default=None,
+            help="The initialization method",
+            choices=[
+                "chainer",
+                "xavier_uniform",
+                "xavier_normal",
+                "kaiming_uniform",
+                "kaiming_normal",
+                None,
+            ],
+        )
+
+        group.add_argument(
+            "--input_size",
+            type=int_or_none,
+            default=None,
+            help="The number of input dimension of the feature",
+        )
+
+        group.add_argument(
+            "--ctc_conf",
+            action=NestedDictAction,
+            default=get_default_kwargs(CTC),
+            help="The keyword arguments for CTC class.",
+        )
+        group.add_argument(
+            "--joint_net_conf",
+            action=NestedDictAction,
+            default=None,
+            help="The keyword arguments for joint network class.",
+        )
+
+        group = parser.add_argument_group(description="Preprocess related")
+        group.add_argument(
+            "--use_preprocessor",
+            type=str2bool,
+            default=True,
+            help="Apply preprocessing to data or not",
+        )
+        group.add_argument(
+            "--use_lang_prompt",
+            type=str2bool,
+            default=False,
+            help="Use language id as prompt",
+        )
+        group.add_argument(
+            "--use_nlp_prompt",
+            type=str2bool,
+            default=False,
+            help="Use natural language phrases as prompt",
+        )
+        group.add_argument(
+            "--token_type",
+            type=str,
+            default="bpe",
+            choices=[
+                "bpe",
+                "char",
+                "word",
+                "phn",
+                "hugging_face",
+                "whisper_en",
+                "whisper_multilingual",
+            ],
+            help="The text will be tokenized " "in the specified level token",
+        )
+        group.add_argument(
+            "--bpemodel",
+            type=str_or_none,
+            default=None,
+            help="The model file of sentencepiece",
+        )
+        parser.add_argument(
+            "--non_linguistic_symbols",
+            type=str_or_none,
+            help="non_linguistic_symbols file path",
+        )
+        group.add_argument(
+            "--cleaner",
+            type=str_or_none,
+            choices=[
+                None,
+                "tacotron",
+                "jaconv",
+                "vietnamese",
+                "whisper_en",
+                "whisper_basic",
+            ],
+            default=None,
+            help="Apply text cleaning",
+        )
+        group.add_argument(
+            "--g2p",
+            type=str_or_none,
+            choices=g2p_choices,
+            default=None,
+            help="Specify g2p method if --token_type=phn",
+        )
+        group.add_argument(
+            "--speech_volume_normalize",
+            type=float_or_none,
+            default=None,
+            help="Scale the maximum amplitude to the given value.",
+        )
+        group.add_argument(
+            "--rir_scp",
+            type=str_or_none,
+            default=None,
+            help="The file path of rir scp file.",
+        )
+        group.add_argument(
+            "--rir_apply_prob",
+            type=float,
+            default=1.0,
+            help="THe probability for applying RIR convolution.",
+        )
+        group.add_argument(
+            "--noise_scp",
+            type=str_or_none,
+            default=None,
+            help="The file path of noise scp file.",
+        )
+        group.add_argument(
+            "--noise_apply_prob",
+            type=float,
+            default=1.0,
+            help="The probability applying Noise adding.",
+        )
+        group.add_argument(
+            "--noise_db_range",
+            type=str,
+            default="13_15",
+            help="The range of noise decibel level.",
+        )
+        group.add_argument(
+            "--short_noise_thres",
+            type=float,
+            default=0.5,
+            help="If len(noise) / len(speech) is smaller than this threshold during "
+            "dynamic mixing, a warning will be displayed.",
+        )
+        group.add_argument(
+            "--aux_ctc_tasks",
+            type=str,
+            nargs="+",
+            default=[],
+            help="Auxillary tasks to train on using CTC loss. ",
+        )
+
+        for class_choices in cls.class_choices_list:
+            # Append --<name> and --<name>_conf.
+            # e.g. --encoder and --encoder_conf
+            class_choices.add_arguments(group)
+
+    @classmethod
+    @typechecked
+    def build_collate_fn(cls, args: argparse.Namespace, train: bool) -> Callable[
+        [Collection[Tuple[str, Dict[str, np.ndarray]]]],
+        Tuple[List[str], Dict[str, torch.Tensor]],
+    ]:
+        # NOTE(kamo): int value = 0 is reserved by CTC-blank symbol
+        return CommonCollateFn(float_pad_value=0.0, int_pad_value=-1)
+
+    @classmethod
+    @typechecked
+    def build_preprocess_fn(
+        cls, args: argparse.Namespace, train: bool
+    ) -> Optional[Callable[[str, Dict[str, np.array]], Dict[str, np.ndarray]]]:
+        if args.use_preprocessor:
+            try:
+                _ = getattr(args, "preprocessor")
+            except AttributeError:
+                setattr(args, "preprocessor", "default")
+                setattr(args, "preprocessor_conf", dict())
+            except Exception as e:
+                raise e
+
+            preprocessor_class = preprocessor_choices.get_class(args.preprocessor)
+            retval = preprocessor_class(
+                train=train,
+                token_type=args.token_type,
+                token_list=args.token_list,
+                bpemodel=args.bpemodel,
+                non_linguistic_symbols=args.non_linguistic_symbols,
+                text_cleaner=args.cleaner,
+                g2p_type=args.g2p,
+                # NOTE(kamo): Check attribute existence for backward compatibility
+                rir_scp=args.rir_scp if hasattr(args, "rir_scp") else None,
+                rir_apply_prob=(
+                    args.rir_apply_prob if hasattr(args, "rir_apply_prob") else 1.0
+                ),
+                noise_scp=args.noise_scp if hasattr(args, "noise_scp") else None,
+                noise_apply_prob=(
+                    args.noise_apply_prob if hasattr(args, "noise_apply_prob") else 1.0
+                ),
+                noise_db_range=(
+                    args.noise_db_range if hasattr(args, "noise_db_range") else "13_15"
+                ),
+                short_noise_thres=(
+                    args.short_noise_thres
+                    if hasattr(args, "short_noise_thres")
+                    else 0.5
+                ),
+                speech_volume_normalize=(
+                    args.speech_volume_normalize if hasattr(args, "rir_scp") else None
+                ),
+                aux_task_names=(
+                    args.aux_ctc_tasks if hasattr(args, "aux_ctc_tasks") else None
+                ),
+                use_lang_prompt=(
+                    args.use_lang_prompt if hasattr(args, "use_lang_prompt") else None
+                ),
+                **args.preprocessor_conf,
+                use_nlp_prompt=(
+                    args.use_nlp_prompt if hasattr(args, "use_nlp_prompt") else None
+                ),
+            )
+        else:
+            retval = None
+        return retval
+
+    @classmethod
+    def required_data_names(
+        cls, train: bool = True, inference: bool = False
+    ) -> Tuple[str, ...]:
+        if not inference:
+            retval = ("speech", "text")
+        else:
+            # Recognition mode
+            retval = ("speech",)
+        return retval
+
+    @classmethod
+    def optional_data_names(
+        cls, train: bool = True, inference: bool = False
+    ) -> Tuple[str, ...]:
+        MAX_REFERENCE_NUM = 4
+
+        retval = ["text_spk{}".format(n) for n in range(2, MAX_REFERENCE_NUM + 1)]
+        retval = retval + ["prompt"]
+        retval = tuple(retval)
+
+        logging.info(f"Optional Data Names: {retval }")
+        return retval
+
+    @classmethod
+    @typechecked
+    def build_model(cls, args: argparse.Namespace) -> ESPnetASRModel:
+        if isinstance(args.token_list, str):
+            with open(args.token_list, encoding="utf-8") as f:
+                token_list = [line.rstrip() for line in f]
+
+            # Overwriting token_list to keep it as "portable".
+            args.token_list = list(token_list)
+        elif isinstance(args.token_list, (tuple, list)):
+            token_list = list(args.token_list)
+        else:
+            raise RuntimeError("token_list must be str or list")
+
+        # If use multi-blank transducer criterion,
+        # big blank symbols are added just before the standard blank
+        if args.model_conf.get("transducer_multi_blank_durations", None) is not None:
+            sym_blank = args.model_conf.get("sym_blank", "<blank>")
+            blank_idx = token_list.index(sym_blank)
+            for dur in args.model_conf.get("transducer_multi_blank_durations"):
+                if f"<blank{dur}>" not in token_list:  # avoid this during inference
+                    token_list.insert(blank_idx, f"<blank{dur}>")
+            args.token_list = token_list
+
+        vocab_size = len(token_list)
+        logging.info(f"Vocabulary size: {vocab_size }")
+
+        # 1. frontend
+        if args.input_size is None:
+            # Extract features in the model
+            frontend_class = frontend_choices.get_class(args.frontend)
+            frontend = frontend_class(**args.frontend_conf)
+            input_size = frontend.output_size()
+        else:
+            # Give features from data-loader
+            args.frontend = None
+            args.frontend_conf = {}
+            frontend = None
+            input_size = args.input_size
+
+        # 2. Data augmentation for spectrogram
+        if args.specaug is not None:
+            specaug_class = specaug_choices.get_class(args.specaug)
+            specaug = specaug_class(**args.specaug_conf)
+        else:
+            specaug = None
+
+        # 3. Normalization layer
+        if args.normalize is not None:
+            normalize_class = normalize_choices.get_class(args.normalize)
+            normalize = normalize_class(**args.normalize_conf)
+        else:
+            normalize = None
+
+        # 4. Pre-encoder input block
+        # NOTE(kan-bayashi): Use getattr to keep the compatibility
+        if getattr(args, "preencoder", None) is not None:
+            preencoder_class = preencoder_choices.get_class(args.preencoder)
+            preencoder = preencoder_class(**args.preencoder_conf)
+            input_size = preencoder.output_size()
+        else:
+            preencoder = None
+
+        # 4. Encoder
+        encoder_class = encoder_choices.get_class(args.encoder)
+        encoder = encoder_class(input_size=input_size, **args.encoder_conf)
+
+        # 5. Post-encoder block
+        # NOTE(kan-bayashi): Use getattr to keep the compatibility
+        encoder_output_size = encoder.output_size()
+        if getattr(args, "postencoder", None) is not None:
+            postencoder_class = postencoder_choices.get_class(args.postencoder)
+            postencoder = postencoder_class(
+                input_size=encoder_output_size, **args.postencoder_conf
+            )
+            encoder_output_size = postencoder.output_size()
+        else:
+            postencoder = None
+
+        # 5. Decoder
+        if getattr(args, "decoder", None) is not None:
+            decoder_class = decoder_choices.get_class(args.decoder)
+
+            if args.decoder == "transducer":
+                decoder = decoder_class(
+                    vocab_size,
+                    embed_pad=0,
+                    **args.decoder_conf,
+                )
+
+                joint_network = JointNetwork(
+                    vocab_size,
+                    encoder.output_size(),
+                    decoder.dunits,
+                    **args.joint_net_conf,
+                )
+            else:
+                decoder = decoder_class(
+                    vocab_size=vocab_size,
+                    encoder_output_size=encoder_output_size,
+                    **args.decoder_conf,
+                )
+                joint_network = None
+        else:
+            decoder = None
+            joint_network = None
+
+        # 6. CTC
+        ctc = CTC(
+            odim=vocab_size, encoder_output_size=encoder_output_size, **args.ctc_conf
+        )
+
+        # 7. Build model
+        try:
+            model_class = model_choices.get_class(args.model)
+        except AttributeError:
+            model_class = model_choices.get_class("espnet")
+        model = model_class(
+            vocab_size=vocab_size,
+            frontend=frontend,
+            specaug=specaug,
+            normalize=normalize,
+            preencoder=preencoder,
+            encoder=encoder,
+            postencoder=postencoder,
+            decoder=decoder,
+            ctc=ctc,
+            joint_network=joint_network,
+            token_list=token_list,
+            **args.model_conf,
+        )
+
+        # FIXME(kamo): Should be done in model?
+        # 8. Initialize
+        if args.init is not None:
+            initialize(model, args.init)
+
+        return model

src/conf/Librispeech/train_asr_Q_transformer3_HierDecayv2.yaml

@@ -0,0 +1,64 @@
+batch_type: numel
+batch_bins: 45000000
+accum_grad: 2
+max_epoch: 100
+patience: none
+# The initialization method for model parameters
+init: xavier_uniform
+best_model_criterion:
+-   - valid
+    - acc
+    - max
+keep_nbest_models: 10
+
+encoder: Q_transformer
+encoder_conf:
+    output_size: 512
+    attention_heads: 8
+    attention_layer_type: HierDecayv2
+    linear_units: 2048
+    num_blocks: 18
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    attention_dropout_rate: 0.1
+    input_layer: conv2d6
+    normalize_before: true
+
+decoder: transformer
+decoder_conf:
+    attention_heads: 8
+    linear_units: 2048
+    num_blocks: 6
+    dropout_rate: 0.1
+    positional_dropout_rate: 0.1
+    self_attention_dropout_rate: 0.1
+    src_attention_dropout_rate: 0.1
+
+model_conf:
+    ctc_weight: 0.3
+    lsm_weight: 0.1
+    length_normalized_loss: false
+
+use_amp: true
+optim: adam
+optim_conf:
+    lr: 0.002
+scheduler: warmuplr
+scheduler_conf:
+    warmup_steps: 25000
+
+specaug: specaug
+specaug_conf:
+    apply_time_warp: true
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 30
+    num_freq_mask: 2
+    apply_time_mask: true
+    time_mask_width_range:
+    - 0
+    - 40
+    num_time_mask: 2