fix: ensure tokens without whitespace group correctly and update hardcoded html

generate_hardcoded.py

@@ -18,8 +18,13 @@ for text in texts:
         if i == 0 and ("<bos>" in item["token"] or "<bos>" in item["word"]):
             continue
         
-        if item["token"].startswith(" ") and len(current_word) > 0:
-            words.append({"tokens": current_word, "max_score": current_max})
+        token_str = item["token"]
+        is_whitespace = (token_str.strip() == '')
+        prev_is_whitespace = (current_word[-1]["token"].strip() == '') if len(current_word) > 0 else False
+        
+        if token_str.startswith(" ") or (len(current_word) > 0 and is_whitespace != prev_is_whitespace):
+            if len(current_word) > 0:
+                words.append({"tokens": current_word, "max_score": current_max})
             current_word = [item]
             current_max = item["score"]
         else:

main.py

@@ -50,7 +50,7 @@ STUDY_TEXTS = [
         "id": 1,
         "topic": "Science",
         "text": "The human brain is a marvel of biological engineering, containing approximately 86 billion neurons interconnected by trillions of synapses. These neural networks are responsible for everything from basic autonomic functions, like breathing and heart rate, to complex cognitive processes such as memory, emotion, and problem-solving. Neuroplasticity, the brain's ability to reorganize itself by forming new neural connections throughout life, allows humans to learn new skills, recover from injuries, and adapt to changing environments. This extraordinary adaptability is what makes our species so resilient and capable of continuous intellectual growth.",
-        "flowread_html": '<span class="token highlighted">The</span><span class="token highlighted"> human</span><span class="token highlighted"> brain</span><span class="token highlighted"> is</span><span class="token highlighted"> a</span><span class="token highlighted"> marvel</span><span class="token"> of</span><span class="token highlighted"> biological</span><span class="token highlighted"> engineering</span><span class="token highlighted">,</span><span class="token highlighted"> containing</span><span class="token"> approximately</span><span class="token"> </span><span class="token">8</span><span class="token">6</span><span class="token highlighted"> billion</span><span class="token highlighted"> neurons</span><span class="token highlighted"> interconnected</span><span class="token highlighted"> by</span><span class="token"> tri</span><span class="token">lli</span><span class="token">ons</span><span class="token"> of</span><span class="token highlighted"> synapses</span><span class="token highlighted">.</span><span class="token highlighted"> These</span><span class="token highlighted"> neural</span><span class="token highlighted"> networks</span><span class="token"> are</span><span class="token"> responsible</span><span class="token highlighted"> for</span><span class="token"> everything</span><span class="token highlighted"> from</span><span class="token highlighted"> basic</span><span class="token highlighted"> autonomic</span><span class="token highlighted"> functions</span><span class="token highlighted">,</span><span class="token highlighted"> like</span><span class="token"> breathing</span><span class="token"> and</span><span class="token"> heart</span><span class="token"> rate</span><span class="token">,</span><span class="token highlighted"> to</span><span class="token"> complex</span><span class="token highlighted"> cognitive</span><span class="token"> processes</span><span class="token"> such</span><span class="token"> as</span><span class="token highlighted"> memory</span><span class="token highlighted">,</span><span class="token"> emotion</span><span class="token">,</span><span class="token"> and</span><span class="token highlighted"> problem</span><span class="token highlighted">-</span><span class="token highlighted">solving</span><span class="token highlighted">.</span><span class="token highlighted"> Neurop</span><span class="token highlighted">lastic</span><span class="token highlighted">ity</span><span class="token highlighted">,</span><span class="token highlighted"> the</span><span class="token highlighted"> brain</span><span class="token highlighted">\'</span><span class="token">s</span><span class="token"> ability</span><span class="token"> to</span><span class="token highlighted"> reorgan</span><span class="token highlighted">ize</span><span class="token"> itself</span><span class="token"> by</span><span class="token"> forming</span><span class="token"> new</span><span class="token"> neural</span><span class="token"> connections</span><span class="token"> throughout</span><span class="token highlighted"> life</span><span class="token highlighted">,</span><span class="token highlighted"> allows</span><span class="token highlighted"> humans</span><span class="token"> to</span><span class="token highlighted"> learn</span><span class="token"> new</span><span class="token"> skills</span><span class="token">,</span><span class="token"> recover</span><span class="token"> from</span><span class="token"> injuries</span><span class="token">,</span><span class="token"> and</span><span class="token"> adapt</span><span class="token"> to</span><span class="token"> changing</span><span class="token highlighted"> environments</span><span class="token highlighted">.</span><span class="token"> This</span><span class="token"> extraordinary</span><span class="token highlighted"> adaptability</span><span class="token"> is</span><span class="token"> what</span><span class="token"> makes</span><span class="token"> our</span><span class="token"> species</span><span class="token"> so</span><span class="token"> resilient</span><span class="token"> and</span><span class="token"> capable</span><span class="token"> of</span><span class="token"> continuous</span><span class="token"> intellectual</span><span class="token"> growth</span><span class="token">.</span>',
+        "flowread_html": '<span class="token highlighted">The</span><span class="token highlighted"> human</span><span class="token highlighted"> brain</span><span class="token highlighted"> is</span><span class="token highlighted"> a</span><span class="token highlighted"> marvel</span><span class="token"> of</span><span class="token highlighted"> biological</span><span class="token highlighted"> engineering</span><span class="token highlighted">,</span><span class="token highlighted"> containing</span><span class="token"> approximately</span><span class="token"> </span><span class="token">8</span><span class="token">6</span><span class="token highlighted"> billion</span><span class="token highlighted"> neurons</span><span class="token highlighted"> interconnected</span><span class="token highlighted"> by</span><span class="token"> tri</span><span class="token">lli</span><span class="token">ons</span><span class="token"> of</span><span class="token highlighted"> synapses</span><span class="token highlighted">.</span><span class="token highlighted"> These</span><span class="token highlighted"> neural</span><span class="token highlighted"> networks</span><span class="token"> are</span><span class="token"> responsible</span><span class="token highlighted"> for</span><span class="token"> everything</span><span class="token highlighted"> from</span><span class="token highlighted"> basic</span><span class="token highlighted"> autonomic</span><span class="token highlighted"> functions</span><span class="token highlighted">,</span><span class="token highlighted"> like</span><span class="token"> breathing</span><span class="token"> and</span><span class="token"> heart</span><span class="token"> rate</span><span class="token">,</span><span class="token highlighted"> to</span><span class="token"> complex</span><span class="token highlighted"> cognitive</span><span class="token"> processes</span><span class="token"> such</span><span class="token"> as</span><span class="token highlighted"> memory</span><span class="token highlighted">,</span><span class="token"> emotion</span><span class="token">,</span><span class="token"> and</span><span class="token highlighted"> problem</span><span class="token highlighted">-</span><span class="token highlighted">solving</span><span class="token highlighted">.</span><span class="token highlighted"> Neurop</span><span class="token highlighted">lastic</span><span class="token highlighted">ity</span><span class="token highlighted">,</span><span class="token highlighted"> the</span><span class="token highlighted"> brain</span><span class="token highlighted">\'</span><span class="token highlighted">s</span><span class="token"> ability</span><span class="token"> to</span><span class="token highlighted"> reorgan</span><span class="token highlighted">ize</span><span class="token"> itself</span><span class="token"> by</span><span class="token"> forming</span><span class="token"> new</span><span class="token"> neural</span><span class="token"> connections</span><span class="token"> throughout</span><span class="token highlighted"> life</span><span class="token highlighted">,</span><span class="token highlighted"> allows</span><span class="token highlighted"> humans</span><span class="token"> to</span><span class="token highlighted"> learn</span><span class="token"> new</span><span class="token"> skills</span><span class="token">,</span><span class="token"> recover</span><span class="token"> from</span><span class="token"> injuries</span><span class="token">,</span><span class="token"> and</span><span class="token"> adapt</span><span class="token"> to</span><span class="token"> changing</span><span class="token highlighted"> environments</span><span class="token highlighted">.</span><span class="token"> This</span><span class="token"> extraordinary</span><span class="token highlighted"> adaptability</span><span class="token"> is</span><span class="token"> what</span><span class="token"> makes</span><span class="token"> our</span><span class="token"> species</span><span class="token"> so</span><span class="token"> resilient</span><span class="token"> and</span><span class="token"> capable</span><span class="token"> of</span><span class="token"> continuous</span><span class="token"> intellectual</span><span class="token"> growth</span><span class="token">.</span>',
         "questions": [
             {
                 "question": "Approximately how many neurons are in the human brain?",
@@ -68,7 +68,7 @@ STUDY_TEXTS = [
         "id": 2,
         "topic": "History",
         "text": "The Industrial Revolution, which began in Britain in the late 18th century, marked a profound turning point in human history. It initiated the transition from agrarian, handicraft economies to industry and machine manufacturing. The invention of the steam engine, pioneered by figures like James Watt, dramatically increased the efficiency of factories and transportation, revolutionizing the textile industry and leading to the expansion of railways. This era brought about unprecedented economic growth and urbanization, fundamentally altering social structures and paving the way for the modern capitalist system, despite also causing significant social inequalities and poor working conditions initially.",
-        "flowread_html": '<span class="token highlighted">The</span><span class="token highlighted"> Industrial</span><span class="token highlighted"> Revolution</span><span class="token highlighted">,</span><span class="token highlighted"> which</span><span class="token highlighted"> began</span><span class="token"> in</span><span class="token highlighted"> Britain</span><span class="token"> in</span><span class="token"> the</span><span class="token"> late</span><span class="token"> </span><span class="token highlighted">1</span><span class="token">8</span><span class="token">th</span><span class="token highlighted"> century</span><span class="token highlighted">,</span><span class="token highlighted"> marked</span><span class="token"> a</span><span class="token highlighted"> profound</span><span class="token"> turning</span><span class="token highlighted"> point</span><span class="token"> in</span><span class="token highlighted"> human</span><span class="token highlighted"> history</span><span class="token highlighted">.</span><span class="token highlighted"> It</span><span class="token highlighted"> initiated</span><span class="token"> the</span><span class="token"> transition</span><span class="token highlighted"> from</span><span class="token highlighted"> agrarian</span><span class="token">,</span><span class="token"> handic</span><span class="token">raft</span><span class="token highlighted"> economies</span><span class="token highlighted"> to</span><span class="token highlighted"> industry</span><span class="token"> and</span><span class="token highlighted"> machine</span><span class="token highlighted"> manufacturing</span><span class="token highlighted">.</span><span class="token highlighted"> The</span><span class="token highlighted"> invention</span><span class="token highlighted"> of</span><span class="token"> the</span><span class="token highlighted"> steam</span><span class="token highlighted"> engine</span><span class="token highlighted">,</span><span class="token"> pioneered</span><span class="token"> by</span><span class="token"> figures</span><span class="token"> like</span><span class="token"> James</span><span class="token"> Watt</span><span class="token">,</span><span class="token highlighted"> dramatically</span><span class="token highlighted"> increased</span><span class="token"> the</span><span class="token"> efficiency</span><span class="token"> of</span><span class="token highlighted"> factories</span><span class="token"> and</span><span class="token highlighted"> transportation</span><span class="token highlighted">,</span><span class="token"> revolution</span><span class="token highlighted">izing</span><span class="token"> the</span><span class="token"> textile</span><span class="token"> industry</span><span class="token"> and</span><span class="token"> leading</span><span class="token"> to</span><span class="token"> the</span><span class="token"> expansion</span><span class="token"> of</span><span class="token"> railways</span><span class="token highlighted">.</span><span class="token"> This</span><span class="token highlighted"> era</span><span class="token highlighted"> brought</span><span class="token"> about</span><span class="token"> unprecedented</span><span class="token highlighted"> economic</span><span class="token highlighted"> growth</span><span class="token"> and</span><span class="token highlighted"> urbanization</span><span class="token highlighted">,</span><span class="token"> fundamentally</span><span class="token highlighted"> altering</span><span class="token"> social</span><span class="token"> structures</span><span class="token"> and</span><span class="token"> paving</span><span class="token"> the</span><span class="token"> way</span><span class="token"> for</span><span class="token"> the</span><span class="token"> modern</span><span class="token"> capitalist</span><span class="token"> system</span><span class="token">,</span><span class="token"> despite</span><span class="token"> also</span><span class="token"> causing</span><span class="token"> significant</span><span class="token"> social</span><span class="token"> inequalities</span><span class="token"> and</span><span class="token"> poor</span><span class="token"> working</span><span class="token"> conditions</span><span class="token"> initially</span><span class="token">.</span>',
+        "flowread_html": '<span class="token highlighted">The</span><span class="token highlighted"> Industrial</span><span class="token highlighted"> Revolution</span><span class="token highlighted">,</span><span class="token highlighted"> which</span><span class="token highlighted"> began</span><span class="token"> in</span><span class="token highlighted"> Britain</span><span class="token"> in</span><span class="token"> the</span><span class="token"> late</span><span class="token highlighted"> </span><span class="token highlighted">1</span><span class="token highlighted">8</span><span class="token highlighted">th</span><span class="token highlighted"> century</span><span class="token highlighted">,</span><span class="token highlighted"> marked</span><span class="token"> a</span><span class="token highlighted"> profound</span><span class="token"> turning</span><span class="token highlighted"> point</span><span class="token"> in</span><span class="token highlighted"> human</span><span class="token highlighted"> history</span><span class="token highlighted">.</span><span class="token highlighted"> It</span><span class="token highlighted"> initiated</span><span class="token"> the</span><span class="token"> transition</span><span class="token highlighted"> from</span><span class="token highlighted"> agrarian</span><span class="token highlighted">,</span><span class="token"> handic</span><span class="token">raft</span><span class="token highlighted"> economies</span><span class="token highlighted"> to</span><span class="token highlighted"> industry</span><span class="token"> and</span><span class="token highlighted"> machine</span><span class="token highlighted"> manufacturing</span><span class="token highlighted">.</span><span class="token highlighted"> The</span><span class="token highlighted"> invention</span><span class="token highlighted"> of</span><span class="token"> the</span><span class="token highlighted"> steam</span><span class="token highlighted"> engine</span><span class="token highlighted">,</span><span class="token"> pioneered</span><span class="token"> by</span><span class="token"> figures</span><span class="token"> like</span><span class="token"> James</span><span class="token"> Watt</span><span class="token">,</span><span class="token highlighted"> dramatically</span><span class="token highlighted"> increased</span><span class="token"> the</span><span class="token"> efficiency</span><span class="token"> of</span><span class="token highlighted"> factories</span><span class="token"> and</span><span class="token highlighted"> transportation</span><span class="token highlighted">,</span><span class="token highlighted"> revolution</span><span class="token highlighted">izing</span><span class="token"> the</span><span class="token"> textile</span><span class="token"> industry</span><span class="token"> and</span><span class="token"> leading</span><span class="token"> to</span><span class="token"> the</span><span class="token"> expansion</span><span class="token"> of</span><span class="token highlighted"> railways</span><span class="token highlighted">.</span><span class="token"> This</span><span class="token highlighted"> era</span><span class="token highlighted"> brought</span><span class="token"> about</span><span class="token"> unprecedented</span><span class="token highlighted"> economic</span><span class="token highlighted"> growth</span><span class="token"> and</span><span class="token highlighted"> urbanization</span><span class="token highlighted">,</span><span class="token"> fundamentally</span><span class="token highlighted"> altering</span><span class="token"> social</span><span class="token"> structures</span><span class="token"> and</span><span class="token"> paving</span><span class="token"> the</span><span class="token"> way</span><span class="token"> for</span><span class="token"> the</span><span class="token"> modern</span><span class="token"> capitalist</span><span class="token"> system</span><span class="token">,</span><span class="token"> despite</span><span class="token"> also</span><span class="token"> causing</span><span class="token"> significant</span><span class="token"> social</span><span class="token"> inequalities</span><span class="token"> and</span><span class="token"> poor</span><span class="token"> working</span><span class="token"> conditions</span><span class="token"> initially</span><span class="token">.</span>',
         "questions": [
             {
                 "question": "Where did the Industrial Revolution begin?",

static/index.html

@@ -471,9 +471,13 @@
             currentTokens.forEach((item, index) => {
                 if (index === 0 && (item.token.includes('<bos>') || item.word.includes('<bos>'))) return;
 
-                // A space at the start of a token usually indicates a new word in Gemma
-                if (item.token.startsWith(' ') && currentWordTokens.length > 0) {
-                    words.push({ tokens: currentWordTokens, maxScore: currentWordMaxScore });
+                const isWhitespace = item.token.trim() === '';
+                const prevIsWhitespace = currentWordTokens.length > 0 && currentWordTokens[currentWordTokens.length - 1].token.trim() === '';
+
+                if (item.token.startsWith(' ') || (currentWordTokens.length > 0 && isWhitespace !== prevIsWhitespace)) {
+                    if (currentWordTokens.length > 0) {
+                        words.push({ tokens: currentWordTokens, maxScore: currentWordMaxScore });
+                    }
                     currentWordTokens = [item];
                     currentWordMaxScore = item.score;
                 } else {