using full script for duplicate

data/executed_lines.txt

@@ -0,0 +1,84 @@
+he felt tightness rising along his neck and shoulders after a long commute. It tightened with movement. He used this simple neck massager during a quiet break, relieving muscle tension, inviting relaxation, and supporting health and wellness. Soon enough
+he felt tightness rising along his neck and shoulders after a long commute. It tightened with movement. So he tried this calming neck massager, easing neck strain, softening the shoulders, and improving circulation. In a moment
+he felt tightness rising along his neck and shoulders after a long commute. It tightened with movement. So he used this everyday neck massager to release the strain, melting tension, calming the mind, and supporting wellness. Soon after
+he felt tightness rising along his neck and shoulders after a long commute. It tightened with movement. That’s when he reached for this soothing neck massager, easing neck strain, softening the shoulders, and improving circulation. Right away
+he felt tightness rising along his neck and shoulders after a long commute. It tightened with movement. Then he chose a relaxing neck and shoulder massager, relieving muscle tension, inviting relaxation, and supporting health and wellness. Almost immediately
+he felt tightness rising along his neck and shoulders after a long commute. It tightened with movement. Then he picked up this home neck massager and reset, bringing relaxation, muscle tension relief, and better circulation. Right away
+he felt tightness rising along his neck and shoulders after a long commute. It tightened with movement. Then he picked up this home neck massager and reset, reducing stress, relaxing tight muscles, and boosting circulation. Soon enough
+he found his neck limited on every turn after carrying boxes upstairs. It tightened with movement. He decided to give this gentle neck massager a chance, relieving muscle tension, inviting relaxation, and supporting health and wellness. In an instant
+he found his neck limited on every turn after carrying boxes upstairs. It tightened with movement. He used this simple neck massager during a quiet break, easing neck strain, softening the shoulders, and improving circulation. Right away
+he found his neck limited on every turn after carrying boxes upstairs. It tightened with movement. He used this simple neck massager during a quiet break, reducing stress, relaxing tight muscles, and boosting circulation. Soon enough
+he found his neck limited on every turn after carrying boxes upstairs. It tightened with movement. He used this simple neck massager during a quiet break, relieving muscle tension, inviting relaxation, and supporting health and wellness. In that moment
+he found his neck limited on every turn after carrying boxes upstairs. It tightened with movement. So he tried this calming neck massager, easing neck strain, softening the shoulders, and improving circulation., with calm relief Right away
+he found his neck limited on every turn after carrying boxes upstairs. It tightened with movement. So he turned to a comforting neck massager for support, relieving muscle tension, inviting relaxation, and supporting health and wellness. Before long
+he found his neck limited on every turn after carrying boxes upstairs. It tightened with movement. So he used this everyday neck massager to release the strain, relieving muscle tension, inviting relaxation, and supporting health and wellness. Right away
+he found his neck limited on every turn after carrying boxes upstairs. It tightened with movement. Then he chose a relaxing neck and shoulder massager, relieving muscle tension, inviting relaxation, and supporting health and wellness. Moments later
+he found his neck limited on every turn after carrying boxes upstairs. It tightened with movement. Then he picked up this home neck massager and reset, easing neck strain, softening the shoulders, and improving circulation. Right away
+he noticed his shoulders lifting with stress while working against a deadline. It tightened with movement. He decided to give this gentle neck massager a chance, easing neck strain, softening the shoulders, and improving circulation. Right away
+he noticed his shoulders lifting with stress while working against a deadline. It tightened with movement. He finally tried this supportive neck massager, relieving muscle tension, inviting relaxation, and supporting health and wellness. In that moment
+he noticed his shoulders lifting with stress while working against a deadline. It tightened with movement. He used this simple neck massager during a quiet break, melting tension, calming the mind, and supporting wellness. Soon enough
+he noticed his shoulders lifting with stress while working against a deadline. It tightened with movement. So he turned to a comforting neck massager for support, bringing relaxation, muscle tension relief, and better circulation. In a moment
+he noticed his shoulders lifting with stress while working against a deadline. It tightened with movement. So he used this everyday neck massager to release the strain, bringing relaxation, muscle tension relief, and better circulation. Right away
+he noticed his shoulders lifting with stress while working against a deadline. It tightened with movement. So he used this everyday neck massager to release the strain, melting tension, calming the mind, and supporting wellness. In that moment
+he noticed his shoulders lifting with stress while working against a deadline. It tightened with movement. That’s when he reached for this soothing neck massager, reducing stress, relaxing tight muscles, and boosting circulation. Almost immediately
+he noticed his shoulders lifting with stress while working against a deadline. It tightened with movement. Then he chose a relaxing neck and shoulder massager, melting tension, calming the mind, and supporting wellness. Before long
+he noticed his shoulders lifting with stress while working against a deadline. It tightened with movement. Then he chose a relaxing neck and shoulder massager, reducing stress, relaxing tight muscles, and boosting circulation. Soon after
+he noticed his shoulders lifting with stress while working against a deadline. It tightened with movement. Then he chose a relaxing neck and shoulder massager, relieving muscle tension, inviting relaxation, and supporting health and wellness. In an instant
+he noticed his shoulders lifting with stress while working against a deadline. It tightened with movement. Then he picked up this home neck massager and reset, relieving muscle tension, inviting relaxation, and supporting health and wellness. Almost immediately
+he rubbed the tense spot where his neck meets his shoulders after yard work. It tightened with movement. He decided to give this gentle neck massager a chance, relieving muscle tension, inviting relaxation, and supporting health and wellness. Right away
+he rubbed the tense spot where his neck meets his shoulders after yard work. It tightened with movement. He finally tried this supportive neck massager, easing neck strain, softening the shoulders, and improving circulation. Before you know it
+he rubbed the tense spot where his neck meets his shoulders after yard work. It tightened with movement. So he relied on this easy neck massager to unwind, easing neck strain, softening the shoulders, and improving circulation. In an instant
+he rubbed the tense spot where his neck meets his shoulders after yard work. It tightened with movement. So he relied on this easy neck massager to unwind, easing neck strain, softening the shoulders, and improving circulation. Right away
+he rubbed the tense spot where his neck meets his shoulders after yard work. It tightened with movement. So he relied on this easy neck massager to unwind, relieving muscle tension, inviting relaxation, and supporting health and wellness. Before you know it
+he rubbed the tense spot where his neck meets his shoulders after yard work. It tightened with movement. So he tried this calming neck massager, bringing relaxation, muscle tension relief, and better circulation. Moments later
+he rubbed the tense spot where his neck meets his shoulders after yard work. It tightened with movement. So he turned to a comforting neck massager for support, reducing stress, relaxing tight muscles, and boosting circulation. In an instant
+he rubbed the tense spot where his neck meets his shoulders after yard work. It tightened with movement. So he turned to a comforting neck massager for support, relieving muscle tension, inviting relaxation, and supporting health and wellness. Soon enough
+he rubbed the tense spot where his neck meets his shoulders after yard work. It tightened with movement. That’s when he reached for this soothing neck massager, easing neck strain, softening the shoulders, and improving circulation. Before long
+he rubbed the tense spot where his neck meets his shoulders after yard work. It tightened with movement. That’s when he reached for this soothing neck massager, reducing stress, relaxing tight muscles, and boosting circulation. Almost immediately
+he rubbed the tense spot where his neck meets his shoulders after yard work. It tightened with movement. That’s when he reached for this soothing neck massager, reducing stress, relaxing tight muscles, and boosting circulation. Moments later
+he rubbed the tense spot where his neck meets his shoulders after yard work. It tightened with movement. Then he chose a relaxing neck and shoulder massager, easing neck strain, softening the shoulders, and improving circulation. Soon after
+he rubbed the tense spot where his neck meets his shoulders after yard work. It tightened with movement. Then he picked up this home neck massager and reset, bringing relaxation, muscle tension relief, and better circulation. In an instant
+he rubbed the tense spot where his neck meets his shoulders after yard work. It tightened with movement. Then he picked up this home neck massager and reset, relieving muscle tension, inviting relaxation, and supporting health and wellness. In that moment
+he winced as shoulder tension crept in while scrolling on the couch. It tightened with movement. He decided to give this gentle neck massager a chance, reducing stress, relaxing tight muscles, and boosting circulation. Soon enough
+he winced as shoulder tension crept in while scrolling on the couch. It tightened with movement. He finally tried this supportive neck massager, bringing relaxation, muscle tension relief, and better circulation. Before you know it
+he winced as shoulder tension crept in while scrolling on the couch. It tightened with movement. He finally tried this supportive neck massager, reducing stress, relaxing tight muscles, and boosting circulation., with calm relief Before long
+he winced as shoulder tension crept in while scrolling on the couch. It tightened with movement. He finally tried this supportive neck massager, relieving muscle tension, inviting relaxation, and supporting health and wellness. Right away
+he winced as shoulder tension crept in while scrolling on the couch. It tightened with movement. So he relied on this easy neck massager to unwind, easing neck strain, softening the shoulders, and improving circulation. Right away
+he winced as shoulder tension crept in while scrolling on the couch. It tightened with movement. So he tried this calming neck massager, reducing stress, relaxing tight muscles, and boosting circulation., with calm relief In an instant
+he winced as shoulder tension crept in while scrolling on the couch. It tightened with movement. So he turned to a comforting neck massager for support, easing neck strain, softening the shoulders, and improving circulation. Before long
+he winced as shoulder tension crept in while scrolling on the couch. It tightened with movement. That’s when he reached for this soothing neck massager, relieving muscle tension, inviting relaxation, and supporting health and wellness. Almost immediately
+she felt a sharp pull beneath her ear after a day of driving. It tightened with movement. She finally tried this supportive neck massager, bringing relaxation, muscle tension relief, and better circulation., with calm relief Right away
+she felt a sharp pull beneath her ear after a day of driving. It tightened with movement. She used this simple neck massager during a quiet break, reducing stress, relaxing tight muscles, and boosting circulation. In a moment
+she felt a sharp pull beneath her ear after a day of driving. It tightened with movement. She used this simple neck massager during a quiet break, reducing stress, relaxing tight muscles, and boosting circulation. Soon after
+she felt a sharp pull beneath her ear after a day of driving. It tightened with movement. So she relied on this easy neck massager to unwind, easing neck strain, softening the shoulders, and improving circulation. Before long
+she felt a sharp pull beneath her ear after a day of driving. It tightened with movement. So she relied on this easy neck massager to unwind, easing neck strain, softening the shoulders, and improving circulation. In that moment
+she felt a sharp pull beneath her ear after a day of driving. It tightened with movement. So she turned to a comforting neck massager for support, bringing relaxation, muscle tension relief, and better circulation. In a moment
+she felt a sharp pull beneath her ear after a day of driving. It tightened with movement. So she turned to a comforting neck massager for support, easing neck strain, softening the shoulders, and improving circulation. In a moment
+she felt a sharp pull beneath her ear after a day of driving. It tightened with movement. So she turned to a comforting neck massager for support, melting tension, calming the mind, and supporting wellness. Right away
+she felt a sharp pull beneath her ear after a day of driving. It tightened with movement. So she turned to a comforting neck massager for support, reducing stress, relaxing tight muscles, and boosting circulation. Soon after
+she felt a sharp pull beneath her ear after a day of driving. It tightened with movement. So she used this everyday neck massager to release the strain, relieving muscle tension, inviting relaxation, and supporting health and wellness. Before long
+she massaged a sore knot along her trapezius after cleaning the house. It tightened with movement. So she tried this calming neck massager, melting tension, calming the mind, and supporting wellness., with calm relief In an instant
+she massaged a sore knot along her trapezius after cleaning the house. It tightened with movement. So she tried this calming neck massager, relieving muscle tension, inviting relaxation, and supporting health and wellness. In an instant
+she massaged a sore knot along her trapezius after cleaning the house. It tightened with movement. So she turned to a comforting neck massager for support, reducing stress, relaxing tight muscles, and boosting circulation. Moments later
+she massaged a sore knot along her trapezius after cleaning the house. It tightened with movement. That’s when she reached for this soothing neck massager, easing neck strain, softening the shoulders, and improving circulation. Almost immediately
+she massaged a sore knot along her trapezius after cleaning the house. It tightened with movement. Then she chose a relaxing neck and shoulder massager, reducing stress, relaxing tight muscles, and boosting circulation. Soon after
+she noticed a dull ache building at the base of her neck during meetings. It tightened with movement. She finally tried this supportive neck massager, bringing relaxation, muscle tension relief, and better circulation. Almost immediately
+she noticed a dull ache building at the base of her neck during meetings. It tightened with movement. So she relied on this easy neck massager to unwind, bringing relaxation, muscle tension relief, and better circulation. Before long
+she noticed a dull ache building at the base of her neck during meetings. It tightened with movement. So she relied on this easy neck massager to unwind, melting tension, calming the mind, and supporting wellness. Almost immediately
+she noticed a dull ache building at the base of her neck during meetings. It tightened with movement. So she turned to a comforting neck massager for support, easing neck strain, softening the shoulders, and improving circulation. Soon enough
+she noticed a dull ache building at the base of her neck during meetings. It tightened with movement. So she used this everyday neck massager to release the strain, reducing stress, relaxing tight muscles, and boosting circulation. Before you know it
+she noticed a dull ache building at the base of her neck during meetings. It tightened with movement. That’s when she reached for this soothing neck massager, easing neck strain, softening the shoulders, and improving circulation. Before long
+she noticed a dull ache building at the base of her neck during meetings. It tightened with movement. That’s when she reached for this soothing neck massager, relieving muscle tension, inviting relaxation, and supporting health and wellness. Before long
+she noticed a dull ache building at the base of her neck during meetings. It tightened with movement. Then she chose a relaxing neck and shoulder massager, melting tension, calming the mind, and supporting wellness. Before long
+she noticed a dull ache building at the base of her neck during meetings. It tightened with movement. Then she picked up this home neck massager and reset, easing neck strain, softening the shoulders, and improving circulation. Right away
+she realized her posture had tightened through her shoulders during study. It tightened with movement. She decided to give this gentle neck massager a chance, melting tension, calming the mind, and supporting wellness. In an instant
+she realized her posture had tightened through her shoulders during study. It tightened with movement. She finally tried this supportive neck massager, bringing relaxation, muscle tension relief, and better circulation., with calm relief In that moment
+she realized her posture had tightened through her shoulders during study. It tightened with movement. She used this simple neck massager during a quiet break, bringing relaxation, muscle tension relief, and better circulation. Soon enough
+she realized her posture had tightened through her shoulders during study. It tightened with movement. She used this simple neck massager during a quiet break, melting tension, calming the mind, and supporting wellness. Moments later
+she realized her posture had tightened through her shoulders during study. It tightened with movement. So she relied on this easy neck massager to unwind, relieving muscle tension, inviting relaxation, and supporting health and wellness. Right away
+she realized her posture had tightened through her shoulders during study. It tightened with movement. So she turned to a comforting neck massager for support, relieving muscle tension, inviting relaxation, and supporting health and wellness. In a moment
+she realized her posture had tightened through her shoulders during study. It tightened with movement. So she turned to a comforting neck massager for support, relieving muscle tension, inviting relaxation, and supporting health and wellness. Moments later
+she realized her posture had tightened through her shoulders during study. It tightened with movement. So she used this everyday neck massager to release the strain, reducing stress, relaxing tight muscles, and boosting circulation. Soon after
+she realized her posture had tightened through her shoulders during study. It tightened with movement. That’s when she reached for this soothing neck massager, bringing relaxation, muscle tension relief, and better circulation. In a moment
+she realized her posture had tightened through her shoulders during study. It tightened with movement. Then she chose a relaxing neck and shoulder massager, bringing relaxation, muscle tension relief, and better circulation. Before you know it
+she realized her posture had tightened through her shoulders during study. It tightened with movement. Then she picked up this home neck massager and reset, reducing stress, relaxing tight muscles, and boosting circulation. In an instant

data/executed_lines_job0.txt

@@ -144,26 +144,3 @@ darkeningpainfromstressYoufeellikeitdims
 shefeltstiffnessclimbingupherneckafterhour
 hefeltpaininhisshouldersfrompushingstrolle
 mountingstraininshoulderaftertasksYoufeel
-herubbedthetensespotwherehisneckmeetshis
-sherealizedherposturehadtightenedthroughher
-sherealizedherposturehadtightenedthroughher
-hewincedasshouldertensioncreptinwhilescrol
-hewincedasshouldertensioncreptinwhilescrol
-hefoundhisnecklimitedoneveryturnaftercarr
-shefeltasharppullbeneathherearafteraday
-shemassagedasoreknotalonghertrapeziusafter
-sherealizedherposturehadtightenedthroughher
-herubbedthetensespotwherehisneckmeetshis
-shefeltasharppullbeneathherearafteraday
-hefelttightnessrisingalonghisneckandshould
-hefelttightnessrisingalonghisneckandshould
-henoticedhisshouldersliftingwithstresswhile
-shemassagedasoreknotalonghertrapeziusafter
-hewincedasshouldertensioncreptinwhilescrol
-hefelttightnessrisingalonghisneckandshould
-sherealizedherposturehadtightenedthroughher
-henoticedhisshouldersliftingwithstresswhile
-hewincedasshouldertensioncreptinwhilescrol
-shenoticedadullachebuildingatthebaseofhe
-hefoundhisnecklimitedoneveryturnaftercarr
-hefoundhisnecklimitedoneveryturnaftercarr

data/executed_lines_job1.txt

@@ -144,34 +144,4 @@ tirednessinshouldersposthoursYoufeellikei
 anchoringrigidityinupperbackYoufeellikeit
 ongoingdiscomfortinneckloadYoufeellikeit
 creepingtensioninnecksilentlyYoufeellikei
-sherealizedherposturehadtightenedthroughher
-shefeltasharppullbeneathherearafteraday
-herubbedthetensespotwherehisneckmeetshis
-henoticedhisshouldersliftingwithstresswhile
-shefeltasharppullbeneathherearafteraday
-hefoundhisnecklimitedoneveryturnaftercarr
-shefeltasharppullbeneathherearafteraday
-shemassagedasoreknotalonghertrapeziusafter
-shemassagedasoreknotalonghertrapeziusafter
-hefoundhisnecklimitedoneveryturnaftercarr
-herubbedthetensespotwherehisneckmeetshis
-shenoticedadullachebuildingatthebaseofhe
-sherealizedherposturehadtightenedthroughher
-henoticedhisshouldersliftingwithstresswhile
-hefelttightnessrisingalonghisneckandshould
-shenoticedadullachebuildingatthebaseofhe
-henoticedhisshouldersliftingwithstresswhile
-herubbedthetensespotwherehisneckmeetshis
-henoticedhisshouldersliftingwithstresswhile
-shenoticedadullachebuildingatthebaseofhe
-sherealizedherposturehadtightenedthroughher
-hefoundhisnecklimitedoneveryturnaftercarr
-hefelttightnessrisingalonghisneckandshould
-shenoticedadullachebuildingatthebaseofhe
-sherealizedherposturehadtightenedthroughher
-herubbedthetensespotwherehisneckmeetshis
-hefelttightnessrisingalonghisneckandshould
-henoticedhisshouldersliftingwithstresswhile
-herubbedthetensespotwherehisneckmeetshis
-hefelttightnessrisingalonghisneckandshould
-hewincedasshouldertensioncreptinwhilescrol
+

data/executed_lines_job2.txt

@@ -1,33 +1,2 @@
 escalatingstrainincervicalareaYoufeellike
-weightytensioninshoulderareaYoufeellikeit
-hewincedasshouldertensioncreptinwhilescrol
-hefoundhisnecklimitedoneveryturnaftercarr
-shemassagedasoreknotalonghertrapeziusafter
-sherealizedherposturehadtightenedthroughher
-herubbedthetensespotwherehisneckmeetshis
-shefeltasharppullbeneathherearafteraday
-hefoundhisnecklimitedoneveryturnaftercarr
-henoticedhisshouldersliftingwithstresswhile
-shenoticedadullachebuildingatthebaseofhe
-henoticedhisshouldersliftingwithstresswhile
-shenoticedadullachebuildingatthebaseofhe
-shenoticedadullachebuildingatthebaseofhe
-henoticedhisshouldersliftingwithstresswhile
-hewincedasshouldertensioncreptinwhilescrol
-sherealizedherposturehadtightenedthroughher
-shefeltasharppullbeneathherearafteraday
-hewincedasshouldertensioncreptinwhilescrol
-shefeltasharppullbeneathherearafteraday
-herubbedthetensespotwherehisneckmeetshis
-shefeltasharppullbeneathherearafteraday
-hefoundhisnecklimitedoneveryturnaftercarr
-henoticedhisshouldersliftingwithstresswhile
-henoticedhisshouldersliftingwithstresswhile
-hefelttightnessrisingalonghisneckandshould
-henoticedhisshouldersliftingwithstresswhile
-sherealizedherposturehadtightenedthroughher
-herubbedthetensespotwherehisneckmeetshis
-sherealizedherposturehadtightenedthroughher
-shenoticedadullachebuildingatthebaseofhe
-herubbedthetensespotwherehisneckmeetshis
-herubbedthetensespotwherehisneckmeetshis
+weightytensioninshoulderareaYoufeellikeit

src/process_csv.py

@@ -18,7 +18,6 @@ import random
 DATA_DIR = Path("data")
 ALL_VIDEO_FILE_INFO = None
 
-
 def get_progress_file(job_index=None):
     """Get the appropriate progress file for this job."""
     if job_index is not None:
@@ -217,13 +216,12 @@ async def process_all_csvs(config, commit=False, job_index=None, total_jobs=None
 
     for global_row_index, csv_name, row in rows_to_process:
         tts_script = row.get("TTS Script (AI Avatar)", "")
-        safe_name = "".join(c for c in tts_script[:50] if c.isalnum()) or "unknown"
 
         # if "her shoulders got tight from typing emails all day" not in tts_script and "my evening bloomed into a cozy gathering with friends" not in tts_script:
         #     continue
 
-        if safe_name in executed:
-            logger.info(f"⏭️  Skipping {safe_name} (already executed)")
+        if tts_script in executed:
+            logger.info(f"⏭️  Skipping {tts_script} (already executed)")
             continue
 
         try:
@@ -236,20 +234,20 @@ async def process_all_csvs(config, commit=False, job_index=None, total_jobs=None
                 success_count += 1
                 # Write progress to THIS JOB's file only
                 with progress_file.open("a") as pf:
-                    pf.write(f"{safe_name}\n")
+                    pf.write(f"{tts_script}\n")
                 
                 # Commit every 5 successful completions to reduce git overhead
                 if success_count % 2 == 0:
                     git_commit_progress(job_index, commit)
                 
-                logger.info(f"✅ {safe_name} completed successfully ({success_count}/{processed_count})")
+                logger.info(f"✅ {tts_script} completed successfully ({success_count}/{processed_count})")
             else:
-                logger.warning(f"⚠️ {safe_name} pipeline failed, NOT marking as complete")
+                logger.warning(f"⚠️ {tts_script} pipeline failed, NOT marking as complete")
                 if success_count % 2 == 1:
                     git_commit_progress(job_index, commit)
 
         except Exception as e:
-            logger.error(f"❌ Error processing {safe_name}: {e}", exc_info=True)
+            logger.error(f"❌ Error processing {tts_script}: {e}", exc_info=True)
             # Continue processing other rows instead of breaking
             continue