Refactor load_song_database to accept config and update data loading logic; add lyric_word_length calculation in create_features

data/song2word_lengths.json

@@ -0,0 +1,871 @@
+{
+    "kising_421": [
+        8,
+        8,
+        4,
+        5,
+        6,
+        2,
+        10,
+        10,
+        4,
+        6,
+        6,
+        8,
+        3,
+        2,
+        4,
+        4,
+        2,
+        3,
+        4,
+        6,
+        14,
+        7,
+        7,
+        8,
+        9,
+        10,
+        4,
+        6,
+        6,
+        8,
+        7,
+        7,
+        7,
+        7,
+        7,
+        8,
+        7,
+        8,
+        7,
+        6,
+        7,
+        6,
+        7,
+        6,
+        12,
+        13,
+        5,
+        12,
+        12
+    ],
+    "kising_422": [
+        10,
+        10,
+        10,
+        10,
+        10,
+        10,
+        10,
+        10,
+        9,
+        9,
+        9,
+        7,
+        9,
+        9,
+        10,
+        6,
+        6,
+        11,
+        10,
+        10,
+        10,
+        9,
+        9,
+        9,
+        7,
+        9,
+        9,
+        9,
+        6,
+        6,
+        9,
+        9,
+        9,
+        7,
+        9,
+        9,
+        9,
+        6,
+        6,
+        6,
+        3
+    ],
+    "kising_423": [
+        1,
+        3,
+        4,
+        2,
+        1,
+        3,
+        3,
+        4,
+        1,
+        3,
+        3,
+        3,
+        4,
+        3,
+        4,
+        17,
+        16,
+        17,
+        13,
+        2,
+        3,
+        6,
+        11,
+        2,
+        1,
+        3,
+        3,
+        2,
+        1,
+        3,
+        3,
+        2,
+        3,
+        3,
+        3,
+        1,
+        3,
+        3,
+        3,
+        17,
+        16,
+        17,
+        13,
+        2,
+        2,
+        6,
+        10,
+        2,
+        5,
+        13,
+        2,
+        4,
+        3,
+        2
+    ],
+    "kising_424": [
+        3,
+        5,
+        3,
+        5,
+        3,
+        5,
+        5,
+        3,
+        5,
+        3,
+        5,
+        3,
+        5,
+        4,
+        5,
+        3,
+        2,
+        3,
+        2,
+        5,
+        5,
+        5,
+        5,
+        5,
+        6,
+        3,
+        5,
+        3,
+        5,
+        3,
+        5,
+        4,
+        5,
+        3,
+        2,
+        5,
+        5,
+        5,
+        8,
+        5,
+        5,
+        6,
+        3,
+        2,
+        3,
+        3,
+        5,
+        5,
+        5,
+        5,
+        5,
+        7,
+        5,
+        5,
+        5,
+        6,
+        3,
+        5
+    ],
+    "kising_425": [
+        3,
+        4,
+        3,
+        3,
+        6,
+        2,
+        3,
+        8,
+        2,
+        2,
+        4,
+        4,
+        2,
+        4,
+        5,
+        5,
+        7,
+        2,
+        5,
+        5,
+        8,
+        5,
+        6,
+        1,
+        3,
+        4,
+        1,
+        4,
+        3,
+        4,
+        1,
+        3,
+        8,
+        2,
+        2,
+        4,
+        4,
+        3,
+        4,
+        5,
+        5,
+        7,
+        2,
+        5,
+        6,
+        8,
+        5,
+        5,
+        9,
+        5,
+        5,
+        8,
+        5,
+        5,
+        7,
+        2,
+        5,
+        5,
+        8,
+        5,
+        5,
+        5,
+        2,
+        1
+    ],
+    "kising_426": [
+        3,
+        8,
+        8,
+        12,
+        3,
+        8,
+        8,
+        12,
+        29,
+        6,
+        24,
+        12,
+        10,
+        9,
+        6,
+        3,
+        5,
+        3,
+        16,
+        3,
+        8,
+        8,
+        12,
+        3,
+        8,
+        8,
+        12,
+        29,
+        6,
+        24,
+        10,
+        8,
+        9,
+        6,
+        3,
+        4
+    ],
+    "kising_427": [
+        7,
+        7,
+        9,
+        6,
+        7,
+        7,
+        8,
+        7,
+        4,
+        4,
+        11,
+        15,
+        15,
+        15,
+        10,
+        6,
+        4,
+        6,
+        10,
+        2,
+        2,
+        7,
+        7,
+        15,
+        4,
+        4,
+        10,
+        15,
+        15,
+        15,
+        10,
+        5,
+        4,
+        6,
+        10,
+        2,
+        17,
+        15,
+        14,
+        10,
+        5,
+        4,
+        16,
+        1,
+        2,
+        15,
+        15,
+        8,
+        7,
+        10,
+        5,
+        4,
+        16,
+        2,
+        2,
+        10,
+        2,
+        2,
+        10,
+        7,
+        2
+    ],
+    "kising_428": [
+        5,
+        1,
+        10,
+        8,
+        6,
+        5,
+        10,
+        9,
+        6,
+        5,
+        13,
+        2,
+        5,
+        2,
+        13,
+        3,
+        7,
+        8,
+        7,
+        12,
+        7,
+        8,
+        7,
+        6,
+        2,
+        1,
+        10,
+        9,
+        11,
+        13,
+        2,
+        7,
+        1,
+        12,
+        3,
+        7,
+        8,
+        7,
+        6,
+        5,
+        7,
+        8,
+        7,
+        5,
+        6,
+        9,
+        7,
+        7,
+        6,
+        5,
+        7,
+        8,
+        7,
+        5,
+        5,
+        4,
+        2
+    ],
+    "kising_429": [
+        4,
+        3,
+        4,
+        3,
+        4,
+        3,
+        4,
+        3,
+        6,
+        5,
+        3,
+        1,
+        5,
+        4,
+        2,
+        1,
+        7,
+        8,
+        19,
+        8,
+        8,
+        19,
+        5,
+        1,
+        4,
+        4,
+        1,
+        4,
+        3,
+        4,
+        3,
+        4,
+        3,
+        4,
+        3,
+        2,
+        5,
+        4,
+        3,
+        7,
+        6,
+        19,
+        7,
+        7,
+        19,
+        7,
+        7,
+        27,
+        7,
+        19,
+        3,
+        3,
+        8,
+        5,
+        7,
+        7,
+        7,
+        19,
+        8,
+        6,
+        19,
+        5,
+        5,
+        5,
+        5,
+        5,
+        5,
+        8,
+        1,
+        1
+    ],
+    "kising_430": [
+        5,
+        7,
+        4,
+        7,
+        15,
+        3,
+        5,
+        5,
+        7,
+        4,
+        7,
+        15,
+        3,
+        4,
+        2,
+        6,
+        6,
+        8,
+        3,
+        6,
+        7,
+        21,
+        6,
+        13,
+        6,
+        17,
+        12,
+        2,
+        6,
+        6,
+        6,
+        12,
+        6,
+        7,
+        19,
+        7,
+        12,
+        28,
+        4,
+        1,
+        1,
+        6,
+        7,
+        12,
+        7,
+        17,
+        3,
+        5,
+        5,
+        2,
+        2
+    ],
+    "kising_431": [
+        7,
+        5,
+        7,
+        5,
+        9,
+        5,
+        8,
+        5,
+        4,
+        4,
+        5,
+        3,
+        3,
+        2,
+        2,
+        4,
+        9,
+        6,
+        3,
+        2,
+        7,
+        5,
+        7,
+        5,
+        9,
+        5,
+        8,
+        5,
+        8,
+        5,
+        8,
+        2,
+        4,
+        9,
+        3,
+        3,
+        3,
+        2,
+        6,
+        1,
+        2,
+        2
+    ],
+    "kising_432": [
+        7,
+        5,
+        7,
+        5,
+        9,
+        5,
+        9,
+        5,
+        4,
+        4,
+        5,
+        3,
+        3,
+        2,
+        2,
+        4,
+        4,
+        5,
+        3,
+        3,
+        3,
+        2,
+        7,
+        5,
+        7,
+        5,
+        9,
+        6,
+        8,
+        5,
+        4,
+        4,
+        5,
+        3,
+        3,
+        2,
+        2,
+        4,
+        4,
+        5,
+        3,
+        3,
+        3,
+        3,
+        3,
+        4,
+        2,
+        2
+    ],
+    "kising_433": [
+        11,
+        12,
+        12,
+        24,
+        11,
+        7,
+        6,
+        8,
+        7,
+        6,
+        9,
+        7,
+        6,
+        9,
+        13,
+        8,
+        8,
+        7,
+        9,
+        7,
+        12,
+        12,
+        12,
+        11,
+        12,
+        10,
+        7,
+        6,
+        9,
+        7,
+        6,
+        9,
+        13,
+        9,
+        13,
+        9,
+        13,
+        8,
+        13,
+        9,
+        13,
+        9,
+        13,
+        8,
+        8,
+        2,
+        8,
+        2
+    ],
+    "kising_434": [
+        9,
+        2,
+        8,
+        11,
+        5,
+        9,
+        14,
+        8,
+        11,
+        11,
+        6,
+        6,
+        2,
+        7,
+        2,
+        6,
+        7,
+        2,
+        7,
+        4,
+        7,
+        6,
+        8,
+        10,
+        5,
+        9,
+        7,
+        6,
+        9,
+        11,
+        11,
+        2,
+        6,
+        6,
+        9,
+        2,
+        6,
+        7,
+        2,
+        8,
+        4,
+        6,
+        7,
+        2,
+        7,
+        2,
+        6,
+        7,
+        2,
+        6,
+        5
+    ],
+    "kising_441-2": [
+        11,
+        4,
+        9,
+        4,
+        5,
+        4,
+        5,
+        14,
+        10,
+        23,
+        12,
+        4
+    ],
+    "kising_441": [
+        2,
+        6,
+        6,
+        6,
+        18,
+        4,
+        24,
+        6,
+        12,
+        12,
+        12,
+        4,
+        7,
+        4,
+        9,
+        4,
+        4,
+        4,
+        13,
+        7,
+        1,
+        7,
+        13,
+        18,
+        4,
+        24,
+        6,
+        12,
+        12,
+        12,
+        4
+    ],
+    "kising_442": [
+        9,
+        8,
+        8,
+        11,
+        9,
+        8,
+        19,
+        8,
+        8,
+        8,
+        11,
+        9,
+        8,
+        8,
+        11,
+        9,
+        8,
+        19,
+        8,
+        8,
+        8,
+        11,
+        8,
+        8,
+        19,
+        9,
+        8,
+        8,
+        12,
+        8,
+        8,
+        8,
+        11,
+        8,
+        8,
+        19
+    ],
+    "kising_443": [
+        7,
+        16,
+        7,
+        8,
+        9,
+        3,
+        1,
+        3,
+        3,
+        33,
+        33,
+        7,
+        17,
+        1,
+        7,
+        8,
+        7,
+        2,
+        33,
+        33,
+        1,
+        5,
+        2,
+        1
+    ]
+}

offline_process/create_features.py

@@ -9,7 +9,19 @@ combined = combined.filter(lambda x: x["singer"] == "barber")
 
 # 3. create a new column, which counts the nonzero numbers in the list in the note_midi column
 combined = combined.map(
-    lambda x: {"note_midi_length": len([n for n in x["note_midi"] if n != 0])}
+    lambda x: {
+        "note_midi_length": len([n for n in x["note_midi"] if n != 0]),
+        "lyric_word_length": len(
+            [word for word in x["note_lyrics"] if word not in ["<AP>", "<SP>", "-"]]
+        ),  # counts the number of actual words (or characters for, e.g., Chinese/Japanese)
+    }
+)
+combined = combined.map(
+    lambda x: {
+        "lyric_word_length": len(
+            [word for word in x["note_lyrics"] if word not in ["<AP>", "<SP>", "-"]]
+        )
+    }  # counts the number of actual words (or characters for, e.g., Chinese/Japanese)
 )
 
 # 4. sort by segment_id
@@ -19,6 +31,7 @@ combined = combined.sort("segment_id")
 prev_songid = None
 prev_song_segment_id = None
 song2note_lengths = {}
+song2word_lengths = {}
 for row in combined:
     # segment_id: kising_barber_{songid}_{song_segment_id}
     _, _, songid, song_segment_id = row["segment_id"].split("_")
@@ -28,30 +41,26 @@ for row in combined:
                 song_segment_id == "001"
             ), f"prev_songid: {prev_songid}, songid: {songid}, song_segment_id: {song_segment_id}"
         song2note_lengths[f"kising_{songid}"] = [row["note_midi_length"]]
+        song2word_lengths[f"kising_{songid}"] = [row["lyric_word_length"]]
     else:
         assert (
             int(song_segment_id) >= int(prev_song_segment_id) + 1
         ), f"prev_song_segment_id: {prev_song_segment_id}, song_segment_id: {song_segment_id}"
         song2note_lengths[f"kising_{songid}"].append(row["note_midi_length"])
+        song2word_lengths[f"kising_{songid}"].append(row["lyric_word_length"])
     prev_songid = songid
     prev_song_segment_id = song_segment_id
 
 # 6. write to json
 import json
 
-with open("song2note_lengths.json", "w") as f:
+with open("data/song2note_lengths.json", "w") as f:
     json.dump(song2note_lengths, f, indent=4)
 
-# 7. convert to pandas DataFrame
-import pandas as pd
-
-df = pd.DataFrame.from_dict(combined)
-df = df.drop(columns=["audio", "singer"])
-df["segment_id"] = df["segment_id"].str.replace("kising_barber_", "kising_")
-# export to csv
-df.to_csv("song_db.csv", index=False)
+with open("data/song2word_lengths.json", "w") as f:
+    json.dump(song2word_lengths, f, indent=4)
 
-# 8. push score segments to hub
+# 7. push score segments to hub
 # remove audio and singer columns
 combined = combined.remove_columns(["audio", "singer"])
 # replace kising_barber_ with kising_

svs_utils.py

@@ -310,14 +310,17 @@ def song_segment_iterator(song_db, metadata):
         raise NotImplementedError(f"song name {song_name} not supported")
 
 
-def load_song_database():
+def load_song_database(config):
     song_db = load_dataset(
         "jhansss/kising_score_segments", cache_dir="cache", split="train"
     ).to_pandas()
     song_db.set_index("segment_id", inplace=True)
-
-    with open("data/song2note_lengths.json", "r") as f:
-        song2note_lengths = json.load(f)
+    if ".take_lyric_continuation" in config.melody_source:
+        with open("data/song2word_lengths.json", "r") as f:
+            song2note_lengths = json.load(f)
+    else:
+        with open("data/song2note_lengths.json", "r") as f:
+            song2note_lengths = json.load(f)
     return song2note_lengths, song_db
 
 
@@ -342,7 +345,7 @@ if __name__ == "__main__":
     if config.melody_source.startswith("random_select"):
         # load song database: jhansss/kising_score_segments
         from datasets import load_dataset
-        song2note_lengths, song_db = load_song_database()
+        song2note_lengths, song_db = load_song_database(config)
 
         # get song_name and phrase_length
         phrase_length, metadata = estimate_sentence_length(None, config, song2note_lengths)