""" Unified embedding generation for multiple tokenization strategies. Generates .npy embedding files in the same format as generate_graph_embeddings.py, allowing all downstream training and inference scripts to work unchanged. Usage: uv run python -m code.encoding_generation.generate_multi_embeddings \ --tokenizer simhash --domain blocks uv run python -m code.encoding_generation.generate_multi_embeddings \ --tokenizer random --domain blocks Output: data/encodings////.npy data/encodings////_goal.npy """ import argparse import os import re import sys import numpy as np import pddl import pddl.logic.predicates from tqdm import tqdm ALL_DOMAINS = ["blocks", "gripper", "logistics", "visitall-from-everywhere"] SPLITS = ["train", "validation", "test-interpolation", "test-extrapolation"] _PREDICATE_REGEX = re.compile(r"\(([\w-]+(?: [\w-]+)*)\)") def progress_enabled() -> bool: return bool(sys.stdout.isatty()) def _extract_goal_atoms(problem) -> list[str]: """Extract goal atoms from a pddl Problem object as strings.""" goals = [] def visit(node): if isinstance(node, pddl.logic.predicates.Predicate): args = [t.name if hasattr(t, "name") else str(t) for t in node.terms] goals.append(f"({node.name} {' '.join(args)})") elif hasattr(node, "operands"): for op in node.operands: visit(op) elif hasattr(node, "_operands"): for op in node._operands: visit(op) visit(problem.goal) return goals def _get_objects(problem, domain) -> list[str]: """Get sorted object names from problem + domain constants.""" objs = set() for o in problem.objects: objs.add(o.name) for o in domain.constants: objs.add(o.name) return sorted(objs) def create_tokenizer(name: str, **kwargs): """Factory function to create a tokenizer by name.""" if name == "wl": from code.tokenization.wl import WLTokenizer return WLTokenizer(iterations=kwargs.get("iterations", 2)) elif name == "simhash": from code.tokenization.simhash import SimHashTokenizer return SimHashTokenizer( hash_dim=kwargs.get("hash_dim", 128), seed=kwargs.get("seed", 42), ) elif name == "shortest_path": from code.tokenization.shortest_path import ShortestPathTokenizer return ShortestPathTokenizer( max_path_length=kwargs.get("max_path_length", 5), ) elif name == "graphbpe": from code.tokenization.graphbpe import GraphBPETokenizer return GraphBPETokenizer( vocab_size=kwargs.get("vocab_size", 1000), num_iterations=kwargs.get("num_iterations", 100), ) elif name == "random": from code.tokenization.random import RandomTokenizer return RandomTokenizer( random_dim=kwargs.get("random_dim", 128), seed=kwargs.get("seed", 42), normalize=kwargs.get("normalize", True), ) else: raise ValueError(f"Unknown tokenizer: {name}") def main(): parser = argparse.ArgumentParser( description="Generate embeddings using multiple tokenization strategies." ) parser.add_argument( "--tokenizer", required=True, choices=["wl", "simhash", "shortest_path", "graphbpe", "random"], help="Tokenization strategy to use.", ) parser.add_argument("--data_dir", default="data") parser.add_argument("--output_dir", default=None, help="Override output directory") parser.add_argument("--model_dir", default=None, help="Override model save dir") parser.add_argument("--domain", type=str, default=None, help="Specific domain") # Tokenizer-specific params parser.add_argument("--iterations", type=int, default=2, help="WL iterations") parser.add_argument("--hash_dim", type=int, default=128, help="SimHash dimension") parser.add_argument("--seed", type=int, default=42, help="Random seed for SimHash") parser.add_argument( "--max_path_length", type=int, default=5, help="ShortestPath max length" ) parser.add_argument("--vocab_size", type=int, default=1000, help="GraphBPE vocab") parser.add_argument( "--num_iterations", type=int, default=100, help="GraphBPE merge iterations" ) parser.add_argument("--random_dim", type=int, default=128, help="Random baseline dimension") parser.add_argument( "--no_random_normalize", action="store_true", help="Disable unit-normalization for random embeddings", ) args = parser.parse_args() # Set output directories if args.output_dir is None: args.output_dir = os.path.join(args.data_dir, "encodings", args.tokenizer) if args.model_dir is None: args.model_dir = os.path.join(args.data_dir, "encodings", "models") os.makedirs(args.output_dir, exist_ok=True) os.makedirs(args.model_dir, exist_ok=True) domains_to_run = [args.domain] if args.domain else ALL_DOMAINS for domain_name in domains_to_run: print(f"\n{'='*60}") print(f"Domain: {domain_name} | Tokenizer: {args.tokenizer}") print(f"{'='*60}") domain_pddl = os.path.join(args.data_dir, "pddl", domain_name, "domain.pddl") train_states_dir = os.path.join(args.data_dir, "states", domain_name, "train") train_pddl_dir = os.path.join(args.data_dir, "pddl", domain_name, "train") if not os.path.exists(domain_pddl): print(f" [Error] Domain PDDL not found: {domain_pddl}") continue if not os.path.exists(train_states_dir): print(f" [Error] Training states not found: {train_states_dir}") continue # ---------- WL uses its own pipeline ---------- if args.tokenizer == "wl": _run_wl_pipeline(args, domain_name, domain_pddl, train_states_dir, train_pddl_dir) continue # ---------- Generic tokenizer pipeline ---------- # 1. Create and fit tokenizer tokenizer = create_tokenizer( args.tokenizer, iterations=args.iterations, hash_dim=args.hash_dim, seed=args.seed, max_path_length=args.max_path_length, vocab_size=args.vocab_size, num_iterations=args.num_iterations, random_dim=args.random_dim, normalize=(not args.no_random_normalize), ) print(f" Fitting {args.tokenizer} tokenizer...") tokenizer.fit(domain_pddl, train_states_dir, train_pddl_dir) print(f" Embedding dimension: {tokenizer.get_embedding_dim()}") # Save vocabulary vocab_path = os.path.join( args.model_dir, f"{domain_name}_{args.tokenizer}.json" ) tokenizer.save_vocabulary(vocab_path) print(f" Saved vocabulary to {vocab_path}") # Parse domain for object/goal extraction domain = pddl.parse_domain(domain_pddl) # 2. Embed all splits for split in SPLITS: print(f" Embedding split: {split}") split_state_dir = os.path.join( args.data_dir, "states", domain_name, split ) split_pddl_dir = os.path.join(args.data_dir, "pddl", domain_name, split) split_out_dir = os.path.join(args.output_dir, domain_name, split) os.makedirs(split_out_dir, exist_ok=True) if not os.path.exists(split_state_dir): print(f" Skipping {split} (states dir not found)") continue traj_files = sorted( [f for f in os.listdir(split_state_dir) if f.endswith(".traj")] ) for t_file in tqdm( traj_files, desc=f" Embedding {split}", disable=(not progress_enabled()), ): prob_name = t_file.replace(".traj", "") prob_pddl = os.path.join(split_pddl_dir, f"{prob_name}.pddl") traj_path = os.path.join(split_state_dir, t_file) out_traj_path = os.path.join(split_out_dir, f"{prob_name}.npy") out_goal_path = os.path.join(split_out_dir, f"{prob_name}_goal.npy") if not os.path.exists(prob_pddl): continue try: problem = pddl.parse_problem(prob_pddl) objects = _get_objects(problem, domain) goal_atoms = _extract_goal_atoms(problem) # Read trajectory with open(traj_path, "r") as f: lines = f.readlines() # Embed each state state_embeddings = [] for line in lines: state_atoms = _PREDICATE_REGEX.findall(line.strip()) # Wrap each match in parens to match expected format state_atoms_str = [f"({a})" for a in state_atoms] emb = tokenizer.transform_state( state_atoms_str, goal_atoms, objects ) state_embeddings.append(emb) traj_matrix = np.array(state_embeddings, dtype=np.float32) # Embed goal goal_vec = tokenizer.transform_goal(goal_atoms, objects) goal_vec = goal_vec.astype(np.float32) # Save np.save(out_traj_path, traj_matrix) np.save(out_goal_path, goal_vec) except Exception as e: print(f" Error embedding {prob_name}: {e}") print("\nDone!") def _run_wl_pipeline(args, domain_name, domain_pddl, train_states_dir, train_pddl_dir): """ Run the WL pipeline using the WLTokenizer wrapper. This produces output identical to generate_graph_embeddings.py but going through the tokenizer abstraction. """ from code.tokenization.wl import WLTokenizer tokenizer = WLTokenizer(iterations=args.iterations) tokenizer.fit(domain_pddl, train_states_dir, train_pddl_dir) print(f" WL Embedding dimension: {tokenizer.get_embedding_dim()}") # Save vocabulary vocab_path = os.path.join(args.model_dir, f"{domain_name}_wl_tok.json") tokenizer.save_vocabulary(vocab_path) print(f" Saved WL vocabulary to {vocab_path}") # Use wlplan directly for embedding (consistent with original pipeline) from wlplan.data import DomainDataset, ProblemDataset from wlplan.planning import Atom, State, parse_domain, parse_problem wl_domain = parse_domain(domain_pddl) pred_map = {p.name: p for p in wl_domain.predicates} def parse_line_to_state(line): line = line.strip() if not line: return State([]) matches = re.findall(r"\(([\w-]+(?: [\w-]+)*)\)", line) atoms = [] for m in matches: parts = m.split() if parts[0] in pred_map: atoms.append(Atom(pred_map[parts[0]], parts[1:])) return State(atoms) for split in SPLITS: print(f" Embedding split: {split}") split_state_dir = os.path.join(args.data_dir, "states", domain_name, split) split_pddl_dir = os.path.join(args.data_dir, "pddl", domain_name, split) split_out_dir = os.path.join(args.output_dir, domain_name, split) os.makedirs(split_out_dir, exist_ok=True) if not os.path.exists(split_state_dir): continue traj_files = sorted( [f for f in os.listdir(split_state_dir) if f.endswith(".traj")] ) for t_file in tqdm( traj_files, desc=f" Embedding {split}", disable=(not progress_enabled()), ): prob_name = t_file.replace(".traj", "") prob_pddl = os.path.join(split_pddl_dir, f"{prob_name}.pddl") traj_path = os.path.join(split_state_dir, t_file) out_traj_path = os.path.join(split_out_dir, f"{prob_name}.npy") out_goal_path = os.path.join(split_out_dir, f"{prob_name}_goal.npy") if not os.path.exists(prob_pddl): continue try: wl_prob = parse_problem(domain_pddl, prob_pddl) with open(traj_path, "r") as f: lines = f.readlines() states = [parse_line_to_state(l) for l in lines] # Embed trajectory via wlplan mini_ds = DomainDataset( wl_domain, [ProblemDataset(wl_prob, states)] ) embs = tokenizer._feature_gen.embed(mini_ds) traj_matrix = np.array(embs, dtype=np.float32) # Embed goal goal_atoms = list(wl_prob.positive_goals) goal_state = State(goal_atoms) goal_ds = DomainDataset( wl_domain, [ProblemDataset(wl_prob, [goal_state])] ) goal_embs = tokenizer._feature_gen.embed(goal_ds) goal_vec = np.array(goal_embs[0], dtype=np.float32) np.save(out_traj_path, traj_matrix) np.save(out_goal_path, goal_vec) except Exception as e: print(f" Error embedding {prob_name}: {e}") if __name__ == "__main__": main()