EdgeASR / source /icefall /checkpoint.py
frankyoujian's picture
Add offline EdgeASR decoder and FFASR runtime materials
496a112 verified
Raw History Blame Contribute Delete
16.1 kB
# Copyright 2021-2022 Xiaomi Corporation (authors: Fangjun Kuang,
# Zengwei Yao)
#
# See ../../LICENSE for clarification regarding multiple authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import glob
import logging
import os
import re
from pathlib import Path
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
import torch
if TYPE_CHECKING:
from torch.amp import GradScaler
import torch.nn as nn
from lhotse.dataset.sampling.base import CutSampler
from torch import Tensor
from torch.nn.parallel import DistributedDataParallel as DDP
from torch.optim import Optimizer
# use duck typing for LRScheduler since we have different possibilities, see
# our class LRScheduler.
LRSchedulerType = object
def save_checkpoint(
filename: Path,
model: Union[nn.Module, DDP],
model_avg: Optional[nn.Module] = None,
params: Optional[Dict[str, Any]] = None,
optimizer: Optional[Optimizer] = None,
scheduler: Optional[LRSchedulerType] = None,
scaler: Optional["GradScaler"] = None,
sampler: Optional[CutSampler] = None,
rank: int = 0,
) -> None:
"""Save training information to a file.
Args:
filename:
The checkpoint filename.
model:
The model to be saved. We only save its `state_dict()`.
model_avg:
The stored model averaged from the start of training.
params:
User defined parameters, e.g., epoch, loss.
optimizer:
The optimizer to be saved. We only save its `state_dict()`.
scheduler:
The scheduler to be saved. We only save its `state_dict()`.
scalar:
The GradScaler to be saved. We only save its `state_dict()`.
rank:
Used in DDP. We save checkpoint only for the node whose rank is 0.
Returns:
Return None.
"""
if rank != 0:
return
logging.info(f"Saving checkpoint to {filename}")
if isinstance(model, DDP):
model = model.module
checkpoint = {
"model": model.state_dict(),
"optimizer": optimizer.state_dict() if optimizer is not None else None,
"scheduler": scheduler.state_dict() if scheduler is not None else None,
"grad_scaler": scaler.state_dict() if scaler is not None else None,
"sampler": sampler.state_dict() if sampler is not None else None,
}
if model_avg is not None:
checkpoint["model_avg"] = model_avg.to(torch.float32).state_dict()
if params:
for k, v in params.items():
assert k not in checkpoint, k
checkpoint[k] = v
torch.save(checkpoint, filename)
def load_checkpoint(
filename: Path,
model: nn.Module,
model_avg: Optional[nn.Module] = None,
optimizer: Optional[Optimizer] = None,
scheduler: Optional[LRSchedulerType] = None,
scaler: Optional["GradScaler"] = None,
sampler: Optional[CutSampler] = None,
strict: bool = False,
) -> Dict[str, Any]:
"""
TODO: document it
"""
logging.info(f"Loading checkpoint from {filename}")
checkpoint = torch.load(filename, map_location="cpu", weights_only=False)
if next(iter(checkpoint["model"])).startswith("module."):
logging.info("Loading checkpoint saved by DDP")
dst_state_dict = model.state_dict()
src_state_dict = checkpoint["model"]
for key in dst_state_dict.keys():
src_key = "{}.{}".format("module", key)
dst_state_dict[key] = src_state_dict.pop(src_key)
assert len(src_state_dict) == 0
model.load_state_dict(dst_state_dict, strict=strict)
else:
model.load_state_dict(checkpoint["model"], strict=strict)
checkpoint.pop("model")
if model_avg is not None and "model_avg" in checkpoint:
logging.info("Loading averaged model")
model_avg.load_state_dict(checkpoint["model_avg"], strict=strict)
checkpoint.pop("model_avg")
def load(name, obj):
s = checkpoint.get(name, None)
if obj and s:
obj.load_state_dict(s)
checkpoint.pop(name)
load("optimizer", optimizer)
load("scheduler", scheduler)
load("grad_scaler", scaler)
load("sampler", sampler)
return checkpoint
def average_checkpoints(
filenames: List[Path], device: torch.device = torch.device("cpu")
) -> dict:
"""Average a list of checkpoints.
Args:
filenames:
Filenames of the checkpoints to be averaged. We assume all
checkpoints are saved by :func:`save_checkpoint`.
device:
Move checkpoints to this device before averaging.
Returns:
Return a dict (i.e., state_dict) which is the average of all
model state dicts contained in the checkpoints.
"""
n = len(filenames)
avg = torch.load(filenames[0], map_location=device, weights_only=False)["model"]
# Identify shared parameters. Two parameters are said to be shared
# if they have the same data_ptr
uniqued: Dict[int, str] = dict()
for k, v in avg.items():
v_data_ptr = v.data_ptr()
if v_data_ptr in uniqued:
continue
uniqued[v_data_ptr] = k
uniqued_names = list(uniqued.values())
for i in range(1, n):
state_dict = torch.load(filenames[i], map_location=device, weights_only=False)[
"model"
]
for k in uniqued_names:
avg[k] += state_dict[k]
for k in uniqued_names:
if avg[k].is_floating_point():
avg[k] /= n
else:
avg[k] //= n
return avg
def save_checkpoint_with_global_batch_idx(
out_dir: Path,
global_batch_idx: int,
model: Union[nn.Module, DDP],
model_avg: Optional[nn.Module] = None,
params: Optional[Dict[str, Any]] = None,
optimizer: Optional[Optimizer] = None,
scheduler: Optional[LRSchedulerType] = None,
scaler: Optional["GradScaler"] = None,
sampler: Optional[CutSampler] = None,
rank: int = 0,
):
"""Save training info after processing given number of batches.
Args:
out_dir:
The directory to save the checkpoint.
global_batch_idx:
The number of batches processed so far from the very start of the
training. The saved checkpoint will have the following filename:
f'out_dir / checkpoint-{global_batch_idx}.pt'
model:
The neural network model whose `state_dict` will be saved in the
checkpoint.
model_avg:
The stored model averaged from the start of training.
params:
A dict of training configurations to be saved.
optimizer:
The optimizer used in the training. Its `state_dict` will be saved.
scheduler:
The learning rate scheduler used in the training. Its `state_dict` will
be saved.
scaler:
The scaler used for mix precision training. Its `state_dict` will
be saved.
sampler:
The sampler used in the training dataset.
rank:
The rank ID used in DDP training of the current node. Set it to 0
if DDP is not used.
"""
out_dir = Path(out_dir)
out_dir.mkdir(parents=True, exist_ok=True)
filename = out_dir / f"checkpoint-{global_batch_idx}.pt"
save_checkpoint(
filename=filename,
model=model,
model_avg=model_avg,
params=params,
optimizer=optimizer,
scheduler=scheduler,
scaler=scaler,
sampler=sampler,
rank=rank,
)
def find_checkpoints(out_dir: Path, iteration: int = 0) -> List[str]:
"""Find all available checkpoints in a directory.
The checkpoint filenames have the form: `checkpoint-xxx.pt`
where xxx is a numerical value.
Assume you have the following checkpoints in the folder `foo`:
- checkpoint-1.pt
- checkpoint-20.pt
- checkpoint-300.pt
- checkpoint-4000.pt
Case 1 (Return all checkpoints)::
find_checkpoints(out_dir='foo')
Case 2 (Return checkpoints newer than checkpoint-20.pt, i.e.,
checkpoint-4000.pt, checkpoint-300.pt, and checkpoint-20.pt)
find_checkpoints(out_dir='foo', iteration=20)
Case 3 (Return checkpoints older than checkpoint-20.pt, i.e.,
checkpoint-20.pt, checkpoint-1.pt)::
find_checkpoints(out_dir='foo', iteration=-20)
Args:
out_dir:
The directory where to search for checkpoints.
iteration:
If it is 0, return all available checkpoints.
If it is positive, return the checkpoints whose iteration number is
greater than or equal to `iteration`.
If it is negative, return the checkpoints whose iteration number is
less than or equal to `-iteration`.
Returns:
Return a list of checkpoint filenames, sorted in descending
order by the numerical value in the filename.
"""
checkpoints = list(glob.glob(f"{out_dir}/checkpoint-[0-9]*.pt"))
pattern = re.compile(r"checkpoint-([0-9]+).pt")
iter_checkpoints = []
for c in checkpoints:
result = pattern.search(c)
if not result:
logging.warn(f"Invalid checkpoint filename {c}")
continue
iter_checkpoints.append((int(result.group(1)), c))
# iter_checkpoints is a list of tuples. Each tuple contains
# two elements: (iteration_number, checkpoint-iteration_number.pt)
iter_checkpoints = sorted(iter_checkpoints, reverse=True, key=lambda x: x[0])
if iteration >= 0:
ans = [ic[1] for ic in iter_checkpoints if ic[0] >= iteration]
else:
ans = [ic[1] for ic in iter_checkpoints if ic[0] <= -iteration]
return ans
def remove_checkpoints(
out_dir: Path,
topk: int,
rank: int = 0,
):
"""Remove checkpoints from the given directory.
We assume that checkpoint filename has the form `checkpoint-xxx.pt`
where xxx is a number, representing the number of processed batches
when saving that checkpoint. We sort checkpoints by filename and keep
only the `topk` checkpoints with the highest `xxx`.
Args:
out_dir:
The directory containing checkpoints to be removed.
topk:
Number of checkpoints to keep.
rank:
If using DDP for training, it is the rank of the current node.
Use 0 if no DDP is used for training.
"""
assert topk >= 1, topk
if rank != 0:
return
checkpoints = find_checkpoints(out_dir)
if len(checkpoints) == 0:
logging.warn(f"No checkpoints found in {out_dir}")
return
if len(checkpoints) <= topk:
return
to_remove = checkpoints[topk:]
for c in to_remove:
os.remove(c)
def update_averaged_model(
params: Dict[str, Tensor],
model_cur: Union[nn.Module, DDP],
model_avg: nn.Module,
) -> None:
"""Update the averaged model:
model_avg = model_cur * (average_period / batch_idx_train)
+ model_avg * ((batch_idx_train - average_period) / batch_idx_train)
Args:
params:
User defined parameters, e.g., epoch, loss.
model_cur:
The current model.
model_avg:
The averaged model to be updated.
"""
weight_cur = params.average_period / params.batch_idx_train
weight_avg = 1 - weight_cur
if isinstance(model_cur, DDP):
model_cur = model_cur.module
cur = model_cur.state_dict()
avg = model_avg.state_dict()
average_state_dict(
state_dict_1=avg,
state_dict_2=cur,
weight_1=weight_avg,
weight_2=weight_cur,
)
def average_checkpoints_with_averaged_model(
filename_start: str,
filename_end: str,
device: torch.device = torch.device("cpu"),
) -> Dict[str, Tensor]:
"""Average model parameters over the range with given
start model (excluded) and end model.
Let start = batch_idx_train of model-start;
end = batch_idx_train of model-end;
interval = end - start.
Then the average model over range from start (excluded) to end is
(1) avg = (model_end * end - model_start * start) / interval.
It can be written as
(2) avg = model_end * weight_end + model_start * weight_start,
where weight_end = end / interval,
weight_start = -start / interval = 1 - weight_end.
Since the terms `weight_end` and `weight_start` would be large
if the model has been trained for lots of batches, which would cause
overflow when multiplying the model parameters.
To avoid this, we rewrite (2) as:
(3) avg = (model_end + model_start * (weight_start / weight_end))
* weight_end
The model index could be epoch number or iteration number.
Args:
filename_start:
Checkpoint filename of the start model. We assume it
is saved by :func:`save_checkpoint`.
filename_end:
Checkpoint filename of the end model. We assume it
is saved by :func:`save_checkpoint`.
device:
Move checkpoints to this device before averaging.
"""
state_dict_start = torch.load(
filename_start, map_location=device, weights_only=False
)
state_dict_end = torch.load(filename_end, map_location=device, weights_only=False)
average_period = state_dict_start["average_period"]
batch_idx_train_start = state_dict_start["batch_idx_train"]
batch_idx_train_start = (batch_idx_train_start // average_period) * average_period
batch_idx_train_end = state_dict_end["batch_idx_train"]
batch_idx_train_end = (batch_idx_train_end // average_period) * average_period
interval = batch_idx_train_end - batch_idx_train_start
assert interval > 0, interval
weight_end = batch_idx_train_end / interval
weight_start = 1 - weight_end
model_end = state_dict_end["model_avg"]
model_start = state_dict_start["model_avg"]
avg = model_end
# scale the weight to avoid overflow
average_state_dict(
state_dict_1=avg,
state_dict_2=model_start,
weight_1=1.0,
weight_2=weight_start / weight_end,
scaling_factor=weight_end,
)
return avg
def average_state_dict(
state_dict_1: Dict[str, Tensor],
state_dict_2: Dict[str, Tensor],
weight_1: float,
weight_2: float,
scaling_factor: float = 1.0,
) -> Dict[str, Tensor]:
"""Average two state_dict with given weights:
state_dict_1 = (state_dict_1 * weight_1 + state_dict_2 * weight_2)
* scaling_factor
It is an in-place operation on state_dict_1 itself.
"""
# Identify shared parameters. Two parameters are said to be shared
# if they have the same data_ptr
uniqued: Dict[int, str] = dict()
for k, v in state_dict_1.items():
v_data_ptr = v.data_ptr()
if v_data_ptr in uniqued:
continue
uniqued[v_data_ptr] = k
uniqued_names = list(uniqued.values())
for k in uniqued_names:
v = state_dict_1[k]
if torch.is_floating_point(v):
v *= weight_1
v += state_dict_2[k].to(device=state_dict_1[k].device) * weight_2
v *= scaling_factor