drive-v5 and the checkpoints it came from, with a model card

ee68aa6 verified 5 days ago

5.4 kB

"""A small encoder-decoder for the pavement mask.
About 1.6 M parameters, all standard convolutions, so it exports to TensorRT
without surprises and runs far inside the 20 fps budget on the Orin. The input is
the 240 x 135 half-resolution frame; the output is one logit per pixel at the
same size. A dilated bottleneck gives the receptive field to see the whole road
shape at once, which matters more than texture: pavement and grass can be the
same color, and the shape of the corridor still says which is which.
"""

from __future__ import annotations

import torch
import torch.nn as nn
import torch.nn.functional as F


def block(cin, cout, stride=1, dilation=1):
    return nn.Sequential(
        nn.Conv2d(cin, cout, 3, stride, padding=dilation, dilation=dilation, bias=False),
        nn.BatchNorm2d(cout),
        nn.ReLU(inplace=True),
        nn.Conv2d(cout, cout, 3, 1, padding=dilation, dilation=dilation, bias=False),
        nn.BatchNorm2d(cout),
        nn.ReLU(inplace=True),
    )


class PavementNet(nn.Module):
    def __init__(self, width=24):
        super().__init__()
        w = width
        self.e1 = block(3, w, stride=2)  # 1/2
        self.e2 = block(w, 2 * w, stride=2)  # 1/4
        self.e3 = block(2 * w, 4 * w, stride=2)  # 1/8
        self.e4 = block(4 * w, 6 * w, stride=2)  # 1/16
        self.context = nn.Sequential(block(6 * w, 6 * w, dilation=2), block(6 * w, 6 * w, dilation=4))
        self.d3 = block(6 * w + 4 * w, 4 * w)
        self.d2 = block(4 * w + 2 * w, 2 * w)
        self.d1 = block(2 * w + w, w)
        self.head = nn.Conv2d(w, 1, 1)

    @staticmethod
    def _up(x, like):
        return F.interpolate(x, size=like.shape[-2:], mode="bilinear", align_corners=False)

    def forward(self, x):
        size = x.shape[-2:]
        x = x / 255.0 - 0.5
        e1 = self.e1(x)
        e2 = self.e2(e1)
        e3 = self.e3(e2)
        e4 = self.context(self.e4(e3))
        d3 = self.d3(torch.cat([self._up(e4, e3), e3], 1))
        d2 = self.d2(torch.cat([self._up(d3, e2), e2], 1))
        d1 = self.d1(torch.cat([self._up(d2, e1), e1], 1))
        return F.interpolate(self.head(d1), size=size, mode="bilinear", align_corners=False)[:, 0]


class DriveNet(PavementNet):
    """PavementNet with a driving head: from the same encoder, the racing line
    as ``n_waypoints`` points in the kart's frame (x forward, y left) and the
    target speed at each. The mask head stays, so the stack can still check the
    driving output against the pavement it sees.
    The head reads the whole bottleneck, not a pooled vector, because where the
    road goes is a matter of where in the image it is. Outputs are scaled to
    about unit size: x / 25 m, y / 10 m, speed / 15 m/s.
    """

    SCALE = (25.0, 10.0, 15.0)

    def __init__(self, width=24, n_waypoints=12, grid=(9, 15), fine=False):
        super().__init__(width)
        self.n_waypoints = n_waypoints
        self.squeeze = nn.Sequential(nn.Conv2d(6 * width, 32, 1), nn.BatchNorm2d(32), nn.ReLU(inplace=True))
        self.grid = grid
        # ``fine``: the head also reads the decoder's quarter-resolution features
        # with two pixel-coordinate channels, then strides them down to the grid.
        # Where the kart sits relative to the line is read from where the edges
        # are 5 to 10 m ahead, a matter of a few pixels the 1/16 bottleneck blurs.
        self.fine = fine
        if fine:
            self.fine_path = nn.Sequential(
                nn.Conv2d(2 * width + 2, 64, 3, 2, 1, bias=False),
                nn.BatchNorm2d(64),
                nn.ReLU(inplace=True),
                nn.Conv2d(64, 64, 3, 2, 1, bias=False),
                nn.BatchNorm2d(64),
                nn.ReLU(inplace=True),
                nn.Conv2d(64, 32, 1),
                nn.BatchNorm2d(32),
                nn.ReLU(inplace=True),
            )
        head_in = 64 if fine else 32
        self.drive = nn.Sequential(
            nn.Flatten(),
            nn.Linear(head_in * grid[0] * grid[1], 256),
            nn.ReLU(inplace=True),
            nn.Dropout(0.1),
            nn.Linear(256, 3 * n_waypoints),
        )

    def forward_all(self, x):
        """(mask logits N x H x W, waypoints N x K x 2 in metres, speeds N x K in m/s)."""
        size = x.shape[-2:]
        x = x / 255.0 - 0.5
        e1 = self.e1(x)
        e2 = self.e2(e1)
        e3 = self.e3(e2)
        e4 = self.context(self.e4(e3))
        d3 = self.d3(torch.cat([self._up(e4, e3), e3], 1))
        d2 = self.d2(torch.cat([self._up(d3, e2), e2], 1))
        d1 = self.d1(torch.cat([self._up(d2, e1), e1], 1))
        mask = F.interpolate(self.head(d1), size=size, mode="bilinear", align_corners=False)[:, 0]
        code = F.adaptive_avg_pool2d(self.squeeze(e4), self.grid)
        if self.fine:
            n, _, h, w = d2.shape
            ys = torch.linspace(-1, 1, h, device=d2.device).view(1, 1, h, 1).expand(n, 1, h, w)
            xs = torch.linspace(-1, 1, w, device=d2.device).view(1, 1, 1, w).expand(n, 1, h, w)
            detail = self.fine_path(torch.cat([d2, ys, xs], 1))
            code = torch.cat([code, F.adaptive_avg_pool2d(detail, self.grid)], 1)
        out = self.drive(code).view(-1, self.n_waypoints, 3)
        sx, sy, sv = self.SCALE
        points = torch.stack([out[..., 0] * sx, out[..., 1] * sy], -1)
        speeds = out[..., 2] * sv
        return mask, points, speeds

	"""A small encoder-decoder for the pavement mask.

	About 1.6 M parameters, all standard convolutions, so it exports to TensorRT
	without surprises and runs far inside the 20 fps budget on the Orin. The input is
	the 240 x 135 half-resolution frame; the output is one logit per pixel at the
	same size. A dilated bottleneck gives the receptive field to see the whole road
	shape at once, which matters more than texture: pavement and grass can be the
	same color, and the shape of the corridor still says which is which.
	"""

	from __future__ import annotations

	import torch
	import torch.nn as nn
	import torch.nn.functional as F


	def block(cin, cout, stride=1, dilation=1):
	return nn.Sequential(
	nn.Conv2d(cin, cout, 3, stride, padding=dilation, dilation=dilation, bias=False),
	nn.BatchNorm2d(cout),
	nn.ReLU(inplace=True),
	nn.Conv2d(cout, cout, 3, 1, padding=dilation, dilation=dilation, bias=False),
	nn.BatchNorm2d(cout),
	nn.ReLU(inplace=True),
	)


	class PavementNet(nn.Module):
	def __init__(self, width=24):
	super().__init__()
	w = width
	self.e1 = block(3, w, stride=2) # 1/2
	self.e2 = block(w, 2 * w, stride=2) # 1/4
	self.e3 = block(2 * w, 4 * w, stride=2) # 1/8
	self.e4 = block(4 * w, 6 * w, stride=2) # 1/16
	self.context = nn.Sequential(block(6 * w, 6 * w, dilation=2), block(6 * w, 6 * w, dilation=4))
	self.d3 = block(6 * w + 4 * w, 4 * w)
	self.d2 = block(4 * w + 2 * w, 2 * w)
	self.d1 = block(2 * w + w, w)
	self.head = nn.Conv2d(w, 1, 1)

	@staticmethod
	def _up(x, like):
	return F.interpolate(x, size=like.shape[-2:], mode="bilinear", align_corners=False)

	def forward(self, x):
	size = x.shape[-2:]
	x = x / 255.0 - 0.5
	e1 = self.e1(x)
	e2 = self.e2(e1)
	e3 = self.e3(e2)
	e4 = self.context(self.e4(e3))
	d3 = self.d3(torch.cat([self._up(e4, e3), e3], 1))
	d2 = self.d2(torch.cat([self._up(d3, e2), e2], 1))
	d1 = self.d1(torch.cat([self._up(d2, e1), e1], 1))
	return F.interpolate(self.head(d1), size=size, mode="bilinear", align_corners=False)[:, 0]


	class DriveNet(PavementNet):
	"""PavementNet with a driving head: from the same encoder, the racing line
	as ``n_waypoints`` points in the kart's frame (x forward, y left) and the
	target speed at each. The mask head stays, so the stack can still check the
	driving output against the pavement it sees.

	The head reads the whole bottleneck, not a pooled vector, because where the
	road goes is a matter of where in the image it is. Outputs are scaled to
	about unit size: x / 25 m, y / 10 m, speed / 15 m/s.
	"""

	SCALE = (25.0, 10.0, 15.0)

	def __init__(self, width=24, n_waypoints=12, grid=(9, 15), fine=False):
	super().__init__(width)
	self.n_waypoints = n_waypoints
	self.squeeze = nn.Sequential(nn.Conv2d(6 * width, 32, 1), nn.BatchNorm2d(32), nn.ReLU(inplace=True))
	self.grid = grid
	# ``fine``: the head also reads the decoder's quarter-resolution features
	# with two pixel-coordinate channels, then strides them down to the grid.
	# Where the kart sits relative to the line is read from where the edges
	# are 5 to 10 m ahead, a matter of a few pixels the 1/16 bottleneck blurs.
	self.fine = fine
	if fine:
	self.fine_path = nn.Sequential(
	nn.Conv2d(2 * width + 2, 64, 3, 2, 1, bias=False),
	nn.BatchNorm2d(64),
	nn.ReLU(inplace=True),
	nn.Conv2d(64, 64, 3, 2, 1, bias=False),
	nn.BatchNorm2d(64),
	nn.ReLU(inplace=True),
	nn.Conv2d(64, 32, 1),
	nn.BatchNorm2d(32),
	nn.ReLU(inplace=True),
	)
	head_in = 64 if fine else 32
	self.drive = nn.Sequential(
	nn.Flatten(),
	nn.Linear(head_in * grid[0] * grid[1], 256),
	nn.ReLU(inplace=True),
	nn.Dropout(0.1),
	nn.Linear(256, 3 * n_waypoints),
	)

	def forward_all(self, x):
	"""(mask logits N x H x W, waypoints N x K x 2 in metres, speeds N x K in m/s)."""
	size = x.shape[-2:]
	x = x / 255.0 - 0.5
	e1 = self.e1(x)
	e2 = self.e2(e1)
	e3 = self.e3(e2)
	e4 = self.context(self.e4(e3))
	d3 = self.d3(torch.cat([self._up(e4, e3), e3], 1))
	d2 = self.d2(torch.cat([self._up(d3, e2), e2], 1))
	d1 = self.d1(torch.cat([self._up(d2, e1), e1], 1))
	mask = F.interpolate(self.head(d1), size=size, mode="bilinear", align_corners=False)[:, 0]
	code = F.adaptive_avg_pool2d(self.squeeze(e4), self.grid)
	if self.fine:
	n, _, h, w = d2.shape
	ys = torch.linspace(-1, 1, h, device=d2.device).view(1, 1, h, 1).expand(n, 1, h, w)
	xs = torch.linspace(-1, 1, w, device=d2.device).view(1, 1, 1, w).expand(n, 1, h, w)
	detail = self.fine_path(torch.cat([d2, ys, xs], 1))
	code = torch.cat([code, F.adaptive_avg_pool2d(detail, self.grid)], 1)
	out = self.drive(code).view(-1, self.n_waypoints, 3)
	sx, sy, sv = self.SCALE
	points = torch.stack([out[..., 0] * sx, out[..., 1] * sy], -1)
	speeds = out[..., 2] * sv
	return mask, points, speeds