单发多框检测（SSD）

（一）代码实现 单发多框检测（SSD）

（1）多尺度锚框

可以先看看上一篇对目标检测算法的介绍，可能会有帮助。
【目标检测算法】https://www.jianshu.com/p/75c9ee311d8c
【锚框】https://www.jianshu.com/p/f196adddf142

其实说实话，我们对这个代码不需要完全明白，因为首先SSD是比较落后的算法了，其次没有人会真的手敲一遍，这难度太大，一般是了解其过程，看得懂代码，然后当一个调包侠。

当然如果你能知道每一个矩阵变化，脑海里有完美的网络结构的话。你是一个大佬，这是很好的。

ssd

%matplotlib inline
import torch 
from d2l import torch as d2l

img = d2l.plt.imread("../img/catdog.jpg")
h,w = img.shape[:2]

# 在特征图上（fmp）上生成锚框（anchors），每个单位（像素）作为锚框的中心。
def display_anchors(fmap_w, fmap_h, s):
    d2l.set_figsize()
    fmap = torch.zeros((1,10,fmap_w,fmap_h))
    anchors = d2l.multibox_prior(fmap,sizes=s,ratios=[1,0.5,2])
    bbox_scale = torch.tensor((w,h,w,h))
    d2l.show_bboxes(d2l.plt.imshow(img).axes,anchors[0]*bbox_scale)

这里我一开始是有疑惑的，为什么和之前的不一样，
不是要对每一个像素规划锚框吗？为什么这里选择了使用（4，4）的feature map
实际上这应该是在多尺度特征块上提取出来的feature map上做锚框，按比例映射到原图上。他并不会在原始图片上画锚框。

display_anchors(4,4,[0.15])

所以在下一个多尺度块的时候，特征提取会将feature map再次宽高减半，所以我们来看看这时候的锚框
这便是上一篇说的浅层检测小的物体，高层可以检测较大的物体。

display_anchors(2,2,s=[0.4])

到最后一层，只剩一个特征值的时候的锚框长什么样呢？

display_anchors(1,1,s=[0.8])

（2）从零SSD实现

%matplotlib inline
import torch 
import torchvision 
from torch import nn 
from torch.nn import functional as F
from d2l import torch as d2l 

# 预测一个锚框的类别
# 输入是一个feature map
# 经过这个卷积层，高宽不变，唯一变化的是通道数
# 输出通道等于锚框数量*（所有类别+1），1为背景类
# 输出通道的意思是，每一个通道是某个锚框在某一类上的预测值
def cls_predictor(num_inputs, num_anchors, num_classes):
    return nn.Conv2d(in_channels=num_inputs,out_channels=num_anchors*(num_classes+1),kernel_size=3,padding=1)

# 预测锚框的偏移量
# 每一个锚框的偏移量是四个值
# 所以锚框数量*4是所有锚框的所有偏移量的数量
# 输出通道表示的是，每一个通道是某个锚框的一个偏移量的值
def bbox_predictor(num_inputs,num_anchors):
    return nn.Conv2d(num_inputs,num_anchors*4,kernel_size=3,padding=1)

# 连接多尺度的预测
def forward(x, block):
    return block(x)

Y1 = forward(torch.zeros((2, 8, 20, 20)), cls_predictor(8, 5, 10))
Y2 = forward(torch.zeros((2, 16, 10, 10)), cls_predictor(16, 3, 10))
Y1.shape, Y2.shape #(torch.Size([2, 55, 20, 20]), torch.Size([2, 33, 10, 10]))

可以看到上述的输出都是会发生变化的，因为feature map的宽高会发生变化，锚框的个数也会发生变化。我们要想办法把这些预测值弄到一起。

def flatten_pred(pred):
    # 这里的操作是，把通道数放到最后，然后从第一维开始拉直，并且batchsize不会变
    # 为什么要把通道数放到最后一维呢？
    # 是因为他要保证一个像素的预测值是连续的
    return torch.flatten(pred.permute(0,2,3,1),start_dim=1)

def concat_preds(preds):
    # 按照维度为1的轴拼接，就是在行的后面直接接上后面的那个预测值
    # 要求批量数一样，
    return torch.cat([flatten_pred(p) for p in preds],dim=1)

concat_preds([Y1,Y2]).shape

# 我们定义一个简单的网络块，用于高宽减半
def down_sample_blk(in_channels, out_channels):
    blk = []
    for _ in range (2):
        blk.append(nn.Conv2d(in_channels, out_channels, kernel_size=3,padding=1))
        blk.append(nn.BatchNorm2d(out_channels))
        blk.append(nn.ReLU())
        in_channels = out_channels
    blk.append(nn.MaxPool2d(2))
    return nn.Sequential(*blk)

forward(torch.zeros((2,3,20,20)),down_sample_blk(3,10)).shape

# 基本网络块,也就是第一个网络块
def base_net():
    blk = []
    num_filters = [3,16,32,64]
    for i in range(len(num_filters)-1):
        blk.append(down_sample_blk(num_filters[i],num_filters[i+1]))
    return nn.Sequential(*blk)

forward(torch.zeros((2,3,256,256)),base_net()).shape

def get_blk(i):
    if i == 0:
        blk = base_net()
    elif i == 1:
        blk = down_sample_blk(64,128)
    elif i == 4:
        blk = nn.AdaptiveMaxPool2d((1,1))
    else:
        blk = down_sample_blk(128,128)
    return blk

def blk_forward(X, blk, size, ratio, cls_predictor, bbox_predictor):
    Y = blk(X)
    anchors = d2l.multibox_prior(Y, sizes=size, ratios=ratio)
    cls_preds = cls_predictor(Y) # 这是卷积层
    bbox_preds = bbox_predictor(Y) # 这是卷积层
    return (Y, anchors, cls_preds, bbox_preds)

sizes = [[0.2, 0.272], [0.37, 0.447], [0.54, 0.619], [0.71, 0.79],[0.88, 0.961]]
ratios = [[1, 2, 0.5]] * 5
num_anchors = len(sizes[0]) + len(ratios[0]) - 1

关于setattr的用法和作用：给对象设置属性并赋值。
Sets the named attribute on the given object to the specified value.
setattr(x, 'y', v) is equivalent to x.y = v''

class TinySSD(nn.Module):
    def __init__(self, num_classes, num_anchors,sizes,ratios, **kwargs) -> None:
        # 等同super().__init__(**kwargs)
        super(TinySSD, self).__init__(**kwargs)
        self.num_classes = num_classes
        idx_to_in_channels = [64,128,128,128,128]
        self.size = sizes
        self.ratio = ratios
        for i in range(5):
            # 整个模型分为5个stage
            setattr(self,f"blk{i}",get_blk(i))
            setattr(self,f"cls{i}",cls_predictor(idx_to_in_channels[i],num_anchors,num_classes))
            setattr(self,f"bbox{i}",bbox_predictor(idx_to_in_channels[i],num_anchors))

    def forward(self,x):
        anchors,cls_preds,bbox_preds = [None]*5,[None]*5,[None]*5
        for i in range(5):
            x,anchors[i],cls_preds[i],bbox_preds[i] = blk_forward(
                x, 
                getattr(self,f"blk{i}"), 
                self.size[i], self.ratio[i], 
                getattr(self,f"cls{i}"), 
                getattr(self,f"bbox{i}"), 
            )
        anchors = torch.cat(anchors,dim=1)
        cls_preds = concat_preds(cls_preds)
        cls_preds = cls_preds.reshape(cls_preds.shape[0],-1,self.num_classes+1)
        bbox_preds = concat_preds(bbox_preds)
        return anchors,cls_preds,bbox_preds

        
net = TinySSD(num_classes=1, num_anchors=num_anchors,sizes=sizes,ratios=ratios)
x = torch.zeros((32,3,256,256))
anchors, cls_preds, bbox_preds = net(x)

print("outshape anchors:",anchors.shape)
print("outshape cls_preds:",cls_preds.shape)
print("outshape bbox_preds:",bbox_preds.shape)
# outshape anchors: torch.Size([1, 5444, 4])
# outshape cls_preds: torch.Size([32, 5444, 2])
# outshape bbox_preds: torch.Size([32, 21776])

batch_size = 32
train_iter, _ = d2l.load_data_bananas(batch_size=batch_size)

device, net = d2l.try_gpu(), TinySSD(num_classes=1, num_anchors=num_anchors,sizes=sizes,ratios=ratios)
trainer = torch.optim.SGD(net.parameters(),lr=0.2, weight_decay=5e-4)

cls_loss = nn.CrossEntropyLoss(reduction="None")
bbox_loss = nn.L1Loss(reduction="None")

# 计算损失（分类损失和偏移损失）
def calc_loss(cls_preds, cls_labels, bbox_preds, bbox_labels, bbox_masks):
    batch_size,num_classes = cls_preds.shape[0], cls_preds.shape[2]
    # 计算锚框对每一个分类和真实分类之间的距离
    cls = cls_loss(cls_preds.reshape(-1, num_classes), cls_labels.reshape(-1)).reshape(batch_size, -1).mean(dim=1)
    bbox = bbox_loss(bbox_preds * bbox_masks, bbox_labels * bbox_masks).mean(dim=1)
    return cls + bbox

# 分类精度
def cls_eval(cls_preds, cls_labels):
    # 由于类别预测结果放在最后一维，argmax需要指定最后一维。
    return float((cls_preds.argmax(dim=-1).type(cls_labels.dtype) == cls_labels).sum())

def bbox_eval(bbox_preds, bbox_labels, bbox_masks):
    return float((torch.abs((bbox_labels - bbox_preds) * bbox_masks)).sum())

num_epochs, timer = 20, d2l.Timer()
animator = d2l.Animator(xlabel='epoch', xlim=[1, num_epochs],
                        legend=['class error', 'bbox mae'])
net = net.to(device)
for epoch in range(num_epochs):
    # 训练精确度的和，训练精确度的和中的示例数
    # 绝对误差的和，绝对误差的和中的示例数
    metric = d2l.Accumulator(4)
    net.train()
    for features, target in train_iter:
        timer.start()
        trainer.zero_grad()
        X, Y = features.to(device), target.to(device)
        # 生成多尺度的锚框，为每个锚框预测类别和偏移量
        anchors, cls_preds, bbox_preds = net(X)
        # 为每个锚框标注类别和偏移量
        bbox_labels, bbox_masks, cls_labels = d2l.multibox_target(anchors, Y)
        # 根据类别和偏移量的预测和标注值计算损失函数
        l = calc_loss(cls_preds, cls_labels, bbox_preds, bbox_labels,
                      bbox_masks)
        l.mean().backward()
        trainer.step()
        metric.add(cls_eval(cls_preds, cls_labels), cls_labels.numel(),
                   bbox_eval(bbox_preds, bbox_labels, bbox_masks),
                   bbox_labels.numel())
    cls_err, bbox_mae = 1 - metric[0] / metric[1], metric[2] / metric[3]
    animator.add(epoch + 1, (cls_err, bbox_mae))
print(f'class err {cls_err:.2e}, bbox mae {bbox_mae:.2e}')
print(f'{len(train_iter.dataset) / timer.stop():.1f} examples/sec on '
      f'{str(device)}')

# 预测一下
X = torchvision.io.read_image('../img/banana.jpg').unsqueeze(0).float()
img = X.squeeze(0).permute(1, 2, 0).long()

def predict(X):
    net.eval()
    anchors, cls_preds, bbox_preds = net(X.to(device))
    cls_probs = F.softmax(cls_preds, dim=2).permute(0, 2, 1)
    output = d2l.multibox_detection(cls_probs, bbox_preds, anchors)
    idx = [i for i, row in enumerate(output[0]) if row[0] != -1]
    return output[0, idx]

output = predict(X)

def display(img, output, threshold):
    d2l.set_figsize((5, 5))
    fig = d2l.plt.imshow(img)
    for row in output:
        score = float(row[1])
        if score < threshold:
            continue
        h, w = img.shape[0:2]
        bbox = [row[2:6] * torch.tensor((w, h, w, h), device=row.device)]
        d2l.show_bboxes(fig.axes, bbox, '%.2f' % score, 'w')

display(img, output.cpu(), threshold=0.9)