# ---------------------------------------------

# Copyright (c) OpenMMLab. All rights reserved.

# ---------------------------------------------

# Modified by Zhiqi Li

# ---------------------------------------------

from mmcv.ops.multi_scale_deform_attn import multi_scale_deformable_attn_pytorch

import warnings

import torch

import torch.nn as nn

import torch.nn.functional as F

from mmcv.cnn import xavier_init, constant_init

from mmcv.cnn.bricks.registry import (ATTENTION,

TRANSFORMER_LAYER,

TRANSFORMER_LAYER_SEQUENCE)

from mmcv.cnn.bricks.transformer import build_attention

import math

from mmcv.runner import force_fp32, auto_fp16

from mmcv.runner.base_module import BaseModule, ModuleList, Sequential

from mmcv.utils import ext_loader

from .multi_scale_deformable_attn_function import MultiScaleDeformableAttnFunction_fp32, \

MultiScaleDeformableAttnFunction_fp16

from projects.mmdet3d_plugin.models.utils.bricks import run_time

ext_module = ext_loader.load_ext(

'_ext', ['ms_deform_attn_backward', 'ms_deform_attn_forward'])

@ATTENTION.register_module()

class SpatialCrossAttention(BaseModule):

"""An attention module used in BEVFormer.

Args:

embed_dims (int): The embedding dimension of Attention.

Default: 256. 是bev线性变换后注意里特征数量

num_cams (int): The number of cameras 摄像头的数量

dropout (float): A Dropout layer on `inp_residual`.

Default: 0.. 为了防止过拟合dropout层参数

init_cfg (obj:`mmcv.ConfigDict`): The Config for initialization.

Default: None. 初始化参数

deformable_attention: (dict): The config for the deformable attention used in SCA. SCA的可变性注意力参数

"""

def __init__(self,

embed_dims=256,

num_cams=6,

pc_range=None,

dropout=0.1,

init_cfg=None,

batch_first=False,

deformable_attention=dict(

type='MSDeformableAttention3D',code>

embed_dims=256,

num_levels=4),

**kwargs

):

super(SpatialCrossAttention, self).__init__(init_cfg)

self.init_cfg = init_cfg

self.dropout = nn.Dropout(dropout)

self.pc_range = pc_range

self.fp16_enabled = False

self.deformable_attention = build_attention(deformable_attention)

self.embed_dims = embed_dims

self.num_cams = num_cams

self.output_proj = nn.Linear(embed_dims, embed_dims)

self.batch_first = batch_first

self.init_weight()

def init_weight(self):

"""Default initialization for Parameters of Module."""

xavier_init(self.output_proj, distribution='uniform', bias=0.)code>

#以上初始化和TSA基本一致，没有笔记内容

@force_fp32(apply_to=('query', 'key', 'value', 'query_pos', 'reference_points_cam'))

def forward(self,

query,

key,

value,

residual=None,

query_pos=None,

key_padding_mask=None,

reference_points=None,

spatial_shapes=None,

reference_points_cam=None,

bev_mask=None,

level_start_index=None,

flag='encoder',code>

**kwargs):

"""Forward Function of Detr3DCrossAtten.

Args:

query (Tensor): Query of Transformer with shape

(num_query, bs, embed_dims).

#Q

#网上解释num_query类似于 DETR里面的 object_queries，也就是最多预测多少个目标

key (Tensor): The key tensor with shape

`(num_key, bs, embed_dims)`.

# k

value (Tensor): The value tensor with shape

`(num_key, bs, embed_dims)`. (B, N, C, H, W)

residual (Tensor): The tensor used for addition, with the

same shape as `x`. Default None. If None, `x` will be used.

#残差

query_pos (Tensor): The positional encoding for `query`.

Default: None.

key_pos (Tensor): The positional encoding for `key`. Default

None.

# q 和 k的位置编码

reference_points (Tensor): The normalized reference

points with shape (bs, num_query, 4),

all elements is range in [0, 1], top-left (0,0),

bottom-right (1, 1), including padding area.

or (N, Length_{query}, num_levels, 4), add

additional two dimensions is (w, h) to

form reference boxes.

# 参考点归一化

数据标准化（ Standardization ）是将数据转换为均值为0，方差为1的数据，也就是将数据按比例缩放，

使得其分布具有标准正态分布。 数据归一化（ Normalization ）

是将数据转换为满足0≤x≤1的数据，也就是将数据缩放到 [0,1]区间。

#num_levels：The number of feature map used in

Attention 被用于注意力的特征地图的数量

key_padding_mask (Tensor): ByteTensor for `query`, with

shape [bs, num_key].

#k 注意力掩码

spatial_shapes (Tensor): Spatial shape of features in

different level. With shape (num_levels, 2),

last dimension represent (h, w).

#空间形状，在不同level的特征的空间形状，最后一个维度2是（h，w）

level_start_index (Tensor): The start index of each level.

A tensor has shape (num_levels) and can be represented

as [0, h_0*w_0, h_0*w_0+h_1*w_1, ...].

# 开始遍历的index

Returns:

Tensor: forwarded results with shape [num_query, bs, embed_dims].

# 返回查询数量 批次 特征数量

"""

if key is None:

key = query

if value is None:

value = key

if residual is None:

inp_residual = query

# 残差链接网络传输值被初始化为query

slots = torch.zeros_like(query)

# 以 query的shape初始化 slot

if query_pos is not None:

query = query + query_pos

# 同样地把线性层学习到的query位置编码和query叠加到一起

bs, num_query, _ = query.size()#(num_query, bs, embed_dims)

#这里是不是错了？ 在input地方的备注维度顺序不同？

D = reference_points_cam.size(3)

indexes = []

for i, mask_per_img in enumerate(bev_mask):

index_query_per_img = mask_per_img[0].sum(-1).nonzero().squeeze(-1)

indexes.append(index_query_per_img)

max_len = max([len(each) for each in indexes])

# 每个特征点对应一个mask点，特征点的值为false,就可以将其在注意力中抛弃

# 举例子说明：如果mask_per_img =m torch.tensor([[1, 0, 1, 0],[1, 1, 0, 1]])

# sum_per_img = mask_per_img.sum(-1) 得到tensor[2,3]

# nonzero_indices = sum_per_img.nonzero() 得到tensor [[0],[1]]

# index_query_per_img = nonzero_indices.squeeze(-1)去除上一步操作后多出来的维度

# 得到[0,1]

# 最后用indexes 储存计算好的indices

# each camera only interacts with its corresponding BEV queries. This step can greatly save GPU memory.

queries_rebatch = query.new_zeros(

[bs, self.num_cams, max_len, self.embed_dims])

reference_points_rebatch = reference_points_cam.new_zeros(

[bs, self.num_cams, max_len, D, 2])

for j in range(bs):

for i, reference_points_per_img in enumerate(reference_points_cam):

index_query_per_img = indexes[i]

queries_rebatch[j, i, :len(index_query_per_img)] = query[j, index_query_per_img]

reference_points_rebatch[j, i, :len(index_query_per_img)] = reference_points_per_img[j, index_query_per_img]

#重新计算q和reference point 根据上一步计算的index

num_cams, l, bs, embed_dims = key.shape

key = key.permute(2, 0, 1, 3).reshape(

bs * self.num_cams, l, self.embed_dims)

value = value.permute(2, 0, 1, 3).reshape(

bs * self.num_cams, l, self.embed_dims)

queries = self.deformable_attention(query=queries_rebatch.view(bs*self.num_cams, max_len, self.embed_dims), key=key, value=value,

reference_points=reference_points_rebatch.view(bs*self.num_cams, max_len, D, 2), spatial_shapes=spatial_shapes,

level_start_index=level_start_index).view(bs, self.num_cams, max_len, self.embed_dims)

# 使用可变形注意力

for j in range(bs):

for i, index_query_per_img in enumerate(indexes):

slots[j, index_query_per_img] += queries[j, i, :len(index_query_per_img)]

# 用计算好的 queries和indexed更新slots

count = bev_mask.sum(-1) > 0

# 将bev_mask 按照最后一个维度相加 判断是否大于0 结果储存在count中

count = count.permute(1, 2, 0).sum(-1)

count = torch.clamp(count, min=1.0) # 将count的元素的 最小值设为1

slots = slots / count[..., None]

slots = self.output_proj(slots)

return self.dropout(slots) + inp_residual # [num_query, bs, embed_dims].

@ATTENTION.register_module()

class MSDeformableAttention3D(BaseModule):

"""An attention module used in BEVFormer based on Deformable-Detr.

`Deformable DETR: Deformable Transformers for End-to-End Object Detection.

<https://arxiv.org/pdf/2010.04159.pdf>`_.

Args:

embed_dims (int): The embedding dimension of Attention.

Default: 256.

num_heads (int): Parallel attention heads. Default: 64.

num_levels (int): The number of feature map used in

Attention. Default: 4.

num_points (int): The number of sampling points for

each query in each head. Default: 4.

im2col_step (int): The step used in image_to_column.

Default: 64.

dropout (float): A Dropout layer on `inp_identity`.

Default: 0.1.

batch_first (bool): Key, Query and Value are shape of

(batch, n, embed_dim)

or (n, batch, embed_dim). Default to False.

norm_cfg (dict): Config dict for normalization layer.

Default: None.

init_cfg (obj:`mmcv.ConfigDict`): The Config for initialization.

Default: None.

"""

# embed_dims（嵌入维度）：注意力机制中的嵌入维度。默认为256，影响了注意力机制中的向量表示维度。

# num_heads（注意力头数）：并行的注意力头数。默认为64，控制了注意力机制中多头注意力的并行数量。

# num_levels（特征图数量）：注意力中使用的特征图数量。默认为4，影响了注意力机制中特征图的层级数。

# num_points（采样点数）：每个注意力头中每个查询点的采样点数。默认为4，决定了每个头部的注意力机制对查询点进行采样的数量。

# im2col_step（image_to_column 步长）：在 image_to_column 操作中使用的步长。

# dropout（丢弃率）：应用于 inp_identity 的 Dropout 层的丢弃率。默认为0.1，用于在训练中随机丢弃输入张量中的一部分元素，以防止过拟合。

# batch_first（批次优先）：用于指定输入张量的维度顺序。如果为 True，表示输入张量的形状是(batch, n, embed_dim)，否则为 (n, batch, embed_dim)。默认为 False。

# norm_cfg（归一化层配置）：用于归一化层的配置字典。默认为 None，

# init_cfg（初始化配置）：初始化配置的配置对象。

def __init__(self,

embed_dims=256,

num_heads=8,

num_levels=4,

num_points=8,

im2col_step=64,

dropout=0.1,

batch_first=True,

norm_cfg=None,

init_cfg=None):

super().__init__(init_cfg)

if embed_dims % num_heads != 0:

raise ValueError(f'embed_dims must be divisible by num_heads, '

f'but got { embed_dims} and { num_heads}')

dim_per_head = embed_dims // num_heads # 每一个头的特征数量

self.norm_cfg = norm_cfg

self.batch_first = batch_first

self.output_proj = None

self.fp16_enabled = False

# you'd better set dim_per_head to a power of 2

# which is more efficient in the CUDA implementation

def _is_power_of_2(n):

if (not isinstance(n, int)) or (n < 0):

raise ValueError(

'invalid input for _is_power_of_2: {} (type: {})'.format(

n, type(n)))

return (n & (n - 1) == 0) and n != 0

if not _is_power_of_2(dim_per_head):

warnings.warn(

"You'd better set embed_dims in "

'MultiScaleDeformAttention to make '

'the dimension of each attention head a power of 2 '

'which is more efficient in our CUDA implementation.')

self.im2col_step = im2col_step

self.embed_dims = embed_dims

self.num_levels = num_levels

self.num_heads = num_heads

self.num_points = num_points

self.sampling_offsets = nn.Linear(

embed_dims, num_heads * num_levels * num_points * 2)

self.attention_weights = nn.Linear(embed_dims,

num_heads * num_levels * num_points)

self.value_proj = nn.Linear(embed_dims, embed_dims)

# 同 TSA的注意力

self.init_weights()

def init_weights(self):

"""Default initialization for Parameters of Module."""

constant_init(self.sampling_offsets, 0.)

#极坐标网格构建

# 创建一个0到2pi 等分为8分的tensor

thetas = torch.arange(

self.num_heads,

dtype=torch.float32) * (2.0 * math.pi / self.num_heads)

# 初始化grid

#利用三角函数计算每个角度对应的余弦和正弦值，然后通过torch.stack在最后一个维度

#将这两个值堆叠在一起形成一个形状为(num_heads, 2)的张量。

# 这个张量的每一行表示一个角度对应的极坐标中的(x, y)坐标,

# 使用grid_init.abs().max(-1, keepdim=True)[0]计算每个行向量的绝对值中的最大值，

# 并在最后一个维度上保持维度。然后，将grid_init除以这个最大值，实现归一化。

# 最后，通过view函数将结果变形成形状为(num_heads, 1, 1, 2)的张量

#最终的输出是一个形状为(num_heads, 1, 1, 2)的张量，

# 表示了num_heads个头部的极坐标网格。每个头部的网格用一个(x, y)坐标表示，

# 这个坐标在单位圆上，且在整个num_heads中均匀分布

grid_init = torch.stack([thetas.cos(), thetas.sin()], -1)

grid_init = (grid_init /

grid_init.abs().max(-1, keepdim=True)[0]).view(

self.num_heads, 1, 1,

2).repeat(1, self.num_levels, self.num_points, 1)

## 遍历第二个维度上，通过这种方式记录是第几个采样点的极坐标

for i in range(self.num_points):

grid_init[:, :, i, :] *= i + 1

#grid_init.view(-1) 将 grid_init 张量展平为一个一维张量

self.sampling_offsets.bias.data = grid_init.view(-1)

constant_init(self.attention_weights, val=0., bias=0.)

xavier_init(self.value_proj, distribution='uniform', bias=0.)code>

xavier_init(self.output_proj, distribution='uniform', bias=0.)code>

self._is_init = True

def forward(self,

query,

key=None,

value=None,

identity=None,

query_pos=None,

key_padding_mask=None,

reference_points=None,

spatial_shapes=None,

level_start_index=None,

**kwargs):

"""Forward Function of MultiScaleDeformAttention.

Args:

query (Tensor): Query of Transformer with shape

( bs, num_query, embed_dims).

key (Tensor): The key tensor with shape

`(bs, num_key, embed_dims)`.

value (Tensor): The value tensor with shape

`(bs, num_key, embed_dims)`.

identity (Tensor): The tensor used for addition, with the

same shape as `query`. Default None. If None,

`query` will be used.

query_pos (Tensor): The positional encoding for `query`.

Default: None.

key_pos (Tensor): The positional encoding for `key`. Default

None.

reference_points (Tensor): The normalized reference

points with shape (bs, num_query, num_levels, 2),

all elements is range in [0, 1], top-left (0,0),

bottom-right (1, 1), including padding area.

or (N, Length_{query}, num_levels, 4), add

additional two dimensions is (w, h) to

form reference boxes.

key_padding_mask (Tensor): ByteTensor for `query`, with

shape [bs, num_key].

spatial_shapes (Tensor): Spatial shape of features in

different levels. With shape (num_levels, 2),

last dimension represents (h, w).

level_start_index (Tensor): The start index of each level.

A tensor has shape ``(num_levels, )`` and can be represented

as [0, h_0*w_0, h_0*w_0+h_1*w_1, ...].

Returns:

Tensor: forwarded results with shape [num_query, bs, embed_dims].

"""

if value is None:

value = query

if identity is None:

identity = query

if query_pos is not None:

query = query + query_pos

if not self.batch_first:

# change to (bs, num_query ,embed_dims)

query = query.permute(1, 0, 2)

value = value.permute(1, 0, 2)

bs, num_query, _ = query.shape

bs, num_value, _ = value.shape

assert (spatial_shapes[:, 0] * spatial_shapes[:, 1]).sum() == num_value

value = self.value_proj(value)

if key_padding_mask is not None:

value = value.masked_fill(key_padding_mask[..., None], 0.0)

value = value.view(bs, num_value, self.num_heads, -1)

sampling_offsets = self.sampling_offsets(query).view(

bs, num_query, self.num_heads, self.num_levels, self.num_points, 2)

attention_weights = self.attention_weights(query).view(

bs, num_query, self.num_heads, self.num_levels * self.num_points)

attention_weights = attention_weights.softmax(-1)

attention_weights = attention_weights.view(bs, num_query,

self.num_heads,

self.num_levels,

self.num_points)

if reference_points.shape[-1] == 2:

"""

For each BEV query, it owns `num_Z_anchors` in 3D space that having different heights.

After proejcting, each BEV query has `num_Z_anchors` reference points in each 2D image.

For each referent point, we sample `num_points` sampling points.

For `num_Z_anchors` reference points, it has overall `num_points * num_Z_anchors` sampling points.

"""

offset_normalizer = torch.stack(

[spatial_shapes[..., 1], spatial_shapes[..., 0]], -1)

bs, num_query, num_Z_anchors, xy = reference_points.shape

reference_points = reference_points[:, :, None, None, None, :, :]

sampling_offsets = sampling_offsets / \

offset_normalizer[None, None, None, :, None, :]

bs, num_query, num_heads, num_levels, num_all_points, xy = sampling_offsets.shape

sampling_offsets = sampling_offsets.view(

bs, num_query, num_heads, num_levels, num_all_points // num_Z_anchors, num_Z_anchors, xy)

sampling_locations = reference_points + sampling_offsets

bs, num_query, num_heads, num_levels, num_points, num_Z_anchors, xy = sampling_locations.shape

assert num_all_points == num_points * num_Z_anchors

sampling_locations = sampling_locations.view(

bs, num_query, num_heads, num_levels, num_all_points, xy)

elif reference_points.shape[-1] == 4:

assert False

else:

raise ValueError(

f'Last dim of reference_points must be'

f' 2 or 4, but get { reference_points.shape[-1]} instead.')

# sampling_locations.shape: bs, num_query, num_heads, num_levels, num_all_points, 2

# attention_weights.shape: bs, num_query, num_heads, num_levels, num_all_points

#准备步骤基本可TSA相同

if torch.cuda.is_available() and value.is_cuda:

if value.dtype == torch.float16:

MultiScaleDeformableAttnFunction = MultiScaleDeformableAttnFunction_fp32

else:

MultiScaleDeformableAttnFunction = MultiScaleDeformableAttnFunction_fp32

output = MultiScaleDeformableAttnFunction.apply(

value, spatial_shapes, level_start_index, sampling_locations,

attention_weights, self.im2col_step)

else:

output = multi_scale_deformable_attn_pytorch(

value, spatial_shapes, sampling_locations, attention_weights)

if not self.batch_first:

output = output.permute(1, 0, 2)

return output

拓展

<code>import torch

import torch.nn as nn

# 假设我们有一个词汇表的大小为10，每个词的嵌入维度为3

vocab_size = 10

embedding_dim = 3

# 创建一个 Embedding 层

embedding_layer = nn.Embedding(vocab_size, embedding_dim)

# 定义一个输入，包含三个词的索引

input_indices = torch.tensor([1, 5, 9], dtype=torch.long)

# 将输入传递给嵌入层，得到嵌入向量

embedded_vector = embedding_layer(input_indices)

# 输出嵌入向量

print(embedded_vector)

tensor([[-2.8465, 0.1365, -0.4851],

[ 0.4402, -0.3163, -0.8770],

[-0.4027, -0.1626, 0.3808]], grad_fn=<EmbeddingBackward0>)

“每个词的嵌入维度为3” 意味着在嵌入层中为每个词分配的嵌入向量的维度是3。嵌入向量是一个实数向量，用于表示模型学习到的词汇表中每个词的语义信息。

在这个特定的例子中，每个词都被映射到一个包含3个实数值的向量。这三个值构成了嵌入向量的三个维度。这种表示是通过训练神经网络从数据中学习得到的，网络通过反向传播算法来调整嵌入向量的权重，以最好地适应训练数据。

通常，嵌入维度的选择是一个超参数，可以根据具体任务和数据的性质进行调整。较大的嵌入维度可以提供更丰富的语义信息，但也需要更多的计算资源。一般而言，在实际应用中，嵌入维度的选择可能会在试验和验证中进行调整，以找到适合特定任务的最佳值

PerceptionTransformer （重点！）

从重中之重 Encoder入手 在modules的 transformer中

运动学对齐信息

输入和之前基本解释了，grid_length=[0.512, 0.512],代表每一个格子的现实长度，用于在格子中作现实世界和虚拟格子的变换

计算上一帧和本帧位置的偏移量，做一个时空对齐：

获取自我运动信息：

delta_x 和 delta_y 分别表示每个样本中车辆自我运动的x和y方向的变化。

ego_angle 表示车辆自我运动的角度计算平移量和旋转角度：

translation_length 表示车辆自我运动的平移量的长度，通过计算x和y方向的平方和的平方根得到。

translation_angle 表示车辆自我运动的平移方向的角度，通过使用反正切函数计算。

bev_angle 表示BEV中的旋转角度，由车辆自我运动的角度减去平移方向的角度得到。

shift_y 和 shift_x 表示BEV中的平移量，通过使用平移长度和旋转角度计算。应用平移操作：

shift_y 和 shift_x 分别乘以 self.use_shift，这是一个控制平移幅度的标志。

shift 是一个 PyTorch 张量，表示BEV中的平移，形状为 (bs, 2)。旋转先前的BEV：

如果存在 prev_bev，则检查其形状是否为 (bs, bev_h * bev_w, embed_dims)，如果不是，则进行维度调整。如果设置了 self.rotate_prev_bev 为 True，则对每个样本中的先前BEV执行旋转操作。

通过获取每个样本的旋转角度，并使用 rotate 函数，对先前BEV进行旋转操作。

最终将旋转后的BEV重新调整为适应模型的形状。

can信息处理

其中can_bus_mlp 可以跳转定义看到 把18维度的can信息映射了新的维度，视频up解说为把can信息也隐式的加入道路学习feature中

encoder学习 （在Perception TF中被调用，构建在encoder.py中）

以上就是6组backbone的循环对应下图 TSA SCA FFN，可以对照论文解读开始研究了。

TSA解读

找参考点

torch.linspace(0.5, H - 0.5, H）生成从 0.5 到 H - 0.5 的等间隔序列，后者同理。

torch.meshgrid 生成了两个张量 ref_y 和 ref_x，它们分别代表垂直和水平方向的坐标。假设 H 表示高度，W 表示宽度，那么 ref_y 的形状是 (H, W)，ref_x 的形状也是 (H, W)。

ref_y = ref_y.reshape(-1)[None] / H:

ref_y.reshape(-1) 将 ref_y 展平为一维张量，形状变为 (H * W)。因为 ref_y 开始是一个二维 (H, W) 的网格，展平后总共有 H * W 个元素。

[None] 在 PyTorch 中是用来增加一个维度的，这里增加了一个维度，将一维张量变成了二维，形状为 (1, H * W)。

最后，对这个二维张量中的每个元素都除以 H，对每个坐标点进行了标准化。因此，最终 ref_y 的形状是 (1, H * W)。

ref_2d = torch.stack((ref_x, ref_y), -1) 因为 ref_x 和 ref_y 都是形状为 (1, HW) 的二维张量，沿着最后一个维度堆叠相当于将它们合并成一个形状为 (1, HW, 2) 的三维张量

ref_2d.repeat(bs, 1, 1)

repeat 函数用于沿着指定的维度重复张量。在这里，bs 表示重复的次数，1 和 1 表示在其余两个维度上不进行重复。这将导致 ref_2d 在第一个维度上被重复 bs 次

unsqueeze(2) 操作在第三个维度上增加一个维度。在这里，它将 ref_2d 的形状从 (bs, HW, 2) 变成了 (bs, HW, 1, 2)

整个操作的目的是将原始的 ref_2d 张量在批处理维度上进行复制，并在最后增加一个额外的维度

扩展

关于stack，ref_2d = torch.stack((ref_x, ref_y), -1)

<code>ref_x = torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5, 0.6]])

ref_y = torch.tensor([[0.6, 0.5, 0.4, 0.3, 0.2, 0.1]])

ref_2d = torch.stack((ref_x, ref_y), -1)

#结果

tensor([[[0.1000, 0.6000],

[0.2000, 0.5000],

[0.3000, 0.4000],

[0.4000, 0.3000],

[0.5000, 0.2000],

[0.6000, 0.1000]]])

利用shift 做参考点时空间对齐

如果有历史，则把历史和现在bev参考点叠加，如果没有则加两次当前的。

进入·BEVFormerLayer类中寻找TSA层

详细代码见temporal_self_attention，value是上一帧pre_bev（事实上是本轮和上一帧一起的stack）,query是本轮bev_query,key没有用。

如果没有上一帧，用两个本轮叠加

位置编码

多头注意力处理 多头注意力优点和用法在transformer学习笔记有记录

位置掩码

为可变形注意力取点，代码里取4个点

1是 批次 40000是bev base 的200*200的特征空间大小 8个多头 2点历史帧 4个注意力的点 2为xy坐标

用于计算到reference point 的偏移量

而attention_weights 用于从4个点中确定最需要关心的点 然后把四个点的权重归一化

2是reference 和 shifted reference

sampling_locations 是真正意义上的采样点

2 是bev 和prev bev

每一个头的4个点就出来了

可变形注意力 在multi_scale_deformable_attn_function 中

代码结构（按照代码运行流程forward顺序再次解读）

推理流程按照forward即代码运行顺序，总结流程，直接在代码中注释，

参考来源

tools\ test.py

2. projects/mmdet3d_plugin/bevformer/apis/test.py

<code>def custom_multi_gpu_test(model, data_loader, tmpdir=None, gpu_collect=False):

"""

Args:

model (nn.Module): Model to be tested.

data_loader (nn.Dataloader): Pytorch data loader.

tmpdir (str): Path of directory to save the temporary results from

different gpus under cpu mode.

gpu_collect (bool): Option to use either gpu or cpu to collect results.

Returns:

list: The prediction results. 包含box框和

"""

model.eval()

# 设置模型为推理模式

bbox_results = []

mask_results = []

dataset = data_loader.dataset

rank, world_size = get_dist_info()

if rank == 0:

prog_bar = mmcv.ProgressBar(len(dataset))

time.sleep(2) # This line can prevent deadlock problem in some cases.

have_mask = False

for i, data in enumerate(data_loader):

#取index 和每一个创建的data单元

with torch.no_grad():

#关闭记录梯度

result = model(return_loss=False, rescale=True, **data)

# 进入到下一个处理环节

# encode mask results

if isinstance(result, dict): # 如果result是字典

if 'bbox_results' in result.keys():#寻找键 bbox。。。

bbox_result = result['bbox_results']

batch_size = len(result['bbox_results'])

bbox_results.extend(bbox_result) # 加入result所有结果

if 'mask_results' in result.keys() and result['mask_results'] is not None:

mask_result = custom_encode_mask_results(result['mask_results'])

mask_results.extend(mask_result)

have_mask = True

else:

batch_size = len(result)

bbox_results.extend(result)

#############省略######################

return { 'bbox_results': bbox_results, 'mask_results': mask_results}

projects/mmdet3d_plugin/bevformer/detectors/bevformer.py

def forward(self, return_loss=True, **kwargs):

"""Calls either forward_train or forward_test depending on whether

return_loss=True.

Note this setting will change the expected inputs. When

`return_loss=True`, img and img_metas are single-nested (i.e.

torch.Tensor and list[dict]), and when `resturn_loss=False`, img and

img_metas should be double nested (i.e. list[torch.Tensor],

list[list[dict]]), with the outer list indicating test time

augmentations.

"""

if return_loss:

return self.forward_train(**kwargs)

else:

return self.forward_test(**kwargs)

# 根据loss标志位决定是测试准确度还是训练模式，不同模式得返回值不同

forward_test

def forward_test(self, img_metas, img=None, **kwargs):

#############省略######################

new_prev_bev, bbox_results = self.simple_test(

img_metas[0], img[0], prev_bev=self.prev_frame_info['prev_bev'], **kwargs)

# During inference, we save the BEV features and ego motion of each timestamp.

return bbox_results

simple_test

def simple_test(self, img_metas, img=None, prev_bev=None, rescale=False):

"""Test function without augmentaiton."""

img_feats = self.extract_feat(img=img, img_metas=img_metas)

# self.extract_feat 主要包括两个步骤 img_backbone、img_neck，通过卷积提取特征

# 网络为resnet + FPN

# 如果是base模型，img_feats 为四个不同尺度的特征层

bbox_list = [dict() for i in range(len(img_metas))]

new_prev_bev, bbox_pts = self.simple_test_pts(

img_feats, img_metas, prev_bev, rescale=rescale)

for result_dict, pts_bbox in zip(bbox_list, bbox_pts):

result_dict['pts_bbox'] = pts_bbox

return new_prev_bev, bbox_list

simple_test_pts

# 对特征层进行编解码

outs = self.pts_bbox_head(x, img_metas, prev_bev=prev_bev)

projects/mmdet3d_plugin/bevformer/dense_heads/bevformer_head.py

class BEVFormerHead(DETRHead):

"""Head of Detr3D.

Args:

with_box_refine (bool): Whether to refine the reference points

in the decoder. Defaults to False.

as_two_stage (bool) : Whether to generate the proposal from

the outputs of encoder.

transformer (obj:`ConfigDict`): ConfigDict is used for building

the Encoder and Decoder.

bev_h, bev_w (int): spatial shape of BEV queries.

"""

if not self.as_two_stage:

#设定位置编码

self.bev_embedding = nn.Embedding(

self.bev_h * self.bev_w, self.embed_dims)

self.query_embedding = nn.Embedding(self.num_query,

self.embed_dims * 2)

#####################省略########################

def forward(self, mlvl_feats, img_metas, prev_bev=None, only_bev=False):

"""Forward function.

Args:

mlvl_feats (tuple[Tensor]): Features from the upstream

network, each is a 5D-tensor with shape

(B, N, C, H, W).

prev_bev: previous bev featues

only_bev: only compute BEV features with encoder.

Returns:

all_cls_scores (Tensor): Outputs from the classification head, \

shape [nb_dec, bs, num_query, cls_out_channels]. Note \

cls_out_channels should includes background.

all_bbox_preds (Tensor): Sigmoid outputs from the regression \

head with normalized coordinate format (cx, cy, w, l, cz, h, theta, vx, vy). \

Shape [nb_dec, bs, num_query, 9].

"""

# mlvl_feats: (tuple[Tensor]) FPN网络输出的多尺度特征

# prev_bev: 上一时刻的 bev_features

# all_cls_scores: 所有的类别得分信息

# all_bbox_preds: 所有预测框信息

bs, num_cam, _, _, _ = mlvl_feats[0].shape

dtype = mlvl_feats[0].dtype

# 给出数据类型

# 特征编码 (900,512) (900,256) concate (900 + 256)

object_query_embeds = self.query_embedding.weight.to(dtype)

# [2500,256] bev特征图的大小，最终bev的大小为 50*50，每个点的channel维度为256。

bev_queries = self.bev_embedding.weight.to(dtype)

# [1, 256, 50, 50] 可学习的位置编码

bev_mask = torch.zeros((bs, self.bev_h, self.bev_w),

device=bev_queries.device).to(dtype)

bev_pos = self.positional_encoding(bev_mask).to(dtype)

if only_bev: # only use encoder to obtain BEV features, TODO: refine the workaround

return self.transformer.get_bev_features(

mlvl_feats,

bev_queries,

self.bev_h,

self.bev_w,

grid_length=(self.real_h / self.bev_h,

self.real_w / self.bev_w),

bev_pos=bev_pos,

img_metas=img_metas,

prev_bev=prev_bev,

)

else:

# 进入transformer1

outputs = self.transformer(

mlvl_feats,

bev_queries,

object_query_embeds,

self.bev_h,

self.bev_w,

grid_length=(self.real_h / self.bev_h,

self.real_w / self.bev_w),

bev_pos=bev_pos,

reg_branches=self.reg_branches if self.with_box_refine else None, # noqa:E501

cls_branches=self.cls_branches if self.as_two_stage else None,

img_metas=img_metas,

prev_bev=prev_bev

)

8.projects/mmdet3d_plugin/bevformer/modules/transformer.py

@auto_fp16(apply_to=('mlvl_feats', 'bev_queries', 'object_query_embed', 'prev_bev', 'bev_pos'))

def forward(self,

mlvl_feats,

bev_queries,

object_query_embed,

bev_h,

bev_w,

grid_length=[0.512, 0.512],

bev_pos=None,

reg_branches=None,

cls_branches=None,

prev_bev=None,

**kwargs):

# 获得bev特征 temporal_self_attention + spatial_cross_attention

bev_embed = self.get_bev_features(

mlvl_feats,

bev_queries,

bev_h,

bev_w,

grid_length=grid_length,

bev_pos=bev_pos,

prev_bev=prev_bev,

**kwargs) # bev_embed shape: bs, bev_h*bev_w, embed_dims

# 可以进入encoder

inter_states, inter_references = self.decoder(

query=query,

key=None,

value=bev_embed,

query_pos=query_pos,

reference_points=reference_points,

reg_branches=reg_branches,

cls_branches=cls_branches,

spatial_shapes=torch.tensor([[bev_h, bev_w]], device=query.device),

level_start_index=torch.tensor([0], device=query.device),

**kwargs)

# decoder 进入 decoder

return bev_embed, inter_states, init_reference_out, inter_references_out

# 返回bevformer_head.py

get_bev_features

def get_bev_features(

self,

mlvl_feats,

bev_queries,

bev_h,

bev_w,

grid_length=[0.512, 0.512],

bev_pos=None,

prev_bev=None,

**kwargs):

# 车身底盘信号:速度、加速度等

# 当前帧的bev特征与历史特征进行 时间、空间上的对齐

bs = mlvl_feats[0].size(0)

bev_queries = bev_queries.unsqueeze(1).repeat(1, bs, 1)

bev_pos = bev_pos.flatten(2).permute(2, 0, 1)

# 展平bev_Pos

# obtain rotation angle and shift with ego motion

delta_x = np.array([each['can_bus'][0]

for each in kwargs['img_metas']])

delta_y = np.array([each['can_bus'][1]

for each in kwargs['img_metas']])

ego_angle = np.array(

[each['can_bus'][-2] / np.pi * 180 for each in kwargs['img_metas']])

grid_length_y = grid_length[0]

grid_length_x = grid_length[1]

translation_length = np.sqrt(delta_x ** 2 + delta_y ** 2)

translation_angle = np.arctan2(delta_y, delta_x) / np.pi * 180

# 计算偏移量

bev_angle = ego_angle - translation_angle

shift_y = translation_length * \

np.cos(bev_angle / 180 * np.pi) / grid_length_y / bev_h

shift_x = translation_length * \

np.sin(bev_angle / 180 * np.pi) / grid_length_x / bev_w

shift_y = shift_y * self.use_shift

shift_x = shift_x * self.use_shift

shift = bev_queries.new_tensor(

[shift_x, shift_y]).permute(1, 0) # xy, bs -> bs, xy

if prev_bev is not None:

if prev_bev.shape[1] == bev_h * bev_w:

prev_bev = prev_bev.permute(1, 0, 2)

if self.rotate_prev_bev:

for i in range(bs):

# num_prev_bev = prev_bev.size(1)

rotation_angle = kwargs['img_metas'][i]['can_bus'][-1]

tmp_prev_bev = prev_bev[:, i].reshape(

bev_h, bev_w, -1).permute(2, 0, 1)

tmp_prev_bev = rotate(tmp_prev_bev, rotation_angle,

center=self.rotate_center)

tmp_prev_bev = tmp_prev_bev.permute(1, 2, 0).reshape(

bev_h * bev_w, 1, -1)

prev_bev[:, i] = tmp_prev_bev[:, 0]

# add can bus signals

# 将can信息加入Q

can_bus = bev_queries.new_tensor(

[each['can_bus'] for each in kwargs['img_metas']]) # [:, :]

can_bus = self.can_bus_mlp(can_bus)[None, :, :]

bev_queries = bev_queries + can_bus * self.use_can_bus

feat_flatten = []

spatial_shapes = []

for lvl, feat in enumerate(mlvl_feats):

bs, num_cam, c, h, w = feat.shape

spatial_shape = (h, w)

feat = feat.flatten(3).permute(1, 0, 3, 2)

if self.use_cams_embeds:

feat = feat + self.cams_embeds[:, None, None, :].to(feat.dtype)

feat = feat + self.level_embeds[None,

None, lvl:lvl + 1, :].to(feat.dtype)

spatial_shapes.append(spatial_shape)

feat_flatten.append(feat)

feat_flatten = torch.cat(feat_flatten, 2)

spatial_shapes = torch.as_tensor(

spatial_shapes, dtype=torch.long, device=bev_pos.device)

# 每一个维度的起始点

level_start_index = torch.cat((spatial_shapes.new_zeros(

(1,)), spatial_shapes.prod(1).cumsum(0)[:-1]))

feat_flatten = feat_flatten.permute(

0, 2, 1, 3) # (num_cam, H*W, bs, embed_dims)

# 获得bev特征

bev_embed = self.encoder(

bev_queries,

feat_flatten,

feat_flatten,

bev_h=bev_h,

bev_w=bev_w,

bev_pos=bev_pos,

spatial_shapes=spatial_shapes,

level_start_index=level_start_index,

prev_bev=prev_bev,

shift=shift,

**kwargs

)

return bev_embed

projects/mmdet3d_plugin/bevformer/modules/encoder.py

def get_reference_points(H, W, Z=8, num_points_in_pillar=4, dim='3d', bs=1, device='cuda', dtype=torch.float):code>

# 获取参考点

"""Get the reference points used in SCA and TSA.

Args:

H, W: spatial shape of bev.

Z: hight of pillar.

D: sample D points uniformly from each pillar.

device (obj:`device`): The device where

reference_points should be.

Returns:

Tensor: reference points used in decoder, has \

shape (bs, num_keys, num_levels, 2).

"""

def forward(self,

bev_query,

key,

value,

*args,

bev_h=None,

bev_w=None,

bev_pos=None,

spatial_shapes=None,

level_start_index=None,

valid_ratios=None,

prev_bev=None,

shift=0.,

**kwargs):

"""Forward function for `TransformerDecoder`.

Args:

bev_query (Tensor): Input BEV query with shape

`(num_query, bs, embed_dims)`.

key & value (Tensor): Input multi-cameta features with shape

(num_cam, num_value, bs, embed_dims)

reference_points (Tensor): The reference

points of offset. has shape

(bs, num_query, 4) when as_two_stage,

otherwise has shape ((bs, num_query, 2).

valid_ratios (Tensor): The radios of valid

points on the feature map, has shape

(bs, num_levels, 2)

Returns:

Tensor: Results with shape [1, num_query, bs, embed_dims] when

return_intermediate is `False`, otherwise it has shape

[num_layers, num_query, bs, embed_dims].

"""

#通过叠加历史特征和偏移量得到当前帧特征点

ref_3d = self.get_reference_points(

bev_h, bev_w, self.pc_range[5]-self.pc_range[2], self.num_points_in_pillar, dim='3d', bs=bev_query.size(1), device=bev_query.device, dtype=bev_query.dtype)code>

ref_2d = self.get_reference_points(

bev_h, bev_w, dim='2d', bs=bev_query.size(1), device=bev_query.device, dtype=bev_query.dtype)code>

reference_points_cam, bev_mask = self.point_sampling(

ref_3d, self.pc_range, kwargs['img_metas'])

for lid, layer in enumerate(self.layers):

output = layer(

bev_query,

key,

value,

*args,

bev_pos=bev_pos,

ref_2d=hybird_ref_2d,

ref_3d=ref_3d,

bev_h=bev_h,

bev_w=bev_w,

spatial_shapes=spatial_shapes,

level_start_index=level_start_index,

reference_points_cam=reference_points_cam,

bev_mask=bev_mask,

prev_bev=prev_bev,

**kwargs) #进行6次重复

BEVFormerLayer 即为论文中的图

<code> for layer in self.operation_order:

# temporal self attention

if layer == 'self_attn': # 时间自注意力

query = self.attentions[attn_index](

query,

prev_bev,

prev_bev,

identity if self.pre_norm else None,

query_pos=bev_pos,

key_pos=bev_pos,

attn_mask=attn_masks[attn_index],

key_padding_mask=query_key_padding_mask,

reference_points=ref_2d,

spatial_shapes=torch.tensor(

[[bev_h, bev_w]], device=query.device),

level_start_index=torch.tensor([0], device=query.device),

**kwargs)

attn_index += 1

identity = query

elif layer == 'norm': #

query = self.norms[norm_index](query)

norm_index += 1

# spaital cross attention

elif layer == 'cross_attn': # 交叉空间注意力

query = self.attentions[attn_index](

query,

key,

value,

identity if self.pre_norm else None,

query_pos=query_pos,

key_pos=key_pos,

reference_points=ref_3d,

reference_points_cam=reference_points_cam,

mask=mask,

attn_mask=attn_masks[attn_index],

key_padding_mask=key_padding_mask,

spatial_shapes=spatial_shapes,

level_start_index=level_start_index,

**kwargs)

attn_index += 1

identity = query

elif layer == 'ffn': # 全连接输出

query = self.ffns[ffn_index](

query, identity if self.pre_norm else None)

ffn_index += 1

return query

projects/mmdet3d_plugin/bevformer/modules/temporal_self_attention.py

时间自注意力

self.im2col_step = im2col_step

self.embed_dims = embed_dims # bev 特征维度

self.num_levels = num_levels #多尺度特征层

self.num_heads = num_heads# 多头数量

self.num_points = num_points#采样点数

self.num_bev_queue = num_bev_queue # bev特征长度 时序上

self.sampling_offsets = nn.Linear(

embed_dims*self.num_bev_queue, num_bev_queue*num_heads * num_levels * num_points * 2) # 利用线形层学习采样偏移

self.attention_weights = nn.Linear(embed_dims*self.num_bev_queue,

num_bev_queue*num_heads * num_levels * num_points)# 利用线形层学习注意权重

self.value_proj = nn.Linear(embed_dims, embed_dims)

self.output_proj = nn.Linear(embed_dims, embed_dims)

def forward(self,

query,

key=None,

value=None,

identity=None,

query_pos=None,

key_padding_mask=None,

reference_points=None,

spatial_shapes=None,

level_start_index=None,

flag='decoder',code>

**kwargs):

"""Forward Function of MultiScaleDeformAttention.

Args:

query (Tensor): Query of Transformer with shape

(num_query, bs, embed_dims).

key (Tensor): The key tensor with shape

`(num_key, bs, embed_dims)`.

value (Tensor): The value tensor with shape

`(num_key, bs, embed_dims)`.

identity (Tensor): The tensor used for addition, with the

same shape as `query`. Default None. If None,

`query` will be used.

query_pos (Tensor): The positional encoding for `query`.

Default: None.

key_pos (Tensor): The positional encoding for `key`. Default

None.

reference_points (Tensor): The normalized reference

points with shape (bs, num_query, num_levels, 2),

all elements is range in [0, 1], top-left (0,0),

bottom-right (1, 1), including padding area.

or (N, Length_{query}, num_levels, 4), add

additional two dimensions is (w, h) to

form reference boxes.

key_padding_mask (Tensor): ByteTensor for `query`, with

shape [bs, num_key].

spatial_shapes (Tensor): Spatial shape of features in

different levels. With shape (num_levels, 2),

last dimension represents (h, w).

level_start_index (Tensor): The start index of each level.

A tensor has shape ``(num_levels, )`` and can be represented

as [0, h_0*w_0, h_0*w_0+h_1*w_1, ...].

Returns:

Tensor: forwarded results with shape [num_query, bs, embed_dims].

"""

# query 以当前时刻得bev特征Q查询上一时刻和上上时刻的bev特征值KV

#query: (1, 2500, 256) 当前时刻的bev特征图

#key: (2, 2500, 256) 上一个时刻的以及上上时刻的bev特征

#value: (2, 2500, 256) 上一个时刻的以及上上时刻的bev特征

#query_pos: 可学习的位置编码

#reference_points:每个bev特征点对应的坐标

value = torch.stack([query, query], 1).reshape(bs*2, len_bev, c)

# value 变化为用当前bev特征叠加

query = query + query_pos

# Q 加上位置编码信息

# 将前一时刻的bev和当前时刻的bev特征再特征维度上进行叠加

query = torch.cat([value[:bs], query], -1)

# 学习前一时刻和当前时刻的bev特征

value = self.value_proj(value)

# 从当前时刻的bev_query 学习到 参考点的偏置

sampling_offsets = self.sampling_offsets(query)

用于学习每个特征点之间的权重

attention_weights = self.attention_weights(query)

#多尺度可变性注意力

if value.dtype == torch.float16:

MultiScaleDeformableAttnFunction = MultiScaleDeformableAttnFunction_fp32

else:

MultiScaleDeformableAttnFunction = MultiScaleDeformableAttnFunction_fp32

output = MultiScaleDeformableAttnFunction.apply(

value, spatial_shapes, level_start_index, sampling_locations,

attention_weights, self.im2col_step)

else:

output = multi_scale_deformable_attn_pytorch(

value, spatial_shapes, sampling_locations, attention_weights)

# 线性层

output = self.output_proj(output)

# 残差链接

return self.dropout(output) + identity

返回到 projects/mmdet3d_plugin/bevformer/modules/encoder.py 中

Projects\mmdet3d_plugin\bevformer\modules\multi_scale_deformable_attn_function.py

Args:

value (Tensor): The value has shape

(bs, num_keys, mum_heads, embed_dims//num_heads)

value_spatial_shapes (Tensor): Spatial shape of

each feature map, has shape (num_levels, 2),

last dimension 2 represent (h, w)

sampling_locations (Tensor): The location of sampling points,

has shape

(bs ,num_queries, num_heads, num_levels, num_points, 2),

the last dimension 2 represent (x, y).

attention_weights (Tensor): The weight of sampling points used

when calculate the attention, has shape

(bs ,num_queries, num_heads, num_levels, num_points),

im2col_step (Tensor): The step used in image to column.

Returns:

Tensor: has shape (bs, num_queries, embed_dims)

"""

projects/mmdet3d_plugin/bevformer/modules/spatial_cross_attention.py

pc_range:真实世界的尺度```

query (Tensor): Query of Transformer with shape

(num_query, bs, embed_dims).

key (Tensor): The key tensor with shape

`(num_key, bs, embed_dims)`.

value (Tensor): The value tensor with shape

`(num_key, bs, embed_dims)`. (B, N, C, H, W)

residual (Tensor): The tensor used for addition, with the

same shape as `x`. Default None. If None, `x` will be used.

query_pos (Tensor): The positional encoding for `query`.

Default: None.

key_pos (Tensor): The positional encoding for `key`. Default

None.

reference_points (Tensor): The normalized reference

points with shape (bs, num_query, 4),

all elements is range in [0, 1], top-left (0,0),

bottom-right (1, 1), including padding area.

or (N, Length_{ query}, num_levels, 4), add

additional two dimensions is (w, h) to

form reference boxes.

key_padding_mask (Tensor): ByteTensor for `query`, with

shape [bs, num_key].

spatial_shapes (Tensor): Spatial shape of features in

different level. With shape (num_levels, 2),

last dimension represent (h, w).

level_start_index (Tensor): The start index of each level.

A tensor has shape (num_levels) and can be represented

as [0, h_0*w_0, h_0*w_0+h_1*w_1, ...].

Returns:

Tensor: forwarded results with shape [num_query, bs, embed_dims]

MSDeformableAttention3D

def forward(self,

query,

key=None,

value=None,

identity=None,

query_pos=None,

key_padding_mask=None,

reference_points=None,

spatial_shapes=None,

level_start_index=None,

**kwargs):

"""Forward Function of MultiScaleDeformAttention.

Args:

query (Tensor): Query of Transformer with shape

( bs, num_query, embed_dims).

key (Tensor): The key tensor with shape

`(bs, num_key, embed_dims)`.

value (Tensor): The value tensor with shape

`(bs, num_key, embed_dims)`.

identity (Tensor): The tensor used for addition, with the

same shape as `query`. Default None. If None,

`query` will be used.

query_pos (Tensor): The positional encoding for `query`.

Default: None.

key_pos (Tensor): The positional encoding for `key`. Default

None.

reference_points (Tensor): The normalized reference

points with shape (bs, num_query, num_levels, 2),

all elements is range in [0, 1], top-left (0,0),

bottom-right (1, 1), including padding area.

or (N, Length_{query}, num_levels, 4), add

additional two dimensions is (w, h) to

form reference boxes.

key_padding_mask (Tensor): ByteTensor for `query`, with

shape [bs, num_key].

spatial_shapes (Tensor): Spatial shape of features in

different levels. With shape (num_levels, 2),

last dimension represents (h, w).

level_start_index (Tensor): The start index of each level.

A tensor has shape ``(num_levels, )`` and can be represented

as [0, h_0*w_0, h_0*w_0+h_1*w_1, ...].

Returns:

Tensor: forwarded results with shape [num_query, bs, embed_dims].

"""

projects/mmdet3d_plugin/bevformer/modules/decoder.py

"""Forward function for `TransformerDecoderLayer`.

**kwargs contains some specific arguments of attentions.

Args:

query (Tensor): The input query with shape

[num_queries, bs, embed_dims] if

self.batch_first is False, else

[bs, num_queries embed_dims].

key (Tensor): The key tensor with shape [num_keys, bs,

embed_dims] if self.batch_first is False, else

[bs, num_keys, embed_dims] .

value (Tensor): The value tensor with same shape as `key`.

query_pos (Tensor): The positional encoding for `query`.

Default: None.

key_pos (Tensor): The positional encoding for `key`.

Default: None.

attn_masks (List[Tensor] | None): 2D Tensor used in

calculation of corresponding attention. The length of

it should equal to the number of `attention` in

`operation_order`. Default: None.

query_key_padding_mask (Tensor): ByteTensor for `query`, with

shape [bs, num_queries]. Only used in `self_attn` layer.

Defaults to None.

key_padding_mask (Tensor): ByteTensor for `query`, with

shape [bs, num_keys]. Default: None.

Returns:

Tensor: forwarded results with shape [num_queries, bs, embed_dims]. """

return output, reference_points

# 返回到 projects/mmdet3d_plugin/bevformer/modules/transformer.py