Faster R-CNN(Tensorflow) 源码解析

1
imdb, roidb = combined_roidb(args.imdb_name)

1
imdb = get_imdb(imdb_name)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
def gt_roidb(self):
    """
    Return the database of ground-truth regions of interest.

    This function loads/saves from/to a cache file to speed up future calls.
    """
    cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl')
    if os.path.exists(cache_file):
        with open(cache_file, 'rb') as fid:
            try:
                roidb = pickle.load(fid)
            except:
                roidb = pickle.load(fid, encoding='bytes')
        print('{} gt roidb loaded from {}'.format(self.name, cache_file))
        return roidb

    # a dict consists of bounding boxes an  objects of one image
    gt_roidb = [self._load_pascal_annotation(index) for index in self.image_index]
    with open(cache_file, 'wb') as fid:
        pickle.dump(gt_roidb, fid, pickle.HIGHEST_PROTOCOL)
    print('wrote gt roidb to {}'.format(cache_file))

    return gt_roidb

1
2
3
self._image = tf.placeholder(tf.float32, shape=[1, None, None, 3])	# [bs, h, w, c]
self._im_info = tf.placeholder(tf.float32, shape=[3])	# [h, w, image_scale]
self._gt_boxes = tf.placeholder(tf.float32, shape=[None, 5])

1
2
3
4
5
6
7
8
9
10
11
12
13
14
with tf.variable_scope(self._scope, self._scope, reuse=reuse):
  net = self._image
  net = slim.repeat(net, 2, slim.conv2d, 64, [3, 3], trainable=False, scope='conv1')
  net = slim.max_pool2d(net, [2, 2], padding='SAME', scope='pool1')
  net = slim.repeat(net, 2, slim.conv2d, 128, [3, 3], trainable=False, scope='conv2')
  net = slim.max_pool2d(net, [2, 2], padding='SAME', scope='pool2')
  net = slim.repeat(net, 3, slim.conv2d, 256, [3, 3], trainable=is_training, scope='conv3')
  net = slim.max_pool2d(net, [2, 2], padding='SAME', scope='pool3')
  net = slim.repeat(net, 3, slim.conv2d, 512, [3, 3], trainable=is_training, scope='conv4')
  net = slim.max_pool2d(net, [2, 2], padding='SAME', scope='pool4')
  net = slim.repeat(net, 3, slim.conv2d, 512, [3, 3], trainable=is_training, scope='conv5')

  self._act_summaries.append(net)
  self._layers['head'] = net

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
def generate_anchors(base_size=16, ratios=[0.5, 1, 2],
                     scales=2 ** np.arange(3, 6)):
    """
    Generate anchor (reference) windows by enumerating aspect ratios X
    scales wrt a reference (0, 0, 15, 15) window.
    """

    base_anchor = np.array([1, 1, base_size, base_size]) - 1
    ratio_anchors = _ratio_enum(base_anchor, ratios)
    #                                 h : w
    # [[ -3.5   2.   18.5  13. ]    0.5 : 1
    #  [  0.    0.   15.   15. ]      1 : 1
    #  [  2.5  -3.   12.5  18. ]]     2 : 1
    
    anchors = np.vstack([_scale_enum(ratio_anchors[i, :], scales)
                         for i in range(ratio_anchors.shape[0])])

    return anchors

1
2
3
4
5
6
7
8
9
10
11
12
13
A = anchors.shape[0]    # A = 9
shift_x = np.arange(0, width) * feat_stride
shift_y = np.arange(0, height) * feat_stride
shift_x, shift_y = np.meshgrid(shift_x, shift_y)

# ravel(): Return a flattened array.
# shift length of [x1, y1, x2, y2]
shifts = np.vstack((shift_x.ravel(), shift_y.ravel(),
                    shift_x.ravel(), shift_y.ravel())).transpose()
K = shifts.shape[0] # w * h
# width changes faster, so here it is H, W, C
anchors = anchors.reshape((1, A, 4)) + shifts.reshape((1, K, 4)).transpose((1, 0, 2))
anchors = anchors.reshape((K * A, 4)).astype(np.float32, copy=False)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
rpn = slim.conv2d(net_conv, cfg.RPN_CHANNELS, [3, 3], 
                  trainable=is_training,
                  weights_initializer=initializer,
                  scope="rpn_conv/3x3")
self._act_summaries.append(rpn)
# [bs, h, w, 9*2]
rpn_cls_score = slim.conv2d(rpn, self._num_anchors * 2, [1, 1], 
                            trainable=is_training,
                            weights_initializer=initializer,
                            padding='VALID', 
                            activation_fn=None, 
                            scope='rpn_cls_score')
...
...
...
# [bs, h, w, 9*4]
rpn_bbox_pred = slim.conv2d(rpn, self._num_anchors*4, [1, 1],
                            trainable=is_training,
                            weights_initializer=initializer,
                            padding='VALID', 
                            activation_fn=None, 
                            scope='rpn_bbox_pred')

1
2
3
4
5
6
7
# change it so that the score has 2 as its channel size
# [bs, h*9, w, 2]
rpn_cls_score_reshape = self._reshape_layer(rpn_cls_score, 2, "rpn_cls_score_reshape")  
rpn_cls_prob_reshape = self._softmax_layer(rpn_cls_score_reshape, "rpn_cls_prob_reshape")
rpn_cls_pred = tf.argmax(tf.reshape(rpn_cls_score_reshape,[-1,2]),axis=1, name="rpn_cls_pred")
# [bs, h, w, 2*9]
rpn_cls_prob = self._reshape_layer(rpn_cls_prob_reshape, self._num_anchors*2, "rpn_cls_prob") 

1
rois, roi_scores = self._proposal_layer(rpn_cls_prob, rpn_bbox_pred, "rois")

1
2
proposals = bbox_transform_inv(anchors, rpn_bbox_pred)
proposals = clip_boxes(proposals, im_info[:2])

1
2
3
4
5
order = scores.ravel().argsort()[::-1]
if pre_nms_topN > 0:
    order = order[:pre_nms_topN]
proposals = proposals[order, :]
scores = scores[order]

1
2
3
4
5
6
7
keep = nms(np.hstack((proposals, scores)), nms_thresh)

# Pick th top region proposals after NMS
if post_nms_topN > 0:
    keep = keep[:post_nms_topN]
proposals = proposals[keep, :]
scores = scores[keep]

1
2
3
# Only support single image as input
batch_inds = np.zeros((proposals.shape[0], 1), dtype=np.float32)
blob = np.hstack((batch_inds, proposals.astype(np.float32, copy=False)))

1
2
3
4
5
6
7
8
9
10
11
12
13
# allow boxes to sit over the edge by a small amount
_allowed_border = 0

# only keep anchors inside the image
inds_inside = np.where(
    (all_anchors[:, 0] >= -_allowed_border) &
    (all_anchors[:, 1] >= -_allowed_border) &
    (all_anchors[:, 2] < im_info[1] + _allowed_border) &  # width
    (all_anchors[:, 3] < im_info[0] + _allowed_border)  # height
)[0]

# keep only inside anchors
anchors = all_anchors[inds_inside, :]

1
2
3
4
5
6
7
8
# overlaps between the anchors and the gt boxes
# overlaps (ex, gt)
overlaps = bbox_overlaps(np.ascontiguousarray(anchors, dtype=np.float),
        			     np.ascontiguousarray(gt_boxes, dtype=np.float))
argmax_overlaps = overlaps.argmax(axis=1)
max_overlaps = overlaps[np.arange(len(inds_inside)), argmax_overlaps]   # N
gt_argmax_overlaps = overlaps.argmax(axis=0)
gt_max_overlaps = overlaps[gt_argmax_overlaps, np.arange(overlaps.shape[1])]  # K

1
2
labels[gt_argmax_overlaps] = 1
labels[max_overlaps >= cfg.TRAIN.RPN_POSITIVE_OVERLAP] = 1

1
2
3
if cfg.TRAIN.RPN_CLOBBER_POSITIVES:
    # assign bg labels last so that negative labels can clobber positives
    labels[max_overlaps < cfg.TRAIN.RPN_NEGATIVE_OVERLAP] = 0

1
2
3
4
5
6
7
8
9
num_fg = int(cfg.TRAIN.RPN_FG_FRACTION * cfg.TRAIN.RPN_BATCHSIZE)
fg_inds = np.where(labels == 1)[0]
if len(fg_inds) > num_fg:
    disable_inds = npr.choice(
        fg_inds, size=(len(fg_inds) - num_fg), replace=False)
    labels[disable_inds] = -1

num_bg = cfg.TRAIN.RPN_BATCHSIZE - np.sum(labels == 1)
......

1
bbox_targets = _compute_targets(anchors, gt_boxes[argmax_overlaps, :])

1
2
3
4
5
6
7
8
9
10
11
12
bbox_outside_weights = np.zeros((len(inds_inside), 4), dtype=np.float32)
if cfg.TRAIN.RPN_POSITIVE_WEIGHT < 0:
    # uniform weighting of examples (given non-uniform sampling)
    num_examples = np.sum(labels >= 0)
    positive_weights = np.ones((1, 4)) * 1.0 / num_examples
    negative_weights = np.ones((1, 4)) * 1.0 / num_examples
else:
    assert ((cfg.TRAIN.RPN_POSITIVE_WEIGHT > 0) & (cfg.TRAIN.RPN_POSITIVE_WEIGHT < 1))
    positive_weights = (cfg.TRAIN.RPN_POSITIVE_WEIGHT / np.sum(labels == 1))
    negative_weights = ((1.0 - cfg.TRAIN.RPN_POSITIVE_WEIGHT) / np.sum(labels == 0))
bbox_outside_weights[labels == 1, :] = positive_weights
bbox_outside_weights[labels == 0, :] = negative_weights

1
2
3
4
5
6
7
8
9
num_images = 1
rois_per_image = cfg.TRAIN.BATCH_SIZE / num_images
fg_rois_per_image = np.round(cfg.TRAIN.FG_FRACTION * rois_per_image)

# Sample rois with classification labels and bounding box regression
# targets
labels, rois, roi_scores, bbox_targets, bbox_inside_weights = _sample_rois(
    all_rois, all_scores, gt_boxes, fg_rois_per_image,
    rois_per_image, _num_classes)

1
2
3
4
5
6
7
# overlaps: (rois x gt_boxes)
overlaps = bbox_overlaps(
    np.ascontiguousarray(all_rois[:, 1:5], dtype=np.float),
    np.ascontiguousarray(gt_boxes[:, :4], dtype=np.float))
gt_assignment = overlaps.argmax(axis=1)
max_overlaps = overlaps.max(axis=1)
labels = gt_boxes[gt_assignment, 4]

1
2
3
fg_inds = np.where(max_overlaps >= cfg.TRAIN.FG_THRESH)[0]
bg_inds = np.where((max_overlaps < cfg.TRAIN.BG_THRESH_HI) &
                   (max_overlaps >= cfg.TRAIN.BG_THRESH_LO))[0]

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
if fg_inds.size > 0 and bg_inds.size > 0:
    fg_rois_per_image = min(fg_rois_per_image, fg_inds.size)    
    fg_inds = npr.choice(fg_inds, size=int(
        fg_rois_per_image), replace=False)
    bg_rois_per_image = rois_per_image - fg_rois_per_image
    to_replace = bg_inds.size < bg_rois_per_image
    bg_inds = npr.choice(bg_inds, size=int(
        bg_rois_per_image), replace=to_replace)
elif fg_inds.size > 0:
    to_replace = fg_inds.size < rois_per_image
    fg_inds = npr.choice(fg_inds, size=int(
        rois_per_image), replace=to_replace)
    fg_rois_per_image = rois_per_image
elif bg_inds.size > 0:
    to_replace = bg_inds.size < rois_per_image
    bg_inds = npr.choice(bg_inds, size=int(
        rois_per_image), replace=to_replace)
    fg_rois_per_image = 0
else:
    import pdb
    pdb.set_trace()

1
2
3
4
5
6
7
8
# The indices that we're selecting (both fg and bg)
keep_inds = np.append(fg_inds, bg_inds)
# Select sampled values from various arrays:
labels = labels[keep_inds]
# Clamp labels for the background RoIs to 0
labels[int(fg_rois_per_image):] = 0
rois = all_rois[keep_inds]
roi_scores = all_scores[keep_inds]

1
2
bbox_target_data = _compute_targets(rois[:, 1:5],
         							gt_boxes[gt_assignment[keep_inds], :4], labels)

1
bbox_targets, bbox_inside_weights = _get_bbox_regression_labels(bbox_target_data, num_classes)

1
2
3
4
5
6
height = (tf.to_float(bottom_shape[1]) - 1.) * np.float32(self._feat_stride[0])
width = (tf.to_float(bottom_shape[2]) - 1.) * np.float32(self._feat_stride[0])
x1 = tf.slice(rois, [0, 1], [-1, 1], name="x1") / width
y1 = tf.slice(rois, [0, 2], [-1, 1], name="y1") / height
x2 = tf.slice(rois, [0, 3], [-1, 1], name="x2") / width
y2 = tf.slice(rois, [0, 4], [-1, 1], name="y2") / height

1
bboxes = tf.stop_gradient(tf.concat([y1, x1, y2, x2], axis=1))

1
2
3
4
5
6
7
pre_pool_size = cfg.POOLING_SIZE * 2
crops = tf.image.crop_and_resize(bottom, 
                                 bboxes, 
                                 tf.to_int32(batch_ids), 
                                 [pre_pool_size, pre_pool_size], 
                                 name="crops")
slim.max_pool2d(crops, [2, 2], padding='SAME')

1
2
3
4
5
6
7
8
with tf.variable_scope(self._scope, self._scope, reuse=reuse):
  pool5_flat = slim.flatten(pool5, scope='flatten')
  fc6 = slim.fully_connected(pool5_flat, 4096, scope='fc6')
  if is_training:
      fc6 = slim.dropout(fc6, keep_prob=0.5, is_training=True, scope='dropout6')
  fc7 = slim.fully_connected(fc6, 4096, scope='fc7')
  if is_training:
      fc7 = slim.dropout(fc7, keep_prob=0.5, is_training=True, scope='dropout7')

1
2
3
4
5
6
7
8
9
10
cls_score = slim.fully_connected(fc7, self._num_classes,
                                 weights_initializer=initializer,
                                 trainable=is_training,
                                 activation_fn=None, scope='cls_score')
cls_prob = self._softmax_layer(cls_score, "cls_prob")
cls_pred = tf.argmax(cls_score, axis=1, name="cls_pred")
bbox_pred = slim.fully_connected(fc7, self._num_classes * 4,
                                 weights_initializer=initializer_bbox,
                                 trainable=is_training,
                                 activation_fn=None, scope='bbox_pred')

1
2
3
4
5
6
7
8
rpn_cls_score = tf.reshape(self._predictions['rpn_cls_score_reshape'], [-1, 2])
rpn_label = tf.reshape(self._anchor_targets['rpn_labels'], [-1])
rpn_select = tf.where(tf.not_equal(rpn_label, -1))
rpn_cls_score = tf.reshape(tf.gather(rpn_cls_score, rpn_select), [-1, 2])
rpn_label = tf.reshape(tf.gather(rpn_label, rpn_select), [-1])
rpn_cross_entropy = tf.reduce_mean(
    tf.nn.sparse_softmax_cross_entropy_with_logits(logits=rpn_cls_score,
                                                   labels=rpn_label))

1
2
3
4
5
6
7
8
9
10
rpn_bbox_pred = self._predictions['rpn_bbox_pred']
rpn_bbox_targets = self._anchor_targets['rpn_bbox_targets']
rpn_bbox_inside_weights = self._anchor_targets['rpn_bbox_inside_weights']
rpn_bbox_outside_weights = self._anchor_targets['rpn_bbox_outside_weights']
rpn_loss_box = self._smooth_l1_loss(rpn_bbox_pred, 
                                    rpn_bbox_targets,
                                    rpn_bbox_inside_weights,
                                    rpn_bbox_outside_weights, 
                                    sigma=sigma_rpn, 
                                    dim=[1, 2, 3])

1
2
3
4
5
cls_score = self._predictions["cls_score"]
label = tf.reshape(self._proposal_targets["labels"], [-1])
cross_entropy = tf.reduce_mean(
    tf.nn.sparse_softmax_cross_entropy_with_logits(logits=cls_score, 
                                                   labels=label))

1
2
3
4
5
6
bbox_pred = self._predictions['bbox_pred']
bbox_targets = self._proposal_targets['bbox_targets']
bbox_inside_weights = self._proposal_targets['bbox_inside_weights']
bbox_outside_weights = self._proposal_targets['bbox_outside_weights']
loss_box = self._smooth_l1_loss(
    bbox_pred, bbox_targets, bbox_inside_weights, bbox_outside_weights)

1
regularization_loss = tf.add_n(tf.losses.get_regularization_losses(), 'regu')