目标分割：Mask RCNN 气球分割案例 part2

桃扇骨 2023-07-21 12:29 58阅读 0赞

# ![20191009191333910.png][] #

# [日萌社][Link 1] #

[人工智能AI：Keras PyTorch MXNet TensorFlow PaddlePaddle 深度学习实战（不定时更新）][AI_Keras PyTorch MXNet TensorFlow PaddlePaddle]

--------------------

[目标分割：Mask RCNN][Mask RCNN]

[目标分割：Mask RCNN 气球分割案例 part1][Mask RCNN _ part1]

[目标分割：Mask RCNN 气球分割案例 part2][Mask RCNN _ part2]

--------------------

### **parallel\_model.py** ###

"""
    Mask R-CNN
    Multi-GPU Support for Keras.
    
    Copyright (c) 2017 Matterport, Inc.
    Licensed under the MIT License (see LICENSE for details)
    Written by Waleed Abdulla
    
    Ideas and a small code snippets from these sources:
    https://github.com/fchollet/keras/issues/2436
    https://medium.com/@kuza55/transparent-multi-gpu-training-on-tensorflow-with-keras-8b0016fd9012
    https://github.com/avolkov1/keras_experiments/blob/master/keras_exp/multigpu/
    https://github.com/fchollet/keras/blob/master/keras/utils/training_utils.py
    """
    
    import tensorflow as tf
    import keras.backend as K
    import keras.layers as KL
    import keras.models as KM
    
    
    class ParallelModel(KM.Model):
        """Subclasses the standard Keras Model and adds multi-GPU support.
        It works by creating a copy of the model on each GPU. Then it slices
        the inputs and sends a slice to each copy of the model, and then
        merges the outputs together and applies the loss on the combined
        outputs.
        """
    
        def __init__(self, keras_model, gpu_count):
            """Class constructor.
            keras_model: The Keras model to parallelize
            gpu_count: Number of GPUs. Must be > 1
            """
            self.inner_model = keras_model
            self.gpu_count = gpu_count
            merged_outputs = self.make_parallel()
            super(ParallelModel, self).__init__(inputs=self.inner_model.inputs,
                                                outputs=merged_outputs)
    
        def __getattribute__(self, attrname):
            """Redirect loading and saving methods to the inner model. That's where
            the weights are stored."""
            if 'load' in attrname or 'save' in attrname:
                return getattr(self.inner_model, attrname)
            return super(ParallelModel, self).__getattribute__(attrname)
    
        def summary(self, *args, **kwargs):
            """Override summary() to display summaries of both, the wrapper
            and inner models."""
            super(ParallelModel, self).summary(*args, **kwargs)
            self.inner_model.summary(*args, **kwargs)
    
        def make_parallel(self):
            """Creates a new wrapper model that consists of multiple replicas of
            the original model placed on different GPUs.
            """
            # Slice inputs. Slice inputs on the CPU to avoid sending a copy
            # of the full inputs to all GPUs. Saves on bandwidth and memory.
            input_slices = {name: tf.split(x, self.gpu_count)
                            for name, x in zip(self.inner_model.input_names,
                                               self.inner_model.inputs)}
    
            output_names = self.inner_model.output_names
            outputs_all = []
            for i in range(len(self.inner_model.outputs)):
                outputs_all.append([])
    
            # Run the model call() on each GPU to place the ops there
            for i in range(self.gpu_count):
                with tf.device('/gpu:%d' % i):
                    with tf.compat.v1.name_scope('tower_%d' % i):
                        # Run a slice of inputs through this replica
                        zipped_inputs = zip(self.inner_model.input_names,
                                            self.inner_model.inputs)
                        inputs = [
                            KL.Lambda(lambda s: input_slices[name][i],
                                      output_shape=lambda s: (None,) + s[1:])(tensor)
                            for name, tensor in zipped_inputs]
                        # Create the model replica and get the outputs
                        outputs = self.inner_model(inputs)
                        if not isinstance(outputs, list):
                            outputs = [outputs]
                        # Save the outputs for merging back together later
                        for l, o in enumerate(outputs):
                            outputs_all[l].append(o)
    
            # Merge outputs on CPU
            with tf.device('/cpu:0'):
                merged = []
                for outputs, name in zip(outputs_all, output_names):
                    # Concatenate or average outputs?
                    # Outputs usually have a batch dimension and we concatenate
                    # across it. If they don't, then the output is likely a loss
                    # or a metric value that gets averaged across the batch.
                    # Keras expects losses and metrics to be scalars.
                    if K.int_shape(outputs[0]) == ():
                        # Average
                        m = KL.Lambda(lambda o: tf.add_n(o) / len(outputs), name=name)(outputs)
                    else:
                        # Concatenate
                        m = KL.Concatenate(axis=0, name=name)(outputs)
                    merged.append(m)
            return merged
    
    
    if __name__ == "__main__":
        # Testing code below. It creates a simple model to train on MNIST and
        # tries to run it on 2 GPUs. It saves the graph so it can be viewed
        # in TensorBoard. Run it as:
        #
        # python3 parallel_model.py
    
        import os
        import numpy as np
        import keras.optimizers
        from keras.datasets import mnist
        from keras.preprocessing.image import ImageDataGenerator
    
        GPU_COUNT = 2
    
        # Root directory of the project
        ROOT_DIR = os.path.abspath("../")
    
        # Directory to save logs and trained model
        MODEL_DIR = os.path.join(ROOT_DIR, "logs")
    
        def build_model(x_train, num_classes):
            # Reset default graph. Keras leaves old ops in the graph,
            # which are ignored for execution but clutter graph
            # visualization in TensorBoard.
            tf.compat.v1.reset_default_graph()
    
            inputs = KL.Input(shape=x_train.shape[1:], name="input_image")
            x = KL.Conv2D(32, (3, 3), activation='relu', padding="same",
                          name="conv1")(inputs)
            x = KL.Conv2D(64, (3, 3), activation='relu', padding="same",
                          name="conv2")(x)
            x = KL.MaxPooling2D(pool_size=(2, 2), name="pool1")(x)
            x = KL.Flatten(name="flat1")(x)
            x = KL.Dense(128, activation='relu', name="dense1")(x)
            x = KL.Dense(num_classes, activation='softmax', name="dense2")(x)
    
            return KM.Model(inputs, x, "digit_classifier_model")
    
        # Load MNIST Data
        (x_train, y_train), (x_test, y_test) = mnist.load_data()
        x_train = np.expand_dims(x_train, -1).astype('float32') / 255
        x_test = np.expand_dims(x_test, -1).astype('float32') / 255
    
        print('x_train shape:', x_train.shape)
        print('x_test shape:', x_test.shape)
    
        # Build data generator and model
        datagen = ImageDataGenerator()
        model = build_model(x_train, 10)
    
        # Add multi-GPU support.
        model = ParallelModel(model, GPU_COUNT)
    
        optimizer = keras.optimizers.SGD(lr=0.01, momentum=0.9, clipnorm=5.0)
    
        model.compile(loss='sparse_categorical_crossentropy',
                      optimizer=optimizer, metrics=['accuracy'])
    
        model.summary()
    
        # Train
        model.fit_generator(
            datagen.flow(x_train, y_train, batch_size=64),
            steps_per_epoch=50, epochs=10, verbose=1,
            validation_data=(x_test, y_test),
            callbacks=[keras.callbacks.TensorBoard(log_dir=MODEL_DIR,
                                                   write_graph=True)]
        )

--------------------

### **utils.py** ###

"""
    Mask R-CNN
    Common utility functions and classes.
    
    Copyright (c) 2017 Matterport, Inc.
    Licensed under the MIT License (see LICENSE for details)
    Written by Waleed Abdulla
    """
    
    import sys
    import os
    import logging
    import math
    import random
    import numpy as np
    import tensorflow as tf
    import scipy
    import skimage.color
    import skimage.io
    import skimage.transform
    import urllib.request
    import shutil
    import warnings
    from distutils.version import LooseVersion
    
    # URL from which to download the latest COCO trained weights
    COCO_MODEL_URL = "https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5"
    
    
    ############################################################
    #  Bounding Boxes
    ############################################################
    
    def extract_bboxes(mask):
        """Compute bounding boxes from masks.
        mask: [height, width, num_instances]. Mask pixels are either 1 or 0.
    
        Returns: bbox array [num_instances, (y1, x1, y2, x2)].
        """
        boxes = np.zeros([mask.shape[-1], 4], dtype=np.int32)
        for i in range(mask.shape[-1]):
            m = mask[:, :, i]
            # Bounding box.
            horizontal_indicies = np.where(np.any(m, axis=0))[0]
            vertical_indicies = np.where(np.any(m, axis=1))[0]
            if horizontal_indicies.shape[0]:
                x1, x2 = horizontal_indicies[[0, -1]]
                y1, y2 = vertical_indicies[[0, -1]]
                # x2 and y2 should not be part of the box. Increment by 1.
                x2 += 1
                y2 += 1
            else:
                # No mask for this instance. Might happen due to
                # resizing or cropping. Set bbox to zeros
                x1, x2, y1, y2 = 0, 0, 0, 0
            boxes[i] = np.array([y1, x1, y2, x2])
        return boxes.astype(np.int32)
    
    
    def compute_iou(box, boxes, box_area, boxes_area):
        """Calculates IoU of the given box with the array of the given boxes.
        box: 1D vector [y1, x1, y2, x2]
        boxes: [boxes_count, (y1, x1, y2, x2)]
        box_area: float. the area of 'box'
        boxes_area: array of length boxes_count.
    
        Note: the areas are passed in rather than calculated here for
        efficiency. Calculate once in the caller to avoid duplicate work.
        """
        # Calculate intersection areas
        y1 = np.maximum(box[0], boxes[:, 0])
        y2 = np.minimum(box[2], boxes[:, 2])
        x1 = np.maximum(box[1], boxes[:, 1])
        x2 = np.minimum(box[3], boxes[:, 3])
        intersection = np.maximum(x2 - x1, 0) * np.maximum(y2 - y1, 0)
        union = box_area + boxes_area[:] - intersection[:]
        iou = intersection / union
        return iou
    
    
    def compute_overlaps(boxes1, boxes2):
        """Computes IoU overlaps between two sets of boxes.
        boxes1, boxes2: [N, (y1, x1, y2, x2)].
    
        For better performance, pass the largest set first and the smaller second.
        """
        # Areas of anchors and GT boxes
        area1 = (boxes1[:, 2] - boxes1[:, 0]) * (boxes1[:, 3] - boxes1[:, 1])
        area2 = (boxes2[:, 2] - boxes2[:, 0]) * (boxes2[:, 3] - boxes2[:, 1])
    
        # Compute overlaps to generate matrix [boxes1 count, boxes2 count]
        # Each cell contains the IoU value.
        overlaps = np.zeros((boxes1.shape[0], boxes2.shape[0]))
        for i in range(overlaps.shape[1]):
            box2 = boxes2[i]
            overlaps[:, i] = compute_iou(box2, boxes1, area2[i], area1)
        return overlaps
    
    
    def compute_overlaps_masks(masks1, masks2):
        """Computes IoU overlaps between two sets of masks.
        masks1, masks2: [Height, Width, instances]
        """
        
        # If either set of masks is empty return empty result
        if masks1.shape[-1] == 0 or masks2.shape[-1] == 0:
            return np.zeros((masks1.shape[-1], masks2.shape[-1]))
        # flatten masks and compute their areas
        masks1 = np.reshape(masks1 > .5, (-1, masks1.shape[-1])).astype(np.float32)
        masks2 = np.reshape(masks2 > .5, (-1, masks2.shape[-1])).astype(np.float32)
        area1 = np.sum(masks1, axis=0)
        area2 = np.sum(masks2, axis=0)
    
        # intersections and union
        intersections = np.dot(masks1.T, masks2)
        union = area1[:, None] + area2[None, :] - intersections
        overlaps = intersections / union
    
        return overlaps
    
    
    def non_max_suppression(boxes, scores, threshold):
        """Performs non-maximum suppression and returns indices of kept boxes.
        boxes: [N, (y1, x1, y2, x2)]. Notice that (y2, x2) lays outside the box.
        scores: 1-D array of box scores.
        threshold: Float. IoU threshold to use for filtering.
        """
        assert boxes.shape[0] > 0
        if boxes.dtype.kind != "f":
            boxes = boxes.astype(np.float32)
    
        # Compute box areas
        y1 = boxes[:, 0]
        x1 = boxes[:, 1]
        y2 = boxes[:, 2]
        x2 = boxes[:, 3]
        area = (y2 - y1) * (x2 - x1)
    
        # Get indicies of boxes sorted by scores (highest first)
        ixs = scores.argsort()[::-1]
    
        pick = []
        while len(ixs) > 0:
            # Pick top box and add its index to the list
            i = ixs[0]
            pick.append(i)
            # Compute IoU of the picked box with the rest
            iou = compute_iou(boxes[i], boxes[ixs[1:]], area[i], area[ixs[1:]])
            # Identify boxes with IoU over the threshold. This
            # returns indices into ixs[1:], so add 1 to get
            # indices into ixs.
            remove_ixs = np.where(iou > threshold)[0] + 1
            # Remove indices of the picked and overlapped boxes.
            ixs = np.delete(ixs, remove_ixs)
            ixs = np.delete(ixs, 0)
        return np.array(pick, dtype=np.int32)
    
    
    def apply_box_deltas(boxes, deltas):
        """Applies the given deltas to the given boxes.
        boxes: [N, (y1, x1, y2, x2)]. Note that (y2, x2) is outside the box.
        deltas: [N, (dy, dx, log(dh), log(dw))]
        """
        boxes = boxes.astype(np.float32)
        # Convert to y, x, h, w
        height = boxes[:, 2] - boxes[:, 0]
        width = boxes[:, 3] - boxes[:, 1]
        center_y = boxes[:, 0] + 0.5 * height
        center_x = boxes[:, 1] + 0.5 * width
        # Apply deltas
        center_y += deltas[:, 0] * height
        center_x += deltas[:, 1] * width
        height *= np.exp(deltas[:, 2])
        width *= np.exp(deltas[:, 3])
        # Convert back to y1, x1, y2, x2
        y1 = center_y - 0.5 * height
        x1 = center_x - 0.5 * width
        y2 = y1 + height
        x2 = x1 + width
        return np.stack([y1, x1, y2, x2], axis=1)
    
    
    def box_refinement_graph(box, gt_box):
        """Compute refinement needed to transform box to gt_box.
        box and gt_box are [N, (y1, x1, y2, x2)]
        """
        box = tf.cast(box, tf.float32)
        gt_box = tf.cast(gt_box, tf.float32)
    
        height = box[:, 2] - box[:, 0]
        width = box[:, 3] - box[:, 1]
        center_y = box[:, 0] + 0.5 * height
        center_x = box[:, 1] + 0.5 * width
    
        gt_height = gt_box[:, 2] - gt_box[:, 0]
        gt_width = gt_box[:, 3] - gt_box[:, 1]
        gt_center_y = gt_box[:, 0] + 0.5 * gt_height
        gt_center_x = gt_box[:, 1] + 0.5 * gt_width
    
        dy = (gt_center_y - center_y) / height
        dx = (gt_center_x - center_x) / width
        dh = tf.math.log(gt_height / height)
        dw = tf.math.log(gt_width / width)
    
        result = tf.stack([dy, dx, dh, dw], axis=1)
        return result
    
    
    def box_refinement(box, gt_box):
        """Compute refinement needed to transform box to gt_box.
        box and gt_box are [N, (y1, x1, y2, x2)]. (y2, x2) is
        assumed to be outside the box.
        """
        box = box.astype(np.float32)
        gt_box = gt_box.astype(np.float32)
    
        height = box[:, 2] - box[:, 0]
        width = box[:, 3] - box[:, 1]
        center_y = box[:, 0] + 0.5 * height
        center_x = box[:, 1] + 0.5 * width
    
        gt_height = gt_box[:, 2] - gt_box[:, 0]
        gt_width = gt_box[:, 3] - gt_box[:, 1]
        gt_center_y = gt_box[:, 0] + 0.5 * gt_height
        gt_center_x = gt_box[:, 1] + 0.5 * gt_width
    
        dy = (gt_center_y - center_y) / height
        dx = (gt_center_x - center_x) / width
        dh = np.log(gt_height / height)
        dw = np.log(gt_width / width)
    
        return np.stack([dy, dx, dh, dw], axis=1)
    
    
    ############################################################
    #  Dataset
    ############################################################
    
    class Dataset(object):
        """The base class for dataset classes.
        To use it, create a new class that adds functions specific to the dataset
        you want to use. For example:
    
        class CatsAndDogsDataset(Dataset):
            def load_cats_and_dogs(self):
                ...
            def load_mask(self, image_id):
                ...
            def image_reference(self, image_id):
                ...
    
        See COCODataset and ShapesDataset as examples.
        """
    
        def __init__(self, class_map=None):
            self._image_ids = []
            self.image_info = []
            # Background is always the first class
            self.class_info = [{"source": "", "id": 0, "name": "BG"}]
            self.source_class_ids = {}
    
        def add_class(self, source, class_id, class_name):
            assert "." not in source, "Source name cannot contain a dot"
            # Does the class exist already?
            for info in self.class_info:
                if info['source'] == source and info["id"] == class_id:
                    # source.class_id combination already available, skip
                    return
            # Add the class
            self.class_info.append({
                "source": source,
                "id": class_id,
                "name": class_name,
            })
    
        def add_image(self, source, image_id, path, **kwargs):
            image_info = {
                "id": image_id,
                "source": source,
                "path": path,
            }
            image_info.update(kwargs)
            self.image_info.append(image_info)
    
        def image_reference(self, image_id):
            """Return a link to the image in its source Website or details about
            the image that help looking it up or debugging it.
    
            Override for your dataset, but pass to this function
            if you encounter images not in your dataset.
            """
            return ""
    
        def prepare(self, class_map=None):
            """Prepares the Dataset class for use.
    
            TODO: class map is not supported yet. When done, it should handle mapping
                  classes from different datasets to the same class ID.
            """
    
            def clean_name(name):
                """Returns a shorter version of object names for cleaner display."""
                return ",".join(name.split(",")[:1])
    
            # Build (or rebuild) everything else from the info dicts.
            self.num_classes = len(self.class_info)
            self.class_ids = np.arange(self.num_classes)
            self.class_names = [clean_name(c["name"]) for c in self.class_info]
            self.num_images = len(self.image_info)
            self._image_ids = np.arange(self.num_images)
    
            # Mapping from source class and image IDs to internal IDs
            self.class_from_source_map = {"{}.{}".format(info['source'], info['id']): id
                                          for info, id in zip(self.class_info, self.class_ids)}
            self.image_from_source_map = {"{}.{}".format(info['source'], info['id']): id
                                          for info, id in zip(self.image_info, self.image_ids)}
    
            # Map sources to class_ids they support
            self.sources = list(set([i['source'] for i in self.class_info]))
            self.source_class_ids = {}
            # Loop over datasets
            for source in self.sources:
                self.source_class_ids[source] = []
                # Find classes that belong to this dataset
                for i, info in enumerate(self.class_info):
                    # Include BG class in all datasets
                    if i == 0 or source == info['source']:
                        self.source_class_ids[source].append(i)
    
        def map_source_class_id(self, source_class_id):
            """Takes a source class ID and returns the int class ID assigned to it.
    
            For example:
            dataset.map_source_class_id("coco.12") -> 23
            """
            return self.class_from_source_map[source_class_id]
    
        def get_source_class_id(self, class_id, source):
            """Map an internal class ID to the corresponding class ID in the source dataset."""
            info = self.class_info[class_id]
            assert info['source'] == source
            return info['id']
    
        @property
        def image_ids(self):
            return self._image_ids
    
        def source_image_link(self, image_id):
            """Returns the path or URL to the image.
            Override this to return a URL to the image if it's available online for easy
            debugging.
            """
            return self.image_info[image_id]["path"]
    
        def load_image(self, image_id):
            """Load the specified image and return a [H,W,3] Numpy array.
            """
            # Load image
            image = skimage.io.imread(self.image_info[image_id]['path'])
            # If grayscale. Convert to RGB for consistency.
            if image.ndim != 3:
                image = skimage.color.gray2rgb(image)
            # If has an alpha channel, remove it for consistency
            if image.shape[-1] == 4:
                image = image[..., :3]
            return image
    
        def load_mask(self, image_id):
            """Load instance masks for the given image.
    
            Different datasets use different ways to store masks. Override this
            method to load instance masks and return them in the form of am
            array of binary masks of shape [height, width, instances].
    
            Returns:
                masks: A bool array of shape [height, width, instance count] with
                    a binary mask per instance.
                class_ids: a 1D array of class IDs of the instance masks.
            """
            # Override this function to load a mask from your dataset.
            # Otherwise, it returns an empty mask.
            logging.warning("You are using the default load_mask(), maybe you need to define your own one.")
            mask = np.empty([0, 0, 0])
            class_ids = np.empty([0], np.int32)
            return mask, class_ids
    
    
    def resize_image(image, min_dim=None, max_dim=None, min_scale=None, mode="square"):
        """Resizes an image keeping the aspect ratio unchanged.
    
        min_dim: if provided, resizes the image such that it's smaller
            dimension == min_dim
        max_dim: if provided, ensures that the image longest side doesn't
            exceed this value.
        min_scale: if provided, ensure that the image is scaled up by at least
            this percent even if min_dim doesn't require it.
        mode: Resizing mode.
            none: No resizing. Return the image unchanged.
            square: Resize and pad with zeros to get a square image
                of size [max_dim, max_dim].
            pad64: Pads width and height with zeros to make them multiples of 64.
                   If min_dim or min_scale are provided, it scales the image up
                   before padding. max_dim is ignored in this mode.
                   The multiple of 64 is needed to ensure smooth scaling of feature
                   maps up and down the 6 levels of the FPN pyramid (2**6=64).
            crop: Picks random crops from the image. First, scales the image based
                  on min_dim and min_scale, then picks a random crop of
                  size min_dim x min_dim. Can be used in training only.
                  max_dim is not used in this mode.
    
        Returns:
        image: the resized image
        window: (y1, x1, y2, x2). If max_dim is provided, padding might
            be inserted in the returned image. If so, this window is the
            coordinates of the image part of the full image (excluding
            the padding). The x2, y2 pixels are not included.
        scale: The scale factor used to resize the image
        padding: Padding added to the image [(top, bottom), (left, right), (0, 0)]
        """
        # Keep track of image dtype and return results in the same dtype
        image_dtype = image.dtype
        # Default window (y1, x1, y2, x2) and default scale == 1.
        h, w = image.shape[:2]
        window = (0, 0, h, w)
        scale = 1
        padding = [(0, 0), (0, 0), (0, 0)]
        crop = None
    
        if mode == "none":
            return image, window, scale, padding, crop
    
        # Scale?
        if min_dim:
            # Scale up but not down
            scale = max(1, min_dim / min(h, w))
        if min_scale and scale < min_scale:
            scale = min_scale
    
        # Does it exceed max dim?
        if max_dim and mode == "square":
            image_max = max(h, w)
            if round(image_max * scale) > max_dim:
                scale = max_dim / image_max
    
        # Resize image using bilinear interpolation
        if scale != 1:
            image = resize(image, (round(h * scale), round(w * scale)),
                           preserve_range=True)
    
        # Need padding or cropping?
        if mode == "square":
            # Get new height and width
            h, w = image.shape[:2]
            top_pad = (max_dim - h) // 2
            bottom_pad = max_dim - h - top_pad
            left_pad = (max_dim - w) // 2
            right_pad = max_dim - w - left_pad
            padding = [(top_pad, bottom_pad), (left_pad, right_pad), (0, 0)]
            image = np.pad(image, padding, mode='constant', constant_values=0)
            window = (top_pad, left_pad, h + top_pad, w + left_pad)
        elif mode == "pad64":
            h, w = image.shape[:2]
            # Both sides must be divisible by 64
            assert min_dim % 64 == 0, "Minimum dimension must be a multiple of 64"
            # Height
            if h % 64 > 0:
                max_h = h - (h % 64) + 64
                top_pad = (max_h - h) // 2
                bottom_pad = max_h - h - top_pad
            else:
                top_pad = bottom_pad = 0
            # Width
            if w % 64 > 0:
                max_w = w - (w % 64) + 64
                left_pad = (max_w - w) // 2
                right_pad = max_w - w - left_pad
            else:
                left_pad = right_pad = 0
            padding = [(top_pad, bottom_pad), (left_pad, right_pad), (0, 0)]
            image = np.pad(image, padding, mode='constant', constant_values=0)
            window = (top_pad, left_pad, h + top_pad, w + left_pad)
        elif mode == "crop":
            # Pick a random crop
            h, w = image.shape[:2]
            y = random.randint(0, (h - min_dim))
            x = random.randint(0, (w - min_dim))
            crop = (y, x, min_dim, min_dim)
            image = image[y:y + min_dim, x:x + min_dim]
            window = (0, 0, min_dim, min_dim)
        else:
            raise Exception("Mode {} not supported".format(mode))
        return image.astype(image_dtype), window, scale, padding, crop
    
    
    def resize_mask(mask, scale, padding, crop=None):
        """Resizes a mask using the given scale and padding.
        Typically, you get the scale and padding from resize_image() to
        ensure both, the image and the mask, are resized consistently.
    
        scale: mask scaling factor
        padding: Padding to add to the mask in the form
                [(top, bottom), (left, right), (0, 0)]
        """
        # Suppress warning from scipy 0.13.0, the output shape of zoom() is
        # calculated with round() instead of int()
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            mask = scipy.ndimage.zoom(mask, zoom=[scale, scale, 1], order=0)
        if crop is not None:
            y, x, h, w = crop
            mask = mask[y:y + h, x:x + w]
        else:
            mask = np.pad(mask, padding, mode='constant', constant_values=0)
        return mask
    
    
    def minimize_mask(bbox, mask, mini_shape):
        """Resize masks to a smaller version to reduce memory load.
        Mini-masks can be resized back to image scale using expand_masks()
    
        See inspect_data.ipynb notebook for more details.
        """
        mini_mask = np.zeros(mini_shape + (mask.shape[-1],), dtype=bool)
        for i in range(mask.shape[-1]):
            # Pick slice and cast to bool in case load_mask() returned wrong dtype
            m = mask[:, :, i].astype(bool)
            y1, x1, y2, x2 = bbox[i][:4]
            m = m[y1:y2, x1:x2]
            if m.size == 0:
                raise Exception("Invalid bounding box with area of zero")
            # Resize with bilinear interpolation
            m = resize(m, mini_shape)
            mini_mask[:, :, i] = np.around(m).astype(np.bool)
        return mini_mask
    
    
    def expand_mask(bbox, mini_mask, image_shape):
        """Resizes mini masks back to image size. Reverses the change
        of minimize_mask().
    
        See inspect_data.ipynb notebook for more details.
        """
        mask = np.zeros(image_shape[:2] + (mini_mask.shape[-1],), dtype=bool)
        for i in range(mask.shape[-1]):
            m = mini_mask[:, :, i]
            y1, x1, y2, x2 = bbox[i][:4]
            h = y2 - y1
            w = x2 - x1
            # Resize with bilinear interpolation
            m = resize(m, (h, w))
            mask[y1:y2, x1:x2, i] = np.around(m).astype(np.bool)
        return mask
    
    
    # TODO: Build and use this function to reduce code duplication
    def mold_mask(mask, config):
        pass
    
    
    def unmold_mask(mask, bbox, image_shape):
        """Converts a mask generated by the neural network to a format similar
        to its original shape.
        mask: [height, width] of type float. A small, typically 28x28 mask.
        bbox: [y1, x1, y2, x2]. The box to fit the mask in.
    
        Returns a binary mask with the same size as the original image.
        """
        threshold = 0.5
        y1, x1, y2, x2 = bbox
        mask = resize(mask, (y2 - y1, x2 - x1))
        mask = np.where(mask >= threshold, 1, 0).astype(np.bool)
    
        # Put the mask in the right location.
        full_mask = np.zeros(image_shape[:2], dtype=np.bool)
        full_mask[y1:y2, x1:x2] = mask
        return full_mask
    
    
    ############################################################
    #  Anchors
    ############################################################
    
    def generate_anchors(scales, ratios, shape, feature_stride, anchor_stride):
        """
        scales: 1D array of anchor sizes in pixels. Example: [32, 64, 128]
        ratios: 1D array of anchor ratios of width/height. Example: [0.5, 1, 2]
        shape: [height, width] spatial shape of the feature map over which
                to generate anchors.
        feature_stride: Stride of the feature map relative to the image in pixels.
        anchor_stride: Stride of anchors on the feature map. For example, if the
            value is 2 then generate anchors for every other feature map pixel.
        """
        # Get all combinations of scales and ratios
        scales, ratios = np.meshgrid(np.array(scales), np.array(ratios))
        scales = scales.flatten()
        ratios = ratios.flatten()
    
        # Enumerate heights and widths from scales and ratios
        heights = scales / np.sqrt(ratios)
        widths = scales * np.sqrt(ratios)
    
        # Enumerate shifts in feature space
        shifts_y = np.arange(0, shape[0], anchor_stride) * feature_stride
        shifts_x = np.arange(0, shape[1], anchor_stride) * feature_stride
        shifts_x, shifts_y = np.meshgrid(shifts_x, shifts_y)
    
        # Enumerate combinations of shifts, widths, and heights
        box_widths, box_centers_x = np.meshgrid(widths, shifts_x)
        box_heights, box_centers_y = np.meshgrid(heights, shifts_y)
    
        # Reshape to get a list of (y, x) and a list of (h, w)
        box_centers = np.stack(
            [box_centers_y, box_centers_x], axis=2).reshape([-1, 2])
        box_sizes = np.stack([box_heights, box_widths], axis=2).reshape([-1, 2])
    
        # Convert to corner coordinates (y1, x1, y2, x2)
        boxes = np.concatenate([box_centers - 0.5 * box_sizes,
                                box_centers + 0.5 * box_sizes], axis=1)
        return boxes
    
    
    def generate_pyramid_anchors(scales, ratios, feature_shapes, feature_strides,
                                 anchor_stride):
        """Generate anchors at different levels of a feature pyramid. Each scale
        is associated with a level of the pyramid, but each ratio is used in
        all levels of the pyramid.
    
        Returns:
        anchors: [N, (y1, x1, y2, x2)]. All generated anchors in one array. Sorted
            with the same order of the given scales. So, anchors of scale[0] come
            first, then anchors of scale[1], and so on.
        """
        # Anchors
        # [anchor_count, (y1, x1, y2, x2)]
        anchors = []
        for i in range(len(scales)):
            anchors.append(generate_anchors(scales[i], ratios, feature_shapes[i],
                                            feature_strides[i], anchor_stride))
        return np.concatenate(anchors, axis=0)
    
    
    ############################################################
    #  Miscellaneous
    ############################################################
    
    def trim_zeros(x):
        """It's common to have tensors larger than the available data and
        pad with zeros. This function removes rows that are all zeros.
    
        x: [rows, columns].
        """
        assert len(x.shape) == 2
        return x[~np.all(x == 0, axis=1)]
    
    
    def compute_matches(gt_boxes, gt_class_ids, gt_masks,
                        pred_boxes, pred_class_ids, pred_scores, pred_masks,
                        iou_threshold=0.5, score_threshold=0.0):
        """Finds matches between prediction and ground truth instances.
    
        Returns:
            gt_match: 1-D array. For each GT box it has the index of the matched
                      predicted box.
            pred_match: 1-D array. For each predicted box, it has the index of
                        the matched ground truth box.
            overlaps: [pred_boxes, gt_boxes] IoU overlaps.
        """
        # Trim zero padding
        # TODO: cleaner to do zero unpadding upstream
        gt_boxes = trim_zeros(gt_boxes)
        gt_masks = gt_masks[..., :gt_boxes.shape[0]]
        pred_boxes = trim_zeros(pred_boxes)
        pred_scores = pred_scores[:pred_boxes.shape[0]]
        # Sort predictions by score from high to low
        indices = np.argsort(pred_scores)[::-1]
        pred_boxes = pred_boxes[indices]
        pred_class_ids = pred_class_ids[indices]
        pred_scores = pred_scores[indices]
        pred_masks = pred_masks[..., indices]
    
        # Compute IoU overlaps [pred_masks, gt_masks]
        overlaps = compute_overlaps_masks(pred_masks, gt_masks)
    
        # Loop through predictions and find matching ground truth boxes
        match_count = 0
        pred_match = -1 * np.ones([pred_boxes.shape[0]])
        gt_match = -1 * np.ones([gt_boxes.shape[0]])
        for i in range(len(pred_boxes)):
            # Find best matching ground truth box
            # 1. Sort matches by score
            sorted_ixs = np.argsort(overlaps[i])[::-1]
            # 2. Remove low scores
            low_score_idx = np.where(overlaps[i, sorted_ixs] < score_threshold)[0]
            if low_score_idx.size > 0:
                sorted_ixs = sorted_ixs[:low_score_idx[0]]
            # 3. Find the match
            for j in sorted_ixs:
                # If ground truth box is already matched, go to next one
                if gt_match[j] > -1:
                    continue
                # If we reach IoU smaller than the threshold, end the loop
                iou = overlaps[i, j]
                if iou < iou_threshold:
                    break
                # Do we have a match?
                if pred_class_ids[i] == gt_class_ids[j]:
                    match_count += 1
                    gt_match[j] = i
                    pred_match[i] = j
                    break
    
        return gt_match, pred_match, overlaps
    
    
    def compute_ap(gt_boxes, gt_class_ids, gt_masks,
                   pred_boxes, pred_class_ids, pred_scores, pred_masks,
                   iou_threshold=0.5):
        """Compute Average Precision at a set IoU threshold (default 0.5).
    
        Returns:
        mAP: Mean Average Precision
        precisions: List of precisions at different class score thresholds.
        recalls: List of recall values at different class score thresholds.
        overlaps: [pred_boxes, gt_boxes] IoU overlaps.
        """
        # Get matches and overlaps
        gt_match, pred_match, overlaps = compute_matches(
            gt_boxes, gt_class_ids, gt_masks,
            pred_boxes, pred_class_ids, pred_scores, pred_masks,
            iou_threshold)
    
        # Compute precision and recall at each prediction box step
        precisions = np.cumsum(pred_match > -1) / (np.arange(len(pred_match)) + 1)
        recalls = np.cumsum(pred_match > -1).astype(np.float32) / len(gt_match)
    
        # Pad with start and end values to simplify the math
        precisions = np.concatenate([[0], precisions, [0]])
        recalls = np.concatenate([[0], recalls, [1]])
    
        # Ensure precision values decrease but don't increase. This way, the
        # precision value at each recall threshold is the maximum it can be
        # for all following recall thresholds, as specified by the VOC paper.
        for i in range(len(precisions) - 2, -1, -1):
            precisions[i] = np.maximum(precisions[i], precisions[i + 1])
    
        # Compute mean AP over recall range
        indices = np.where(recalls[:-1] != recalls[1:])[0] + 1
        mAP = np.sum((recalls[indices] - recalls[indices - 1]) *
                     precisions[indices])
    
        return mAP, precisions, recalls, overlaps
    
    
    def compute_ap_range(gt_box, gt_class_id, gt_mask,
                         pred_box, pred_class_id, pred_score, pred_mask,
                         iou_thresholds=None, verbose=1):
        """Compute AP over a range or IoU thresholds. Default range is 0.5-0.95."""
        # Default is 0.5 to 0.95 with increments of 0.05
        iou_thresholds = iou_thresholds or np.arange(0.5, 1.0, 0.05)
        
        # Compute AP over range of IoU thresholds
        AP = []
        for iou_threshold in iou_thresholds:
            ap, precisions, recalls, overlaps =\
                compute_ap(gt_box, gt_class_id, gt_mask,
                            pred_box, pred_class_id, pred_score, pred_mask,
                            iou_threshold=iou_threshold)
            if verbose:
                print("AP @{:.2f}:\t {:.3f}".format(iou_threshold, ap))
            AP.append(ap)
        AP = np.array(AP).mean()
        if verbose:
            print("AP @{:.2f}-{:.2f}:\t {:.3f}".format(
                iou_thresholds[0], iou_thresholds[-1], AP))
        return AP
    
    
    def compute_recall(pred_boxes, gt_boxes, iou):
        """Compute the recall at the given IoU threshold. It's an indication
        of how many GT boxes were found by the given prediction boxes.
    
        pred_boxes: [N, (y1, x1, y2, x2)] in image coordinates
        gt_boxes: [N, (y1, x1, y2, x2)] in image coordinates
        """
        # Measure overlaps
        overlaps = compute_overlaps(pred_boxes, gt_boxes)
        iou_max = np.max(overlaps, axis=1)
        iou_argmax = np.argmax(overlaps, axis=1)
        positive_ids = np.where(iou_max >= iou)[0]
        matched_gt_boxes = iou_argmax[positive_ids]
    
        recall = len(set(matched_gt_boxes)) / gt_boxes.shape[0]
        return recall, positive_ids
    
    
    # ## Batch Slicing
    # Some custom layers support a batch size of 1 only, and require a lot of work
    # to support batches greater than 1. This function slices an input tensor
    # across the batch dimension and feeds batches of size 1. Effectively,
    # an easy way to support batches > 1 quickly with little code modification.
    # In the long run, it's more efficient to modify the code to support large
    # batches and getting rid of this function. Consider this a temporary solution
    def batch_slice(inputs, graph_fn, batch_size, names=None):
        """Splits inputs into slices and feeds each slice to a copy of the given
        computation graph and then combines the results. It allows you to run a
        graph on a batch of inputs even if the graph is written to support one
        instance only.
    
        inputs: list of tensors. All must have the same first dimension length
        graph_fn: A function that returns a TF tensor that's part of a graph.
        batch_size: number of slices to divide the data into.
        names: If provided, assigns names to the resulting tensors.
        """
        if not isinstance(inputs, list):
            inputs = [inputs]
    
        outputs = []
        for i in range(batch_size):
            inputs_slice = [x[i] for x in inputs]
            output_slice = graph_fn(*inputs_slice)
            if not isinstance(output_slice, (tuple, list)):
                output_slice = [output_slice]
            outputs.append(output_slice)
        # Change outputs from a list of slices where each is
        # a list of outputs to a list of outputs and each has
        # a list of slices
        outputs = list(zip(*outputs))
    
        if names is None:
            names = [None] * len(outputs)
    
        result = [tf.stack(o, axis=0, name=n)
                  for o, n in zip(outputs, names)]
        if len(result) == 1:
            result = result[0]
    
        return result
    
    
    def download_trained_weights(coco_model_path, verbose=1):
        """Download COCO trained weights from Releases.
    
        coco_model_path: local path of COCO trained weights
        """
        if verbose > 0:
            print("Downloading pretrained model to " + coco_model_path + " ...")
        with urllib.request.urlopen(COCO_MODEL_URL) as resp, open(coco_model_path, 'wb') as out:
            shutil.copyfileobj(resp, out)
        if verbose > 0:
            print("... done downloading pretrained model!")
    
    
    def norm_boxes(boxes, shape):
        """Converts boxes from pixel coordinates to normalized coordinates.
        boxes: [N, (y1, x1, y2, x2)] in pixel coordinates
        shape: [..., (height, width)] in pixels
    
        Note: In pixel coordinates (y2, x2) is outside the box. But in normalized
        coordinates it's inside the box.
    
        Returns:
            [N, (y1, x1, y2, x2)] in normalized coordinates
        """
        h, w = shape
        scale = np.array([h - 1, w - 1, h - 1, w - 1])
        shift = np.array([0, 0, 1, 1])
        return np.divide((boxes - shift), scale).astype(np.float32)
    
    
    def denorm_boxes(boxes, shape):
        """Converts boxes from normalized coordinates to pixel coordinates.
        boxes: [N, (y1, x1, y2, x2)] in normalized coordinates
        shape: [..., (height, width)] in pixels
    
        Note: In pixel coordinates (y2, x2) is outside the box. But in normalized
        coordinates it's inside the box.
    
        Returns:
            [N, (y1, x1, y2, x2)] in pixel coordinates
        """
        h, w = shape
        scale = np.array([h - 1, w - 1, h - 1, w - 1])
        shift = np.array([0, 0, 1, 1])
        return np.around(np.multiply(boxes, scale) + shift).astype(np.int32)
    
    
    def resize(image, output_shape, order=1, mode='constant', cval=0, clip=True,
               preserve_range=False, anti_aliasing=False, anti_aliasing_sigma=None):
        """A wrapper for Scikit-Image resize().
    
        Scikit-Image generates warnings on every call to resize() if it doesn't
        receive the right parameters. The right parameters depend on the version
        of skimage. This solves the problem by using different parameters per
        version. And it provides a central place to control resizing defaults.
        """
        if LooseVersion(skimage.__version__) >= LooseVersion("0.14"):
            # New in 0.14: anti_aliasing. Default it to False for backward
            # compatibility with skimage 0.13.
            return skimage.transform.resize(
                image, output_shape,
                order=order, mode=mode, cval=cval, clip=clip,
                preserve_range=preserve_range, anti_aliasing=anti_aliasing,
                anti_aliasing_sigma=anti_aliasing_sigma)
        else:
            return skimage.transform.resize(
                image, output_shape,
                order=order, mode=mode, cval=cval, clip=clip,
                preserve_range=preserve_range)

--------------------

### **visualize.py** ###

"""
    Mask R-CNN
    Display and Visualization Functions.
    
    Copyright (c) 2017 Matterport, Inc.
    Licensed under the MIT License (see LICENSE for details)
    Written by Waleed Abdulla
    """
    
    import os
    import sys
    import random
    import itertools
    import colorsys
    
    import numpy as np
    from skimage.measure import find_contours
    import matplotlib.pyplot as plt
    from matplotlib import patches,  lines
    from matplotlib.patches import Polygon
    import IPython.display
    
    # Root directory of the project
    ROOT_DIR = os.path.abspath("../")
    
    # Import Mask RCNN
    sys.path.append(ROOT_DIR)  # To find local version of the library
    from mrcnn import utils
    
    
    ############################################################
    #  Visualization
    ############################################################
    
    def display_images(images, titles=None, cols=4, cmap=None, norm=None,
                       interpolation=None):
        """Display the given set of images, optionally with titles.
        images: list or array of image tensors in HWC format.
        titles: optional. A list of titles to display with each image.
        cols: number of images per row
        cmap: Optional. Color map to use. For example, "Blues".
        norm: Optional. A Normalize instance to map values to colors.
        interpolation: Optional. Image interpolation to use for display.
        """
        titles = titles if titles is not None else [""] * len(images)
        rows = len(images) // cols + 1
        plt.figure(figsize=(14, 14 * rows // cols))
        i = 1
        for image, title in zip(images, titles):
            plt.subplot(rows, cols, i)
            plt.title(title, fontsize=9)
            plt.axis('off')
            plt.imshow(image.astype(np.uint8), cmap=cmap,
                       norm=norm, interpolation=interpolation)
            i += 1
        plt.show()
    
    
    def random_colors(N, bright=True):
        """
        Generate random colors.
        To get visually distinct colors, generate them in HSV space then
        convert to RGB.
        """
        brightness = 1.0 if bright else 0.7
        hsv = [(i / N, 1, brightness) for i in range(N)]
        colors = list(map(lambda c: colorsys.hsv_to_rgb(*c), hsv))
        random.shuffle(colors)
        return colors
    
    
    def apply_mask(image, mask, color, alpha=0.5):
        """Apply the given mask to the image.
        """
        for c in range(3):
            image[:, :, c] = np.where(mask == 1,
                                      image[:, :, c] *
                                      (1 - alpha) + alpha * color[c] * 255,
                                      image[:, :, c])
        return image
    
    
    def display_instances(image, boxes, masks, class_ids, class_names,
                          scores=None, title="",
                          figsize=(16, 16), ax=None,
                          show_mask=True, show_bbox=True,
                          colors=None, captions=None):
        """
        boxes: [num_instance, (y1, x1, y2, x2, class_id)] in image coordinates.
        masks: [height, width, num_instances]
        class_ids: [num_instances]
        class_names: list of class names of the dataset
        scores: (optional) confidence scores for each box
        title: (optional) Figure title
        show_mask, show_bbox: To show masks and bounding boxes or not
        figsize: (optional) the size of the image
        colors: (optional) An array or colors to use with each object
        captions: (optional) A list of strings to use as captions for each object
        """
        # Number of instances
        N = boxes.shape[0]
        if not N:
            print("\n*** No instances to display *** \n")
        else:
            assert boxes.shape[0] == masks.shape[-1] == class_ids.shape[0]
    
        # If no axis is passed, create one and automatically call show()
        auto_show = False
        if not ax:
            _, ax = plt.subplots(1, figsize=figsize)
            auto_show = True
    
        # Generate random colors
        colors = colors or random_colors(N)
    
        # Show area outside image boundaries.
        height, width = image.shape[:2]
        ax.set_ylim(height + 10, -10)
        ax.set_xlim(-10, width + 10)
        ax.axis('off')
        ax.set_title(title)
    
        masked_image = image.astype(np.uint32).copy()
        for i in range(N):
            color = colors[i]
    
            # Bounding box
            if not np.any(boxes[i]):
                # Skip this instance. Has no bbox. Likely lost in image cropping.
                continue
            y1, x1, y2, x2 = boxes[i]
            if show_bbox:
                p = patches.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=2,
                                    alpha=0.7, linestyle="dashed",
                                    edgecolor=color, facecolor='none')
                ax.add_patch(p)
    
            # Label
            if not captions:
                class_id = class_ids[i]
                score = scores[i] if scores is not None else None
                label = class_names[class_id]
                caption = "{} {:.3f}".format(label, score) if score else label
            else:
                caption = captions[i]
            ax.text(x1, y1 + 8, caption,
                    color='w', size=11, backgroundcolor="none")
    
            # Mask
            mask = masks[:, :, i]
            if show_mask:
                masked_image = apply_mask(masked_image, mask, color)
    
            # Mask Polygon
            # Pad to ensure proper polygons for masks that touch image edges.
            padded_mask = np.zeros(
                (mask.shape[0] + 2, mask.shape[1] + 2), dtype=np.uint8)
            padded_mask[1:-1, 1:-1] = mask
            contours = find_contours(padded_mask, 0.5)
            for verts in contours:
                # Subtract the padding and flip (y, x) to (x, y)
                verts = np.fliplr(verts) - 1
                p = Polygon(verts, facecolor="none", edgecolor=color)
                ax.add_patch(p)
        ax.imshow(masked_image.astype(np.uint8))
        if auto_show:
            plt.show()
    
    
    def display_differences(image,
                            gt_box, gt_class_id, gt_mask,
                            pred_box, pred_class_id, pred_score, pred_mask,
                            class_names, title="", ax=None,
                            show_mask=True, show_box=True,
                            iou_threshold=0.5, score_threshold=0.5):
        """Display ground truth and prediction instances on the same image."""
        # Match predictions to ground truth
        gt_match, pred_match, overlaps = utils.compute_matches(
            gt_box, gt_class_id, gt_mask,
            pred_box, pred_class_id, pred_score, pred_mask,
            iou_threshold=iou_threshold, score_threshold=score_threshold)
        # Ground truth = green. Predictions = red
        colors = [(0, 1, 0, .8)] * len(gt_match)\
               + [(1, 0, 0, 1)] * len(pred_match)
        # Concatenate GT and predictions
        class_ids = np.concatenate([gt_class_id, pred_class_id])
        scores = np.concatenate([np.zeros([len(gt_match)]), pred_score])
        boxes = np.concatenate([gt_box, pred_box])
        masks = np.concatenate([gt_mask, pred_mask], axis=-1)
        # Captions per instance show score/IoU
        captions = ["" for m in gt_match] + ["{:.2f} / {:.2f}".format(
            pred_score[i],
            (overlaps[i, int(pred_match[i])]
                if pred_match[i] > -1 else overlaps[i].max()))
                for i in range(len(pred_match))]
        # Set title if not provided
        title = title or "Ground Truth and Detections\n GT=green, pred=red, captions: score/IoU"
        # Display
        display_instances(
            image,
            boxes, masks, class_ids,
            class_names, scores, ax=ax,
            show_bbox=show_box, show_mask=show_mask,
            colors=colors, captions=captions,
            title=title)
    
    
    def draw_rois(image, rois, refined_rois, mask, class_ids, class_names, limit=10):
        """
        anchors: [n, (y1, x1, y2, x2)] list of anchors in image coordinates.
        proposals: [n, 4] the same anchors but refined to fit objects better.
        """
        masked_image = image.copy()
    
        # Pick random anchors in case there are too many.
        ids = np.arange(rois.shape[0], dtype=np.int32)
        ids = np.random.choice(
            ids, limit, replace=False) if ids.shape[0] > limit else ids
    
        fig, ax = plt.subplots(1, figsize=(12, 12))
        if rois.shape[0] > limit:
            plt.title("Showing {} random ROIs out of {}".format(
                len(ids), rois.shape[0]))
        else:
            plt.title("{} ROIs".format(len(ids)))
    
        # Show area outside image boundaries.
        ax.set_ylim(image.shape[0] + 20, -20)
        ax.set_xlim(-50, image.shape[1] + 20)
        ax.axis('off')
    
        for i, id in enumerate(ids):
            color = np.random.rand(3)
            class_id = class_ids[id]
            # ROI
            y1, x1, y2, x2 = rois[id]
            p = patches.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=2,
                                  edgecolor=color if class_id else "gray",
                                  facecolor='none', linestyle="dashed")
            ax.add_patch(p)
            # Refined ROI
            if class_id:
                ry1, rx1, ry2, rx2 = refined_rois[id]
                p = patches.Rectangle((rx1, ry1), rx2 - rx1, ry2 - ry1, linewidth=2,
                                      edgecolor=color, facecolor='none')
                ax.add_patch(p)
                # Connect the top-left corners of the anchor and proposal for easy visualization
                ax.add_line(lines.Line2D([x1, rx1], [y1, ry1], color=color))
    
                # Label
                label = class_names[class_id]
                ax.text(rx1, ry1 + 8, "{}".format(label),
                        color='w', size=11, backgroundcolor="none")
    
                # Mask
                m = utils.unmold_mask(mask[id], rois[id]
                                      [:4].astype(np.int32), image.shape)
                masked_image = apply_mask(masked_image, m, color)
    
        ax.imshow(masked_image)
        plt.savefig("../images/display_mask.png")
    
        # Print stats
        print("Positive ROIs: ", class_ids[class_ids > 0].shape[0])
        print("Negative ROIs: ", class_ids[class_ids == 0].shape[0])
        print("Positive Ratio: {:.2f}".format(
            class_ids[class_ids > 0].shape[0] / class_ids.shape[0]))
    
    
    # TODO: Replace with matplotlib equivalent?
    def draw_box(image, box, color):
        """Draw 3-pixel width bounding boxes on the given image array.
        color: list of 3 int values for RGB.
        """
        y1, x1, y2, x2 = box
        image[y1:y1 + 2, x1:x2] = color
        image[y2:y2 + 2, x1:x2] = color
        image[y1:y2, x1:x1 + 2] = color
        image[y1:y2, x2:x2 + 2] = color
        return image
    
    
    def display_top_masks(image, mask, class_ids, class_names, limit=4):
        """Display the given image and the top few class masks."""
        to_display = []
        titles = []
        to_display.append(image)
        titles.append("H x W={}x{}".format(image.shape[0], image.shape[1]))
        # Pick top prominent classes in this image
        unique_class_ids = np.unique(class_ids)
        mask_area = [np.sum(mask[:, :, np.where(class_ids == i)[0]])
                     for i in unique_class_ids]
        top_ids = [v[0] for v in sorted(zip(unique_class_ids, mask_area),
                                        key=lambda r: r[1], reverse=True) if v[1] > 0]
        # Generate images and titles
        for i in range(limit):
            class_id = top_ids[i] if i < len(top_ids) else -1
            # Pull masks of instances belonging to the same class.
            m = mask[:, :, np.where(class_ids == class_id)[0]]
            m = np.sum(m * np.arange(1, m.shape[-1] + 1), -1)
            to_display.append(m)
            titles.append(class_names[class_id] if class_id != -1 else "-")
        display_images(to_display, titles=titles, cols=limit + 1, cmap="Blues_r")
    
    
    def plot_precision_recall(AP, precisions, recalls):
        """Draw the precision-recall curve.
    
        AP: Average precision at IoU >= 0.5
        precisions: list of precision values
        recalls: list of recall values
        """
        # Plot the Precision-Recall curve
        _, ax = plt.subplots(1)
        ax.set_title("Precision-Recall Curve. AP@50 = {:.3f}".format(AP))
        ax.set_ylim(0, 1.1)
        ax.set_xlim(0, 1.1)
        _ = ax.plot(recalls, precisions)
    
    
    def plot_overlaps(gt_class_ids, pred_class_ids, pred_scores,
                      overlaps, class_names, threshold=0.5):
        """Draw a grid showing how ground truth objects are classified.
        gt_class_ids: [N] int. Ground truth class IDs
        pred_class_id: [N] int. Predicted class IDs
        pred_scores: [N] float. The probability scores of predicted classes
        overlaps: [pred_boxes, gt_boxes] IoU overlaps of predictions and GT boxes.
        class_names: list of all class names in the dataset
        threshold: Float. The prediction probability required to predict a class
        """
        gt_class_ids = gt_class_ids[gt_class_ids != 0]
        pred_class_ids = pred_class_ids[pred_class_ids != 0]
    
        plt.figure(figsize=(12, 10))
        plt.imshow(overlaps, interpolation='nearest', cmap=plt.cm.Blues)
        plt.yticks(np.arange(len(pred_class_ids)),
                   ["{} ({:.2f})".format(class_names[int(id)], pred_scores[i])
                    for i, id in enumerate(pred_class_ids)])
        plt.xticks(np.arange(len(gt_class_ids)),
                   [class_names[int(id)] for id in gt_class_ids], rotation=90)
    
        thresh = overlaps.max() / 2.
        for i, j in itertools.product(range(overlaps.shape[0]),
                                      range(overlaps.shape[1])):
            text = ""
            if overlaps[i, j] > threshold:
                text = "match" if gt_class_ids[j] == pred_class_ids[i] else "wrong"
            color = ("white" if overlaps[i, j] > thresh
                     else "black" if overlaps[i, j] > 0
                     else "grey")
            plt.text(j, i, "{:.3f}\n{}".format(overlaps[i, j], text),
                     horizontalalignment="center", verticalalignment="center",
                     fontsize=9, color=color)
    
        plt.tight_layout()
        plt.xlabel("Ground Truth")
        plt.ylabel("Predictions")
    
    
    def draw_boxes(image, boxes=None, refined_boxes=None,
                   masks=None, captions=None, visibilities=None,
                   title="", ax=None):
        """Draw bounding boxes and segmentation masks with different
        customizations.
    
        boxes: [N, (y1, x1, y2, x2, class_id)] in image coordinates.
        refined_boxes: Like boxes, but draw with solid lines to show
            that they're the result of refining 'boxes'.
        masks: [N, height, width]
        captions: List of N titles to display on each box
        visibilities: (optional) List of values of 0, 1, or 2. Determine how
            prominent each bounding box should be.
        title: An optional title to show over the image
        ax: (optional) Matplotlib axis to draw on.
        """
        # Number of boxes
        assert boxes is not None or refined_boxes is not None
        N = boxes.shape[0] if boxes is not None else refined_boxes.shape[0]
    
        # Matplotlib Axis
        if not ax:
            _, ax = plt.subplots(1, figsize=(12, 12))
    
        # Generate random colors
        colors = random_colors(N)
    
        # Show area outside image boundaries.
        margin = image.shape[0] // 10
        ax.set_ylim(image.shape[0] + margin, -margin)
        ax.set_xlim(-margin, image.shape[1] + margin)
        ax.axis('off')
    
        ax.set_title(title)
    
        masked_image = image.astype(np.uint32).copy()
        for i in range(N):
            # Box visibility
            visibility = visibilities[i] if visibilities is not None else 1
            if visibility == 0:
                color = "gray"
                style = "dotted"
                alpha = 0.5
            elif visibility == 1:
                color = colors[i]
                style = "dotted"
                alpha = 1
            elif visibility == 2:
                color = colors[i]
                style = "solid"
                alpha = 1
    
            # Boxes
            if boxes is not None:
                if not np.any(boxes[i]):
                    # Skip this instance. Has no bbox. Likely lost in cropping.
                    continue
                y1, x1, y2, x2 = boxes[i]
                p = patches.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=2,
                                      alpha=alpha, linestyle=style,
                                      edgecolor=color, facecolor='none')
                ax.add_patch(p)
    
            # Refined boxes
            if refined_boxes is not None and visibility > 0:
                ry1, rx1, ry2, rx2 = refined_boxes[i].astype(np.int32)
                p = patches.Rectangle((rx1, ry1), rx2 - rx1, ry2 - ry1, linewidth=2,
                                      edgecolor=color, facecolor='none')
                ax.add_patch(p)
                # Connect the top-left corners of the anchor and proposal
                if boxes is not None:
                    ax.add_line(lines.Line2D([x1, rx1], [y1, ry1], color=color))
    
            # Captions
            if captions is not None:
                caption = captions[i]
                # If there are refined boxes, display captions on them
                if refined_boxes is not None:
                    y1, x1, y2, x2 = ry1, rx1, ry2, rx2
                ax.text(x1, y1, caption, size=11, verticalalignment='top',
                        color='w', backgroundcolor="none",
                        bbox={'facecolor': color, 'alpha': 0.5,
                              'pad': 2, 'edgecolor': 'none'})
    
            # Masks
            if masks is not None:
                mask = masks[:, :, i]
                masked_image = apply_mask(masked_image, mask, color)
                # Mask Polygon
                # Pad to ensure proper polygons for masks that touch image edges.
                padded_mask = np.zeros(
                    (mask.shape[0] + 2, mask.shape[1] + 2), dtype=np.uint8)
                padded_mask[1:-1, 1:-1] = mask
                contours = find_contours(padded_mask, 0.5)
                for verts in contours:
                    # Subtract the padding and flip (y, x) to (x, y)
                    verts = np.fliplr(verts) - 1
                    p = Polygon(verts, facecolor="none", edgecolor=color)
                    ax.add_patch(p)
        ax.imshow(masked_image.astype(np.uint8))
        plt.savefig("../images/display_boxes.png")
    
    
    def display_table(table):
        """Display values in a table format.
        table: an iterable of rows, and each row is an iterable of values.
        """
        html = ""
        for row in table:
            row_html = ""
            for col in row:
                row_html += "<td>{:40}</td>".format(str(col))
            html += "<tr>" + row_html + "</tr>"
        html = "<table>" + html + "</table>"
        IPython.display.display(IPython.display.HTML(html))
    
    
    def display_weight_stats(model):
        """Scans all the weights in the model and returns a list of tuples
        that contain stats about each weight.
        """
        layers = model.get_trainable_layers()
        table = [["WEIGHT NAME", "SHAPE", "MIN", "MAX", "STD"]]
        for l in layers:
            weight_values = l.get_weights()  # list of Numpy arrays
            weight_tensors = l.weights  # list of TF tensors
            for i, w in enumerate(weight_values):
                weight_name = weight_tensors[i].name
                # Detect problematic layers. Exclude biases of conv layers.
                alert = ""
                if w.min() == w.max() and not (l.__class__.__name__ == "Conv2D" and i == 1):
                    alert += "<span style='color:red'>*** dead?</span>"
                if np.abs(w.min()) > 1000 or np.abs(w.max()) > 1000:
                    alert += "<span style='color:red'>*** Overflow?</span>"
                # Add row
                table.append([
                    weight_name + alert,
                    str(w.shape),
                    "{:+9.4f}".format(w.min()),
                    "{:+10.4f}".format(w.max()),
                    "{:+9.4f}".format(w.std()),
                ])
        display_table(table)

--------------------

# **utils** #

### **balloon\_dataset.py** ###

import os
    import json
    import sys
    import numpy as np
    sys.path.append("../")
    from mrcnn import utils, visualize
    from mrcnn.config import Config
    import skimage
    
    
    class BalloonConfig(Config):
        """继承MaskRCNN的模型配置信息
        修改其中需要的训练集数据信息
        """
        # 给配置一个名称
        NAME = "balloon"
    
        IMAGES_PER_GPU = 2
    
        # 类别数量（包括背景），气球类别+1
        NUM_CLASSES = 1 + 1
    
        # 一个epoch的步数
        STEPS_PER_EPOCH = 100
    
        # 检测的时候过滤置信度的阈值
        DETECTION_MIN_CONFIDENCE = 0.9
    
    
    class BalloonDataset(utils.Dataset):
        """气球分割数据集获取类
        """
        def load_balloon(self, dataset_dir, subset):
            """
            加载数据集
            :param dataset_dir: 数据集目录
            :param subset: 训练集还是测试机
            :return:
            """
            # 添加数据集类别数量
            self.add_class("balloon", 1, "balloon")
    
            # 是否提供在训练或者验证集字符串
            assert subset in ["train", "val"]
            dataset_dir = os.path.join(dataset_dir, subset)
            # Load annotations
            # { 'filename': '28503151_5b5b7ec140_b.jpg',
            #   'regions': {
            #       '0': {
            #           'region_attributes': {},
            #           'shape_attributes': {
            #               'all_points_x': [...],
            #               'all_points_y': [...],
            #               'name': 'polygon'}},
            #       ... more regions ...
            #   },
            # }
            # 读取标注区域：
            annotations = json.load(open(os.path.join(dataset_dir, "via_region_data.json")))
            annotations = list(annotations.values())
    
            # 如果annotations不存在直接跳过
            annotations = [a for a in annotations if a['regions']]
            # 添加每张图片的坐标
            for a in annotations:
                # 获取所有多边形的x, y 的所有点坐标,存储在shape_attributes
                # 判断其中类型是否是字典，若果字典
                if isinstance(a['regions'], dict):
                    polygons = [r['shape_attributes'] for r in a['regions'].values()]
                else:
                    polygons = [r['shape_attributes'] for r in a['regions']]
    
                # 读取图片内容获取长宽
                image_path = os.path.join(dataset_dir, a['filename'])
                image = skimage.io.imread(image_path)
                height, width = image.shape[:2]
    
                # 加入到image_info字典当中
                self.add_image(
                    "balloon",
                    image_id=a['filename'],
                    path=image_path,
                    width=width, height=height,
                    polygons=polygons)
    
        def load_mask(self, image_id):
            """加载图片中的mask返回每个图片的mask及其id
            :param image_id: 图片ID
            :return: masks: 一个实例的布尔形状 [height, width, instance count]
            class_ids: 类别的 1D 数组
            """
            # 如果不是balloon类别的图片数据，默认返回空
            image_info = self.image_info[image_id]
            if image_info["source"] != "balloon":
                return super(self.__class__, self).load_mask(image_id)
    
            # 将坐标转换成bitmap [height, width, instance_count]
            info = self.image_info[image_id]
            mask = np.zeros([info["height"], info["width"], len(info["polygons"])],
                            dtype=np.uint8)
            for i, p in enumerate(info["polygons"]):
                # Get indexes of pixels inside the polygon and set them to 1
                # 获取图片像素中的这个mask多边形区域中像素下标，将其标记为1
                rr, cc = skimage.draw.polygon(p['all_points_y'], p['all_points_x'])
                mask[rr, cc, i] = 1
    
            # 返回mask区域标记 [height, width, instance count]
            # 以及mask物体的个数
            return mask.astype(np.bool), np.ones([mask.shape[-1]], dtype=np.int32)
    
    
    if __name__ == '__main__':
        dataset_train = BalloonDataset()
        dataset_train.load_balloon("../balloon_data/", "train")
        dataset_train.prepare()
    
        # 打印结果
        print("图片数量: {}".format(len(dataset_train.image_ids))) #61
        print("类别数量: {}".format(dataset_train.num_classes)) #2
        for i, info in enumerate(dataset_train.class_info):
            """0. BG
              1. balloon"""
            print("{}. {}".format(i, info['name']))
    
        # 1、随机选择部分图片进行展示mask区域
        image_id = np.random.choice(dataset_train.image_ids, 1)[0]
        image = dataset_train.load_image(image_id)
        mask, class_ids = dataset_train.load_mask(image_id)
        # 显示mask
        visualize.display_top_masks(image, mask, class_ids, dataset_train.class_names)
    
        # 2、计算bbox
        bbox = utils.extract_bboxes(mask)
    
        from mrcnn.model import log
        log("image", image) #shape: (1421, 2048, 3)       min:    0.00000  max:  255.00000  uint8
        log("mask", mask)   #shape: (1421, 2048, 3)       min:    0.00000  max:    1.00000  bool
        log("class_ids", class_ids) #shape: (3,)                  min:    1.00000  max:    1.00000  int32
        log("bbox", bbox)           #shape: (3, 4)                min:    2.00000  max: 1652.00000  int32
        # 显示mask,以及bbox
        visualize.display_instances(image, bbox, mask, class_ids, dataset_train.class_names)
    
        # 3、计算anchor结果
        config = BalloonConfig()
        # resnet101网络每一层特征图形状：添加一个特征图大小属性（这里做测试需要设置一下才能用generate_pyramid_anchors），dataset中直接计算出来特征图大小
        # resnet默认有5个特征图，第一层特征图大小256x256，每一层特征图当中的每一个像素点有3个预测anchors先验框
        config.BACKBONE_SHAPES = [[256, 256], [128, 128], [64, 64], [32, 32], [16, 16]]
        # utils.generate_pyramid_anchors 需要rpn anchor的大小，rpn anchor的长宽比率等参数
        anchors = utils.generate_pyramid_anchors(config.RPN_ANCHOR_SCALES,
                                                 config.RPN_ANCHOR_RATIOS,
                                                 config.BACKBONE_SHAPES,
                                                 config.BACKBONE_STRIDES,
                                                 config.RPN_ANCHOR_STRIDE)
    
        # 打印anchor相关信息
        num_levels = len(config.BACKBONE_SHAPES) # 特征图总共有5级特征，每一层特征图的每一个像素有多少种不同类别的rpn生成的anchors先验框
        anchors_per_cell = len(config.RPN_ANCHOR_RATIOS) #anchors的比率值为3。5种尺度，3种比率，一共3*5=15种不同类别的 anchors先验框
        # 手动生成的anchors先验框的数量有多少
        print("Count: ", anchors.shape[0])           #5个特征图一共有 261888 个anchors先验框
        print("Scales: ", config.RPN_ANCHOR_SCALES) #anchors先验框 5种尺度 (32, 64, 128, 256, 512)
        print("ratios: ", config.RPN_ANCHOR_RATIOS) #anchors先验框 3种比率 [0.5, 1, 2]
        print("Anchors per Cell: ", anchors_per_cell)  #3种比率
        print("Levels: ", num_levels)                   #5种尺度
        anchors_per_level = []
        #手动打印每一层特征的anchors先验框的数量
        for l in range(num_levels):
            num_cells = config.BACKBONE_SHAPES[l][0] * config.BACKBONE_SHAPES[l][1]
            anchors_per_level.append(anchors_per_cell * num_cells // config.RPN_ANCHOR_STRIDE ** 2)
            """  resnet默认有5个特征图，第一层特征图大小256x256，每一层特征图当中的每一个像素点有3个预测anchors先验框。
                注意：每一层特征图当中的大部分预测anchors先验框都是没用的需要被过滤掉
                    Anchors in Level 0: 196608  （256x256x3=196608）
                    Anchors in Level 1: 49152   （128x128x3=49152）
                    Anchors in Level 2: 12288
                    Anchors in Level 3: 3072
                    Anchors in Level 4: 768
            """
            print("Anchors in Level {}: {}".format(l, anchors_per_level[l]))
    
        """
        数据集的准备阶段结果分析
        （1）根据参数配置的RPN_TRAIN_ANCHORS_PER_IMAGE = 256，对默认标记的RPN样本统计正负样本，正样本位置进行refine显示，
             ROIs感兴趣区域会进行正负样本标记，总共标记256个正负样本用于RPN训练使用，
             即每张图产生多少anchors(标记256个正负样本)用于RPN训练使用。
        （2）根据参数配置的TRAIN_ROIS_PER_IMAGE = 200，对于RPN产生的261888去做感兴趣区域计算得到默认200个输入到msrcnn中，
             即一张图片最终RPN网络只产生200个ROIs感兴趣区域给maskrcnn训练。
        """
    
        # 4、anchor到rois感兴趣区域
        from mrcnn import model
        random_rois = 2000
        """
        1.random_rois = 2000：
                当传入random_rois是大于0的值，那么返回的outputs值不为空，2000值没有意义，仅提供大于0值即可。
                训练期间不使用，用作调试debug使用或者单独训练不带有RPN网络的maskrcnn结构时使用。
                此处仅在测试期间使用，用于查看maskrcnn的输入输出结果情况。
                调用build_detection_targets函数返回感兴趣区域rois以及感兴趣区域的bboxes框位置和mask封装到outputs列表中。
                outputs用作提供给我们进行调试来看第二阶段maskrcnn的输入输出情况。
        2.DataGenerator 的 __getitem__：
                默认返回感兴趣区域rois，是一个建立RPN网络目标值，根据anchor先验框和目标GT之间生成目标值，
                也就是anchor先验框和目标GT之间做匹配(IoU值判断)，然后根据目标值对候选框进行正负样本标记。   
                anchor先验框在图片中是一个坐标位置值，而网络输出和训练损失值的时候都是使用偏移值计算的，
                因此要进行GT和anchor先验框之间的偏移值结果的计算，
                通过调用build_rpn_targets函数实现(对于目标GT以及RPN的众多acnhor，进行正负样本匹配，并且将转换极坐标到中心坐标)，
                最后anchor先验框偏移变成bbox的结果，最后build_rpn_targets函数返回rpn_match, rpn_bbox 。
        """
        # 获取4个数据测试看结果
        #获取anchor的代码以及进行获取感兴趣区域结果，model.DataGenerator会在内部返回结果
        g = model.DataGenerator(dataset_train, config,
                                shuffle=True,
                                random_rois=random_rois,
                                detection_targets=True)
        # 针对数据集的GT计算得到rpn的预测框以及mrcnn的输出预测框
        # inputs, outputs = __getitem__(0)：返回第一个批次的数据，random_rois大于0，也返回outputs列表
        [normalized_images, image_meta, rpn_match, rpn_bbox, gt_class_ids, gt_boxes, gt_masks, rpn_rois, rois], \
        [mrcnn_class_ids, mrcnn_bbox, mrcnn_mask] = g.__getitem__(0)
    
        # 打印rois以及mrcnn
        #shape: (2, 200, 4)：RPN的anchor过滤之后传入maskrcnn阶段的200个感兴趣区域，2指的样本数默认最小返回数量即2个图片为一个批量，4表示bbox框的4个位置
        log("rois", rois)
        """outputs列表：mrcnn_class_ids、mrcnn_bbox、mrcnn_mask
           根据参数配置的TRAIN_ROIS_PER_IMAGE = 200，对于RPN产生的261888去做感兴趣区域计算得到默认200个输入到msrcnn中，
           即一张图片最终RPN网络只产生200个ROIs感兴趣区域给maskrcnn训练。
        """
        #shape: (2, 200, 1)：1表示msrnn中用于训练的候选框的ID，200表示 RPN的anchor过滤之后传入maskrcnn阶段的200个感兴趣区域，第一个2表示2个图片为一个批量
        log("mrcnn_class_ids", mrcnn_class_ids)
        #shape: (2, 200, 2, 4)：4表示mrcnn中bbox的4个位置，第一个2表示2个图片为一个批量
        log("mrcnn_bbox", mrcnn_bbox)
        #shape: (2, 200, 28, 28, 2) 28x28表示感兴趣区域的大小，最后一个2表示有两个mask标记的物体，第一个2表示2个图片为一个批量
        log("mrcnn_mask", mrcnn_mask)
    
        # 打印GT结果
        #shape: (2, 100)：第一个2表示2个图片为一个批量
        log("gt_class_ids", gt_class_ids)
        #msrcnn最终会有100个框
        #shape: (2, 100, 4)：100表示GT框数量，根据参数配置MAX_GT_INSTANCES = 100 设置每张图的GT实例数量的最大值，第一个2表示2个图片为一个批量
        log("gt_boxes", gt_boxes)
        #shape: (2, 56, 56, 100)：第一个2表示2个图片为一个批量
        log("gt_masks", gt_masks)
        #rpn的anchor标记结果
        #shape: (2, 261888, 1)：resnet101网络有5个特征图一共有 261888 个anchors先验框，1表示要标记的正负样本的类别值(1=positive anchor, -1=negative, 0=neutral)
        log("rpn_match", rpn_match, )
        #每张图 RPN 使用256个正负样本 进行训练，第一个2表示2个图片为一个批量
        #shape: (2, 256, 4) ：anchor与GT之间计算的偏移值，从261888个anchors先验框中 计算出256个ROIs感兴趣区域 标记正负样本 用于 RPN网络训练，4表示bbox框的4个位置
        log("rpn_bbox", rpn_bbox)
        #此图片ID
        image_id = image_meta[0][0]
        print("image_id: ", image_id)
    
        """
        1.261888个先验框与GT目标框进行匹配，得到 Positive/Negative/Neutral 即 10+246=256和256+261632=261888的 RPN网络标记的 正负样本和无效样本，
          然后正样本继续和GT目标框进行做偏移值计算，即RPN网络会继续拿正样本和GT目标框的偏移值继续来训练网络，
          网络预测的即是正样本候选框与GT目标框之间的偏移值，那么最终网络拿预测的偏移值结果和正样本候选框加起来计算就能越接近GT目标框。
          rpn_bbox保存了所有的先验框和所有的GT目标框之间的偏移值。
        2.RPN网络训练完成之后预测输出的结果(200个的候选框)提供给msrcnn训练：176(背景数量)+24(气球数量)=200个的候选框(类别结果)输入msrcnn中进行训练。
          msrcnn的感兴趣区域ROIS的正负样本标记：176(背景数量)+24(气球数量)=200个的候选框，其中正样本占200个样本的比率是0.13，
          正样本占200个样本的比率一般是0.33，但也并不是每次都一定是0.33。
        3.对于标记之后的结果，做正负样本数量统计，并且对于正样本的数据做微调之后结果打印在图片中，负样本同时也打印在图片中显示出来。
        """
        # 5、对于其中一张图片进行anchor的refine
        # 获取正负样本匹配结果
        b = 0
        #rpn_match 默认有两个图片，shape(2, 261888, 1) 第一个2表示2个图片为一个批量，因此此处rpn_match[0]获取第一张图片的正负样本
        #rpn_match[0]==比较的1、-1、0：表示(1=positive anchor, -1=negative, 0=neutral) 正负样本分类值判断
        positive_anchor_ids = np.where(rpn_match[b] == 1)[0]
        ## RPN标记正样本数量 10+246=256
        print("Positive anchors: {}".format(len(positive_anchor_ids))) #正样本 数量 10
        negative_anchor_ids = np.where(rpn_match[b] == -1)[0]
        print("Negative anchors: {}".format(len(negative_anchor_ids))) #负样本 数量 246
        neutral_anchor_ids = np.where(rpn_match[b] == 0)[0]
        print("Neutral anchors: {}".format(len(neutral_anchor_ids))) #RPN标记无效框数量 261632
    
        # 对于标记为正样本anchor进行位置refine(提纯)计算
        #取出第一张图的正样本的索引
        indices = np.where(rpn_match[b] == 1)[0]
        #对anchors[indices]正样本的候选框进行训练时，首先要进行anchors正样本和目标GT做偏移值计算
        refined_anchors = utils.apply_box_deltas(anchors[indices], rpn_bbox[b, :len(indices)] * config.RPN_BBOX_STD_DEV)
        log("anchors", anchors) # anchors总数量 shape: (261888, 4)
        log("refined_anchors", refined_anchors) # 正样本进行refined之后的anchor数量  shape: (10, 4)
    
        # 获取其中默认第一张图片的数据，打印正样本标记结果和负样本标记结果
        sample_image = model.unmold_image(normalized_images[b], config)
        # ROI的类别数量
        #mrcnn_class_ids的shape: (2, 200, 1)：2表示2个图片为一个批量，1表示msrnn中用于训练的候选框的ID，200表示RPN的anchor过滤之后传入maskrcnn阶段的200个感兴趣区域，
        for c, n in zip(dataset_train.class_names, np.bincount(mrcnn_class_ids[b].flatten())):
            """
            # 其中这200个bbox框的类别结果：用于输入到msrcnn当中176+24=200个的候选框数量(类别结果)
            # 背景数量
            BG                     : 176
            # 气球数量
            balloon                : 24
            """
            if n:
                print("{:23}: {}".format(c[:20], n))
    
        # 展示正样本输出结果
        import matplotlib.pyplot as plt
        fig, ax = plt.subplots(1, figsize=(16, 16))
        # 展示正样本输出结果
        visualize.draw_boxes(sample_image, boxes=anchors[positive_anchor_ids],
                             refined_boxes=refined_anchors, ax=ax)
        # 展示负样本输出
        visualize.draw_boxes(sample_image, boxes=anchors[negative_anchor_ids])
    
        """
        正样本占其中全部样本的的比率是0.33
        Positive ROIs:  66
        Negative ROIs:  134
        Positive Ratio: 0.33
        """
        # 6、正负rois区域比例
        print("Positive ROIs: ", mrcnn_class_ids[b][mrcnn_class_ids[b] > 0].shape[0])
        print("Negative ROIs: ", mrcnn_class_ids[b][mrcnn_class_ids[b] == 0].shape[0])
        print("Positive Ratio: {:.2f}".format(
            mrcnn_class_ids[b][mrcnn_class_ids[b] > 0].shape[0] / mrcnn_class_ids[b].shape[0]))

--------------------

### **draw\_segmention\_utils.py** ###

import numpy as np
    import skimage
    
    
    def draw_segmentation(image, mask):
        """
        对图片进行分割区域的标记
        :param image: 输出图片 RGB image [height, width, 3]
        :param mask: 分割区域[height, width, instance count]
        :return: 返回黑白图片，并且将分割区域保留原来的颜色
        """
        # 1、将彩色图片变成灰度图，并保留image以及同份灰色的图片
        # 注意从RGB到灰度图格式格式会从float32到unit8转换
        gray = skimage.color.gray2rgb(skimage.color.rgb2gray(image)) * 255
    
        # 2、将彩色格式中mask部分保留其余部分都设置成灰色，
        if mask.shape[-1] > 0:
            """
            np.where(condition, x, y) 满足条件(condition)，输出x，不满足输出y
            >>> aa = np.arange(10)
            >>> aa
            array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
            >>> np.where(aa, 1, -1)
            array([-1,  1,  1,  1,  1,  1,  1,  1,  1,  1])
            >>> np.where(aa > 5, 1, -1)
            array([-1, -1, -1, -1, -1, -1,  1,  1,  1,  1])
            """
            # 求出每个像素点需要保留和显示灰度图的判断
            mask = (np.sum(mask, -1, keepdims=True) >= 1)
            segmentation = np.where(mask, image, gray).astype(np.uint8)
        else:
            segmentation = gray.astype(np.uint8)
        return segmentation
    
    
    def detect_and_draw_segmentation(args, model):
        """
        检测结果并画出分割区域
        :param args: 命令行参数
        :param model: 模型
        :return:
        """
        if not args.image or not args.video:
            raise ValueError("请提供要检测的图片或者视频路径之一")
    
        # 传入的图片
        if args.image:
            print("正在分割图片：{}".format(args.image))
            # 1、读取图片
            image = skimage.io.imread(args.image)
            # 2、模型检测返回结果
            r = model.detect([image], verbose=1)[0]
            # 3、画出分割区域
            segmentation = draw_segmentation(image, r['masks'])
            # 4、保存输出
            file_name = "./images/segment_{}".format(args.image.split("/")[-1])
            skimage.io.imsave(file_name, segmentation)
    
        if args.video:
            import cv2
            # 1、获取视频的读取
            vcapture = cv2.VideoCapture(args.video)
            width = int(vcapture.get(cv2.CAP_PROP_FRAME_WIDTH))
            height = int(vcapture.get(cv2.CAP_PROP_FRAME_HEIGHT))
            fps = vcapture.get(cv2.CAP_PROP_FPS)
    
            # 2、定义video writer后续写入
            file_name = "./images/segmentation_{}".format(args.video.split("/")[-1])
            vwriter = cv2.VideoWriter(file_name,
                                      cv2.VideoWriter_fourcc(*'mp4v'),
                                      fps, (width, height))
    
            # 3、循环获取每帧数据进行处理，完成之后写入本地文件
            count = 0
            success = True
            while success:
                print("帧数: ", count)
                # 读取图片
                success, image = vcapture.read()
                if success:
                    # OpenCV 返回的BGR格式转换成RGB
                    image = image[..., ::-1]
                    # 模型检测mask
                    r = model.detect([image], verbose=0)[0]
                    # 画出区域
                    segmentation = draw_segmentation(image, r['masks'])
                    # RGB -> BGR
                    segmentation = segmentation[..., ::-1]
                    # 添加这张图到video writer
                    vwriter.write(segmentation)
                    count += 1
            vwriter.release()
    
        print("保存到检测结果到路径文件：", file_name)

--------------------

### **balloon\_main.py** ###

import numpy as np
    import skimage.draw
    import argparse
    
    from mrcnn import model as maskrcnn
    from utils.balloon_dataset import BalloonDataset, BalloonConfig
    from utils.draw_segmention_utils import detect_and_draw_segmentation
    import os
    os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
    
    
    # 命令行参数
    parser = argparse.ArgumentParser(
        description='气球分割模型maskrcnn训练')
    parser.add_argument("--command", type=str, default='test',
                        help="'train' or 'test' 训练还是进行测试")
    parser.add_argument('--dataset', type=str, default='./balloon_data',
                        help='气球分割数据集目录')
    #权重下载链接 https://github-production-release-asset-2e65be.s3.amazonaws.com/64878964/7ee9c8c6-5e1c-11e6-95f9-0ce2eddabcab?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20200403%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20200403T091749Z&X-Amz-Expires=300&X-Amz-Signature=54a83d5413957b8eceb0eb4f8557741fb606d3b7e7fd63e182a5c410a68cf963&X-Amz-SignedHeaders=host&actor_id=29617050&response-content-disposition=attachment%3B%20filename%3Dresnet50_weights_tf_dim_ordering_tf_kernels_notop.h5&response-content-type=application%2Foctet-stream
    parser.add_argument('--weights', type=str, default='imagenet',
                        help="预训练模型权重"
                             "imagenet:https://github.com/fchollet/"
                             "deep-learning-models/releases/"
                             "download/v0.2/resnet50_weights"
                             "_tf_dim_ordering_tf_kernels_notop.h5")
    parser.add_argument('--logs', type=str, default='./logs/',
                        help='打印日志目录')
    parser.add_argument('--image', type=str, default='./images/2917282960_06beee649a_b.jpg',
                        help='需要进行检测分割的图片目录')
    parser.add_argument('--video', type=str, default='./images/v0200fd10000bq043q9pskdh7ri20vm0.MP4',
                        help='需要进行检测分割的视频目录')
    """
    1.第一个预训练模型权重：mask_rcnn_balloon.h5 专门用于气球检测模型
    2.第二个预训练模型权重：resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5 预训练模型权重下载，该模型无法直接应用于气球检测
        https://github-production-release-asset-2e65be.s3.amazonaws.com/64878964/7ee9c8c6-5e1c-11e6-95f9-0ce2eddabcab?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20200403%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20200403T091749Z&X-Amz-Expires=300&X-Amz-Signature=54a83d5413957b8eceb0eb4f8557741fb606d3b7e7fd63e182a5c410a68cf963&X-Amz-SignedHeaders=host&actor_id=29617050&response-content-disposition=attachment%3B%20filename%3Dresnet50_weights_tf_dim_ordering_tf_kernels_notop.h5&response-content-type=application%2Foctet-stream
    """
    parser.add_argument('--model', type=str, default='./logs/mask_rcnn_balloon.h5',
                        help='指定测试使用的训练好的模型文件')
    # parser.add_argument('--model', type=str, default='./logs/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5',
    #                     help='指定测试使用的训练好的模型文件')
    
    def train(model):
        """训练模型逻辑
        :param model: maskrcnn模型
        :return:
        """
        # 1、获取分割数据集
        dataset_train = BalloonDataset()
        dataset_train.load_balloon(args.dataset, "train")
        dataset_train.prepare()
    
        # 2、获取分割验证数据集
        dataset_val = BalloonDataset()
        dataset_val.load_balloon(args.dataset, "val")
        dataset_val.prepare()
    
        # 3、开始训练
        print("开始训练网络：")
        model.train(dataset_train, dataset_val,
                    learning_rate=config.LEARNING_RATE,
                    epochs=30,
                    layers='heads')
    
    
    if __name__ == '__main__':
    
        args = parser.parse_args()
    
        # 1、进行参数传入判断
        if args.command == "train":
            assert args.dataset, "指定训练的时候必须传入 --dataset数据目录"
        elif args.command == "test":
            assert args.image or args.video,\
                   "指定测试的时候必须提供图片或者视频"
    
        # 2、配置模型的参数、数据集的训练读取配置
        if args.command == "train":
            config = BalloonConfig()
        else:
            # 测试的配置修改：设置batch_size为1，Batch size = GPU_COUNT * IMAGES_PER_GPU
            class InferenceConfig(BalloonConfig):
                GPU_COUNT = 1
                IMAGES_PER_GPU = 1
            config = InferenceConfig()
        config.display()
    
        # 3、创建模型
        if args.command == "train":
            model = maskrcnn.MaskRCNN(mode="training", config=config,
                                      model_dir=args.logs)
        else:
            model = maskrcnn.MaskRCNN(mode="inference", config=config,
                                      model_dir=args.logs)
        # 4、训练测试逻辑实现
        if args.command == "train":
            # # 选择加载的预训练模型类别并下载
            # if args.weights.lower() == "imagenet":
            #     weights_path = model.get_imagenet_weights()
            # else:
            #     raise ValueError("提供一种预训练模型种类")
    
            # # 加载预训练模型权重
            # print("Loading weights ", weights_path)
            # model.load_weights(weights_path, by_name=True)
    
            # 加载预训练模型权重
            print("Loading weights ", args.model)
            model.load_weights(args.model, by_name=True)
    
            # 进行训练
            train(model)
    
        elif args.command == "test":
            model.load_weights(args.model, by_name=True)
            # 进行 图片检测+视频检测
            detect_and_draw_segmentation(args, model)
        else:
            print("'{}' 传入参数无法识别. "
                  "请使用 'train' or 'test'".format(args.command))

[20191009191333910.png]: /images/20230528/874f2f2366e94aef8db807666a7a9069.png
[Link 1]: http://www.rimengshe.com/
[AI_Keras PyTorch MXNet TensorFlow PaddlePaddle]: https://blog.csdn.net/zimiao552147572/article/details/88867161
[Mask RCNN]: https://blog.csdn.net/zimiao552147572/article/details/105300202
[Mask RCNN _ part1]: https://blog.csdn.net/zimiao552147572/article/details/105301072
[Mask RCNN _ part2]: https://blog.csdn.net/zimiao552147572/article/details/105301253