Imports

import os
import supervisely_lib as sly
import cv2
import numpy as np

# PyPlot for drawing images in Jupyter.
%matplotlib inline
import matplotlib.pyplot as plt

def draw_labeled_image(img, ann, draw_fill=False):
    canvas_draw_contour = img.copy()
    if ann is not None:
        if draw_fill is True:
            ann.draw(canvas_draw_contour)
        else:
            ann.draw_contour(canvas_draw_contour, thickness=7)
    fig = plt.figure(figsize=(30, 30))
    fig.add_subplot(1, 2, 1)
    plt.imshow(img)
    if ann is not None:
        fig.add_subplot(1, 2, 2)
        plt.imshow(canvas_draw_contour)
    plt.show()

Initialize API access with your credentials

address = os.environ['SERVER_ADDRESS']
token = os.environ['API_TOKEN']

print("Server address: ", address)
#print("Your API token: ", token)

Server address:  proxy

Initialize the API access object

api = sly.Api(address, token)

Script parameters

team_name = "max"
workspace_name = "detection_segmentation_pipeline_example"

agent_name = "maxim_server"

model_detection_name = "YOLO v3 (COCO)"
model_segmentation_name = "DeepLab v3 plus (VOC2012)"

Verify parameters

team = api.team.get_info_by_name(team_name)
if team is None:
    raise RuntimeError("Team {!r} not found".format(team_name))

workspace = api.workspace.get_info_by_name(team.id, workspace_name)
if workspace is None:
    raise RuntimeError("Workspace {!r} not found".format(workspace_name))

print("Team: id={}, name={}".format(team.id, team.name))
print("Workspace: id={}, name={}".format(workspace.id, workspace.name))

model_detection = api.model.get_info_by_name(workspace.id, model_detection_name)
if model_detection is None:
    clone_task_id = api.model.clone_from_explore("Supervisely/Model Zoo/" + model_detection_name,
                                                 workspace.id,
                                                 model_detection_name)
    api.task.wait(clone_task_id, api.task.Status.FINISHED)
    model_detection = api.model.get_info_by_name(workspace.id, model_detection_name)

print("Model (detection): id={}, name={}".format(model_detection.id, model_detection.name))


model_segmentation = api.model.get_info_by_name(workspace.id, model_segmentation_name)
if model_segmentation is None:
    clone_task_id = api.model.clone_from_explore("Supervisely/Model Zoo/" + model_segmentation_name,
                                                 workspace.id,
                                                 model_segmentation_name)
    api.task.wait(clone_task_id, api.task.Status.FINISHED)
    model_segmentation = api.model.get_info_by_name(workspace.id, model_segmentation_name)

print("Model (segmentation): id={}, name={}".format(model_segmentation.id, model_segmentation.name))


agent = api.agent.get_info_by_name(team.id, agent_name)
if agent is None:
    raise RuntimeError("Agent {!r} not found".format(agent_name))
if agent.status is api.agent.Status.WAITING:
    raise RuntimeError("Agent {!r} is not running".format(agent_name))
print("Agent: id={}, name={}".format(agent.id, agent.name))

Team: id=600, name=max
Workspace: id=26686, name=detection_segmentation_pipeline_example
Model (detection): id=13912, name=YOLO v3 (COCO)
Model (segmentation): id=13913, name=DeepLab v3 plus (VOC2012)
Agent: id=377, name=maxim_server

Deploy both detection and segmentataion models (if not deployed before)

detploy_task_id_detection = api.task.deploy_model(agent.id, model_detection.id)
detploy_task_id_segmentation = api.task.deploy_model(agent.id, model_segmentation.id)
print("Debug info:")
print("Deploy task_id for detection model: {}".format(detploy_task_id_detection))
print("Deploy task_id for segmentation model: {}".format(detploy_task_id_segmentation))

Debug info:
Deploy task_id for detection model: 134678
Deploy task_id for segmentation model: 134679

Get models ouptut classes (+ check that models are deployed correctly)

# dummy config is used to slightly rename model output class
dummy_detection_config = {
    "model_tags": {
      "add_suffix": "_det",
      "save_names": "__all__"
    },
    "model_classes": {
      "add_suffix": "_det",
      "save_classes": "__all__"
    }
}

meta_json = api.model.get_output_meta(model_detection.id, inference_mode=dummy_detection_config)
model_detection_meta = sly.ProjectMeta.from_json(meta_json)
print("Output classes (detection model)")
print(model_detection_meta)

Output classes (detection model)
ProjectMeta:
Object Classes
+--------------------+-----------+-----------------+
|        Name        |   Shape   |      Color      |
+--------------------+-----------+-----------------+
|     person_det     | Rectangle | [146, 208, 134] |
|    bicycle_det     | Rectangle | [116, 127, 233] |
|      car_det       | Rectangle | [233, 189, 207] |
|   motorbike_det    | Rectangle | [111, 190, 245] |
|   aeroplane_det    | Rectangle |  [92, 126, 104] |
|      bus_det       | Rectangle | [212, 239, 134] |
|     train_det      | Rectangle | [140, 180, 183] |
|     truck_det      | Rectangle | [231, 222, 180] |
|      boat_det      | Rectangle |  [213, 86, 211] |
| traffic light_det  | Rectangle | [137, 206, 104] |
|  fire hydrant_det  | Rectangle | [194, 160, 183] |
|   stop sign_det    | Rectangle | [131, 156, 191] |
| parking meter_det  | Rectangle |  [96, 163, 96]  |
|     bench_det      | Rectangle | [232, 202, 225] |
|      bird_det      | Rectangle | [253, 192, 185] |
|      cat_det       | Rectangle | [109, 250, 167] |
|      dog_det       | Rectangle | [214, 227, 223] |
|     horse_det      | Rectangle | [215, 164, 135] |
|     sheep_det      | Rectangle | [208, 112, 181] |
|      cow_det       | Rectangle | [100, 211, 137] |
|    elephant_det    | Rectangle | [178, 189, 166] |
|      bear_det      | Rectangle | [117, 129, 129] |
|     zebra_det      | Rectangle | [160, 207, 150] |
|    giraffe_det     | Rectangle |  [91, 155, 186] |
|    backpack_det    | Rectangle | [228, 217, 157] |
|    umbrella_det    | Rectangle | [136, 169, 229] |
|    handbag_det     | Rectangle | [100, 181, 251] |
|      tie_det       | Rectangle |  [95, 201, 229] |
|    suitcase_det    | Rectangle | [182, 227, 200] |
|    frisbee_det     | Rectangle |  [102, 168, 94] |
|      skis_det      | Rectangle |  [116, 166, 87] |
|   snowboard_det    | Rectangle | [231, 152, 160] |
|  sports ball_det   | Rectangle | [253, 239, 246] |
|      kite_det      | Rectangle | [107, 158, 211] |
|  baseball bat_det  | Rectangle | [123, 100, 233] |
| baseball glove_det | Rectangle | [225, 126, 184] |
|   skateboard_det   | Rectangle | [216, 171, 174] |
|   surfboard_det    | Rectangle | [144, 216, 188] |
| tennis racket_det  | Rectangle | [182, 156, 250] |
|     bottle_det     | Rectangle | [230, 209, 159] |
|   wine glass_det   | Rectangle |  [183, 254, 98] |
|      cup_det       | Rectangle | [215, 243, 120] |
|      fork_det      | Rectangle | [148, 247, 126] |
|     knife_det      | Rectangle | [175, 100, 183] |
|     spoon_det      | Rectangle | [245, 171, 198] |
|      bowl_det      | Rectangle |  [96, 216, 100] |
|     banana_det     | Rectangle | [123, 135, 104] |
|     apple_det      | Rectangle | [209, 147, 152] |
|    sandwich_det    | Rectangle | [211, 209, 131] |
|     orange_det     | Rectangle | [115, 132, 226] |
|    broccoli_det    | Rectangle | [108, 234, 113] |
|     carrot_det     | Rectangle | [136, 121, 238] |
|    hot dog_det     | Rectangle |  [101, 87, 230] |
|     pizza_det      | Rectangle | [128, 233, 240] |
|     donut_det      | Rectangle | [217, 254, 187] |
|      cake_det      | Rectangle | [118, 198, 160] |
|     chair_det      | Rectangle |  [213, 96, 120] |
|      sofa_det      | Rectangle | [240, 145, 177] |
|  pottedplant_det   | Rectangle | [238, 211, 241] |
|      bed_det       | Rectangle | [186, 198, 157] |
|  diningtable_det   | Rectangle | [200, 219, 127] |
|     toilet_det     | Rectangle | [175, 247, 104] |
|   tvmonitor_det    | Rectangle | [121, 243, 189] |
|     laptop_det     | Rectangle | [126, 239, 127] |
|     mouse_det      | Rectangle | [171, 138, 156] |
|     remote_det     | Rectangle | [251, 104, 192] |
|    keyboard_det    | Rectangle | [128, 202, 223] |
|   cell phone_det   | Rectangle | [108, 201, 122] |
|   microwave_det    | Rectangle | [248, 218, 143] |
|      oven_det      | Rectangle | [178, 158, 127] |
|    toaster_det     | Rectangle |  [120, 119, 97] |
|      sink_det      | Rectangle | [216, 216, 127] |
|  refrigerator_det  | Rectangle |  [94, 129, 108] |
|      book_det      | Rectangle | [178, 127, 145] |
|     clock_det      | Rectangle |  [147, 86, 212] |
|      vase_det      | Rectangle | [136, 159, 104] |
|    scissors_det    | Rectangle | [183, 114, 216] |
|   teddy bear_det   | Rectangle |  [99, 174, 203] |
|   hair drier_det   | Rectangle | [148, 189, 224] |
|   toothbrush_det   | Rectangle | [164, 225, 168] |
+--------------------+-----------+-----------------+
Tags
+----------------+------------+-----------------+
|      Name      | Value type | Possible values |
+----------------+------------+-----------------+
| confidence_det | any_number |       None      |
+----------------+------------+-----------------+

# dummy config is used to slightly rename model output class
dummy_segmentation_config = {
    "model_tags": {
      "add_suffix": "_seg",
      "save_names": "__all__"
    },
    "model_classes": {
      "add_suffix": "_seg",
      "save_classes": "__all__"
    }
}

meta_json = api.model.get_output_meta(model_segmentation.id, inference_mode=dummy_segmentation_config)
model_segmentation_meta = sly.ProjectMeta.from_json(meta_json)
print("Output classes (segmentation model)")
print(model_segmentation_meta)

Output classes (segmentation model)
ProjectMeta:
Object Classes
+------------------+--------+-----------------+
|       Name       | Shape  |      Color      |
+------------------+--------+-----------------+
|      bg_seg      | Bitmap | [182, 182, 159] |
|  aeroplane_seg   | Bitmap |  [255, 0, 153]  |
|   bicycle_seg    | Bitmap |  [191, 0, 115]  |
|     bird_seg     | Bitmap |   [128, 0, 77]  |
|     boat_seg     | Bitmap |  [179, 0, 255]  |
|    bottle_seg    | Bitmap |  [134, 0, 191]  |
|     bus_seg      | Bitmap |   [90, 0, 128]  |
|     car_seg      | Bitmap |   [9, 0, 255]   |
|     cat_seg      | Bitmap |   [6, 0, 191]   |
|    chair_seg     | Bitmap |   [4, 0, 128]   |
|     cow_seg      | Bitmap |  [0, 161, 255]  |
| diningtable_seg  | Bitmap |  [0, 121, 191]  |
|     dog_seg      | Bitmap |   [0, 81, 128]  |
|    horse_seg     | Bitmap |  [0, 255, 179]  |
|  motorbike_seg   | Bitmap |  [0, 191, 134]  |
|    person_seg    | Bitmap |   [0, 128, 90]  |
| potted-plant_seg | Bitmap |   [0, 255, 0]   |
|    sheep_seg     | Bitmap |   [0, 191, 0]   |
|     sofa_seg     | Bitmap |   [0, 128, 0]   |
|    train_seg     | Bitmap |  [170, 255, 0]  |
|  tv-monitor_seg  | Bitmap |  [127, 191, 0]  |
+------------------+--------+-----------------+
Tags
+------+------------+-----------------+
| Name | Value type | Possible values |
+------+------------+-----------------+
+------+------------+-----------------+

img = sly.image.read('collage_03.jpg')

draw_labeled_image(img, None)

Apply detection model (sliding window)

In current example image has big resolution, and the models were trained with the smaller one.

Thus we are going to aply models in a sliding window manner. Learn more here:

https://supervise.ly/explore/notebooks/guide-04-neural-network-inference-20/overview

https://supervise.ly/explore/notebooks/guide-06-neural-network-inference-modes-215/overview

detection_sliding_window_config = {
    "name": "sliding_window_det",

    # Sliding window parameters.

    # Width and height in pixels.
    # Cannot be larger than the original image.
    "window": {
      "width": 1000,
      "height": 1000,
    },

    # Minimum overlap for each dimension. The last
    # window in every dimension may have higher overlap
    # with the previous one if necessary to fit the whole
    # window within the original image.
    "min_overlap": {
      "x": 200,
      "y": 200,
    },

    # Whether to save each sliding window instance as a
    # bounding box rectangle.
    "save": False,

    # If saving the sliding window bounding boxes, which
    # class name to use.
    "class_name": 'sliding_window_bbox',

    "nms_after": {

      # Whether to run non-maximum suppression after accumulating
      # all the detection results from the sliding windows.
      "enable": True,

      # Intersection over union threshold above which the same-class
      # detection labels are considered to be significantly inersected
      # for non-maximum suppression.
      "iou_threshold": 0.2,

      # Tag name from which to read detection confidence by which we
      # rank the detections. This tag must be added by the model to
      # every detection label.
      "confidence_tag_name": "confidence"
    },

    # Class renaming and filtering settings.
    # See "Full image inference" example for details.
    "model_classes": {
      "add_suffix": "_det",
      "save_classes": ["tie"]
    },

    "model_tags": {
      "add_suffix": "_det",
      "save_names": "__all__"
    }
}

ann_json = api.model.inference(model_detection.id, img, mode=detection_sliding_window_config)
print(ann_json)

{'description': '', 'size': {'height': 5000, 'width': 3137}, 'tags': [], 'objects': [{'classTitle': 'tie_det', 'description': '', 'tags': [{'name': 'confidence_det', 'value': 0.9625}], 'points': {'exterior': [[1766, 674], [1851, 990]], 'interior': []}}, {'classTitle': 'tie_det', 'description': '', 'tags': [{'name': 'confidence_det', 'value': 0.9773}], 'points': {'exterior': [[1966, 3737], [2350, 3922]], 'interior': []}}, {'classTitle': 'tie_det', 'description': '', 'tags': [{'name': 'confidence_det', 'value': 0.9807}], 'points': {'exterior': [[375, 2608], [733, 2866]], 'interior': []}}, {'classTitle': 'tie_det', 'description': '', 'tags': [{'name': 'confidence_det', 'value': 0.9827}], 'points': {'exterior': [[976, 908], [1050, 1086]], 'interior': []}}, {'classTitle': 'tie_det', 'description': '', 'tags': [{'name': 'confidence_det', 'value': 0.9957}], 'points': {'exterior': [[525, 4317], [774, 4807]], 'interior': []}}], 'success': True}

# Deserialize the annotation using the model meta information that
# we received previously.
ann_det = sly.Annotation.from_json(ann_json, model_detection_meta)

# Render the inference results.
draw_labeled_image(img, ann_det)

Apply segmentation model (sliding window)

segmentation_sliding_window_config = {
    "name": "sliding_window",

    # Sliding window parameters.

    # Width and height in pixels.
    # Cannot be larger than the original image.
    "window": {
      "width": 2000,
      "height": 2000,
    },

    # Minimum overlap for each dimension. The last
    # window in every dimension may have higher overlap
    # with the previous one if necessary to fit the whole
    # window within the original image.
    "min_overlap": {
      "x": 400,
      "y": 400,
    },

    # Whether to save each sliding window instance as a
    # bounding box rectangle.
    #"save": False,

    # If saving the sliding window bounding boxes, which
    # class name to use.
    #"class_name": 'sliding_window_bbox',

    # Class renaming and filtering settings.
    # See "Full image inference" example for details.
    "model_classes": {
      "add_suffix": "_seg",
      "save_classes": ["person", "dog", "cat"]
    },

    #"model_tags": {
    #  "add_suffix": "",
    #  "save_names": "__all__"
    #}
}

ann_json = api.model.inference(model_segmentation.id, img, mode=segmentation_sliding_window_config)
#print(ann_json)

# Deserialize the annotation using the model meta information that
# we received previously.
ann_seg = sly.Annotation.from_json(ann_json, model_segmentation_meta)

# Render the inference results.
draw_labeled_image(img, ann_seg, draw_fill=True)

Postprocessing step: keep only ties that are on the cats

def _render_labels_for_class_name(labels, class_name, canvas):
    for label in labels:
        if label.obj_class.name == class_name:
            label.geometry.draw(canvas, True)

def safe_ratio(num, denom):
    return (num / denom) if denom != 0 else 0

# rasterize all objects of class "cat"
mask_cat = np.full(ann_seg.img_size, False)
_render_labels_for_class_name(ann_seg.labels, "cat_seg", mask_cat)
draw_labeled_image(mask_cat, None)

filtered_ties = []
for label in ann_det.labels:
    if label.obj_class.name == "tie_det":
        temp_mask = np.full(ann_det.img_size, False)
        label.geometry.draw(temp_mask, True)

        # construct intersection and union bitmaps, calculate IoU metric
        intersection = mask_cat & temp_mask
        intersection_ratio = safe_ratio(intersection.sum(), temp_mask.sum())
        print(intersection_ratio)
        if intersection_ratio > 0.7:
            filtered_ties.append(label)
        #uncomment for debug visualization
        #draw_labeled_image(temp_mask, None)

0.0
0.0
0.9915681698411504
0.0
1.0

# replace labels in detection results
ann_det_filtered = ann_det.clone(labels=filtered_ties)

# Render the filtered results.
draw_labeled_image(img, ann_det_filtered)

Mission complete!