1. Imports

[1]:
import os
import supervisely_lib as sly
import cv2
import numpy as np
[2]:
# PyPlot for drawing images in Jupyter.
%matplotlib inline
import matplotlib.pyplot as plt

def draw_labeled_image(img, ann, draw_fill=False):
    canvas_draw_contour = img.copy()
    if ann is not None:
        if draw_fill is True:
            ann.draw(canvas_draw_contour)
        else:
            ann.draw_contour(canvas_draw_contour, thickness=7)
    fig = plt.figure(figsize=(30, 30))
    fig.add_subplot(1, 2, 1)
    plt.imshow(img)
    if ann is not None:
        fig.add_subplot(1, 2, 2)
        plt.imshow(canvas_draw_contour)
    plt.show()

2. Initialize API Access with your Credentials

[3]:
address = os.environ['SERVER_ADDRESS']
token = os.environ['API_TOKEN']

print("Server address: ", address)
#print("Your API token: ", token)
Server address:  https://app.supervise.ly/

3. Initialize the API Access Object

[16]:
api = sly.Api(address, token)

#defince gpu device for deploying (hotfix - will be added to sdk in a next version)
def deploy_model_to_device(api, agent_id, model_id, device_id = 0):
    task_ids = api.model.get_deploy_tasks(model_id)
    if len(task_ids) == 0:
        task_id = api.task._deploy_model(agent_id, model_id, settings={'gpu_device': device_id})
    else:
        task_id = task_ids[0]
    api.task.wait(task_id, api.task.Status.DEPLOYED)
    return task_id

4. Script parameters

[6]:
team_name = "max"
workspace_name = "detection_segmentation_pipeline_example"

agent_name = "maxim_server"

model_detection_name = "YOLO v3 (COCO)"
model_segmentation_name = "DeepLab v3 plus (VOC2012)"

5. Verify Parameters

[7]:
team = api.team.get_info_by_name(team_name)
if team is None:
    raise RuntimeError("Team {!r} not found".format(team_name))

workspace = api.workspace.get_info_by_name(team.id, workspace_name)
if workspace is None:
    raise RuntimeError("Workspace {!r} not found".format(workspace_name))

print("Team: id={}, name={}".format(team.id, team.name))
print("Workspace: id={}, name={}".format(workspace.id, workspace.name))

model_detection = api.model.get_info_by_name(workspace.id, model_detection_name)
if model_detection is None:
    clone_task_id = api.model.clone_from_explore("Supervisely/Model Zoo/" + model_detection_name,
                                                 workspace.id,
                                                 model_detection_name)
    api.task.wait(clone_task_id, api.task.Status.FINISHED)
    model_detection = api.model.get_info_by_name(workspace.id, model_detection_name)

print("Model (detection): id={}, name={}".format(model_detection.id, model_detection.name))


model_segmentation = api.model.get_info_by_name(workspace.id, model_segmentation_name)
if model_segmentation is None:
    clone_task_id = api.model.clone_from_explore("Supervisely/Model Zoo/" + model_segmentation_name,
                                                 workspace.id,
                                                 model_segmentation_name)
    api.task.wait(clone_task_id, api.task.Status.FINISHED)
    model_segmentation = api.model.get_info_by_name(workspace.id, model_segmentation_name)

print("Model (segmentation): id={}, name={}".format(model_segmentation.id, model_segmentation.name))


agent = api.agent.get_info_by_name(team.id, agent_name)
if agent is None:
    raise RuntimeError("Agent {!r} not found".format(agent_name))
if agent.status is api.agent.Status.WAITING:
    raise RuntimeError("Agent {!r} is not running".format(agent_name))
print("Agent: id={}, name={}".format(agent.id, agent.name))
Team: id=600, name=max
Workspace: id=26686, name=detection_segmentation_pipeline_example
Model (detection): id=13912, name=YOLO v3 (COCO)
Model (segmentation): id=13913, name=DeepLab v3 plus (VOC2012)
Agent: id=377, name=maxim_server

6. Deploy both Detection and Segmentataion Models (if not deployed before)

[18]:
#detploy_task_id_detection = deploy_model_to_device(api, agent.id, model_detection.id, device_id=0)
[19]:
#detploy_task_id_detection = api.task.deploy_model(agent.id, model_detection.id)
#detploy_task_id_segmentation = api.task.deploy_model(agent.id, model_segmentation.id)
detploy_task_id_detection = deploy_model_to_device(api, agent.id, model_detection.id, device_id=0)
detploy_task_id_segmentation = deploy_model_to_device(api, agent.id, model_segmentation.id, device_id=1)

print("Debug info:")
print("Deploy task_id for detection model: {}".format(detploy_task_id_detection))
print("Deploy task_id for segmentation model: {}".format(detploy_task_id_segmentation))
Debug info:
Deploy task_id for detection model: 135490
Deploy task_id for segmentation model: 135515

7. Get Models Ouptut Classes (+ check that models are deployed correctly)

[20]:
# dummy config is used to slightly rename model output class
dummy_detection_config = {
    "model_tags": {
      "add_suffix": "_det",
      "save_names": "__all__"
    },
    "model_classes": {
      "add_suffix": "_det",
      "save_classes": "__all__"
    }
}
[21]:
meta_json = api.model.get_output_meta(model_detection.id, inference_mode=dummy_detection_config)
model_detection_meta = sly.ProjectMeta.from_json(meta_json)
print("Output classes (detection model)")
print(model_detection_meta)
Output classes (detection model)
ProjectMeta:
Object Classes
+--------------------+-----------+-----------------+
|        Name        |   Shape   |      Color      |
+--------------------+-----------+-----------------+
|     person_det     | Rectangle | [146, 208, 134] |
|    bicycle_det     | Rectangle | [116, 127, 233] |
|      car_det       | Rectangle | [233, 189, 207] |
|   motorbike_det    | Rectangle | [111, 190, 245] |
|   aeroplane_det    | Rectangle |  [92, 126, 104] |
|      bus_det       | Rectangle | [212, 239, 134] |
|     train_det      | Rectangle | [140, 180, 183] |
|     truck_det      | Rectangle | [231, 222, 180] |
|      boat_det      | Rectangle |  [213, 86, 211] |
| traffic light_det  | Rectangle | [137, 206, 104] |
|  fire hydrant_det  | Rectangle | [194, 160, 183] |
|   stop sign_det    | Rectangle | [131, 156, 191] |
| parking meter_det  | Rectangle |  [96, 163, 96]  |
|     bench_det      | Rectangle | [232, 202, 225] |
|      bird_det      | Rectangle | [253, 192, 185] |
|      cat_det       | Rectangle | [109, 250, 167] |
|      dog_det       | Rectangle | [214, 227, 223] |
|     horse_det      | Rectangle | [215, 164, 135] |
|     sheep_det      | Rectangle | [208, 112, 181] |
|      cow_det       | Rectangle | [100, 211, 137] |
|    elephant_det    | Rectangle | [178, 189, 166] |
|      bear_det      | Rectangle | [117, 129, 129] |
|     zebra_det      | Rectangle | [160, 207, 150] |
|    giraffe_det     | Rectangle |  [91, 155, 186] |
|    backpack_det    | Rectangle | [228, 217, 157] |
|    umbrella_det    | Rectangle | [136, 169, 229] |
|    handbag_det     | Rectangle | [100, 181, 251] |
|      tie_det       | Rectangle |  [95, 201, 229] |
|    suitcase_det    | Rectangle | [182, 227, 200] |
|    frisbee_det     | Rectangle |  [102, 168, 94] |
|      skis_det      | Rectangle |  [116, 166, 87] |
|   snowboard_det    | Rectangle | [231, 152, 160] |
|  sports ball_det   | Rectangle | [253, 239, 246] |
|      kite_det      | Rectangle | [107, 158, 211] |
|  baseball bat_det  | Rectangle | [123, 100, 233] |
| baseball glove_det | Rectangle | [225, 126, 184] |
|   skateboard_det   | Rectangle | [216, 171, 174] |
|   surfboard_det    | Rectangle | [144, 216, 188] |
| tennis racket_det  | Rectangle | [182, 156, 250] |
|     bottle_det     | Rectangle | [230, 209, 159] |
|   wine glass_det   | Rectangle |  [183, 254, 98] |
|      cup_det       | Rectangle | [215, 243, 120] |
|      fork_det      | Rectangle | [148, 247, 126] |
|     knife_det      | Rectangle | [175, 100, 183] |
|     spoon_det      | Rectangle | [245, 171, 198] |
|      bowl_det      | Rectangle |  [96, 216, 100] |
|     banana_det     | Rectangle | [123, 135, 104] |
|     apple_det      | Rectangle | [209, 147, 152] |
|    sandwich_det    | Rectangle | [211, 209, 131] |
|     orange_det     | Rectangle | [115, 132, 226] |
|    broccoli_det    | Rectangle | [108, 234, 113] |
|     carrot_det     | Rectangle | [136, 121, 238] |
|    hot dog_det     | Rectangle |  [101, 87, 230] |
|     pizza_det      | Rectangle | [128, 233, 240] |
|     donut_det      | Rectangle | [217, 254, 187] |
|      cake_det      | Rectangle | [118, 198, 160] |
|     chair_det      | Rectangle |  [213, 96, 120] |
|      sofa_det      | Rectangle | [240, 145, 177] |
|  pottedplant_det   | Rectangle | [238, 211, 241] |
|      bed_det       | Rectangle | [186, 198, 157] |
|  diningtable_det   | Rectangle | [200, 219, 127] |
|     toilet_det     | Rectangle | [175, 247, 104] |
|   tvmonitor_det    | Rectangle | [121, 243, 189] |
|     laptop_det     | Rectangle | [126, 239, 127] |
|     mouse_det      | Rectangle | [171, 138, 156] |
|     remote_det     | Rectangle | [251, 104, 192] |
|    keyboard_det    | Rectangle | [128, 202, 223] |
|   cell phone_det   | Rectangle | [108, 201, 122] |
|   microwave_det    | Rectangle | [248, 218, 143] |
|      oven_det      | Rectangle | [178, 158, 127] |
|    toaster_det     | Rectangle |  [120, 119, 97] |
|      sink_det      | Rectangle | [216, 216, 127] |
|  refrigerator_det  | Rectangle |  [94, 129, 108] |
|      book_det      | Rectangle | [178, 127, 145] |
|     clock_det      | Rectangle |  [147, 86, 212] |
|      vase_det      | Rectangle | [136, 159, 104] |
|    scissors_det    | Rectangle | [183, 114, 216] |
|   teddy bear_det   | Rectangle |  [99, 174, 203] |
|   hair drier_det   | Rectangle | [148, 189, 224] |
|   toothbrush_det   | Rectangle | [164, 225, 168] |
+--------------------+-----------+-----------------+
Tags
+----------------+------------+-----------------+
|      Name      | Value type | Possible values |
+----------------+------------+-----------------+
| confidence_det | any_number |       None      |
+----------------+------------+-----------------+

[22]:
# dummy config is used to slightly rename model output class
dummy_segmentation_config = {
    "model_tags": {
      "add_suffix": "_seg",
      "save_names": "__all__"
    },
    "model_classes": {
      "add_suffix": "_seg",
      "save_classes": "__all__"
    }
}
[23]:
meta_json = api.model.get_output_meta(model_segmentation.id, inference_mode=dummy_segmentation_config)
model_segmentation_meta = sly.ProjectMeta.from_json(meta_json)
print("Output classes (segmentation model)")
print(model_segmentation_meta)
Output classes (segmentation model)
ProjectMeta:
Object Classes
+------------------+--------+-----------------+
|       Name       | Shape  |      Color      |
+------------------+--------+-----------------+
|      bg_seg      | Bitmap | [182, 182, 159] |
|  aeroplane_seg   | Bitmap |  [255, 0, 153]  |
|   bicycle_seg    | Bitmap |  [191, 0, 115]  |
|     bird_seg     | Bitmap |   [128, 0, 77]  |
|     boat_seg     | Bitmap |  [179, 0, 255]  |
|    bottle_seg    | Bitmap |  [134, 0, 191]  |
|     bus_seg      | Bitmap |   [90, 0, 128]  |
|     car_seg      | Bitmap |   [9, 0, 255]   |
|     cat_seg      | Bitmap |   [6, 0, 191]   |
|    chair_seg     | Bitmap |   [4, 0, 128]   |
|     cow_seg      | Bitmap |  [0, 161, 255]  |
| diningtable_seg  | Bitmap |  [0, 121, 191]  |
|     dog_seg      | Bitmap |   [0, 81, 128]  |
|    horse_seg     | Bitmap |  [0, 255, 179]  |
|  motorbike_seg   | Bitmap |  [0, 191, 134]  |
|    person_seg    | Bitmap |   [0, 128, 90]  |
| potted-plant_seg | Bitmap |   [0, 255, 0]   |
|    sheep_seg     | Bitmap |   [0, 191, 0]   |
|     sofa_seg     | Bitmap |   [0, 128, 0]   |
|    train_seg     | Bitmap |  [170, 255, 0]  |
|  tv-monitor_seg  | Bitmap |  [127, 191, 0]  |
+------------------+--------+-----------------+
Tags
+------+------------+-----------------+
| Name | Value type | Possible values |
+------+------------+-----------------+
+------+------------+-----------------+

[24]:
img = sly.image.read('collage_03.jpg')
[25]:
draw_labeled_image(img, None)
../../../../../../_images/repo_help_jupyterlab_scripts_src_tutorials_09_detection_segmentation_pipeline_multi_gpu_detection_segmentation_pipeline_20_0.png

8. Apply Detection Model (sliding window)

In current example image has big resolution, and the models were trained with the smaller one.

Thus we are going to aply models in a sliding window manner. Learn more here:

https://supervise.ly/explore/notebooks/guide-04-neural-network-inference-20/overview

https://supervise.ly/explore/notebooks/guide-06-neural-network-inference-modes-215/overview

[26]:
detection_sliding_window_config = {
    "name": "sliding_window_det",

    # Sliding window parameters.

    # Width and height in pixels.
    # Cannot be larger than the original image.
    "window": {
      "width": 1000,
      "height": 1000,
    },

    # Minimum overlap for each dimension. The last
    # window in every dimension may have higher overlap
    # with the previous one if necessary to fit the whole
    # window within the original image.
    "min_overlap": {
      "x": 200,
      "y": 200,
    },

    # Whether to save each sliding window instance as a
    # bounding box rectangle.
    "save": False,

    # If saving the sliding window bounding boxes, which
    # class name to use.
    "class_name": 'sliding_window_bbox',

    "nms_after": {

      # Whether to run non-maximum suppression after accumulating
      # all the detection results from the sliding windows.
      "enable": True,

      # Intersection over union threshold above which the same-class
      # detection labels are considered to be significantly inersected
      # for non-maximum suppression.
      "iou_threshold": 0.2,

      # Tag name from which to read detection confidence by which we
      # rank the detections. This tag must be added by the model to
      # every detection label.
      "confidence_tag_name": "confidence"
    },

    # Class renaming and filtering settings.
    # See "Full image inference" example for details.
    "model_classes": {
      "add_suffix": "_det",
      "save_classes": ["tie"]
    },

    "model_tags": {
      "add_suffix": "_det",
      "save_names": "__all__"
    }
}
[27]:
ann_json = api.model.inference(model_detection.id, img, mode=detection_sliding_window_config)
print(ann_json)
{'description': '', 'size': {'height': 5000, 'width': 3137}, 'tags': [], 'objects': [{'classTitle': 'tie_det', 'description': '', 'tags': [{'name': 'confidence_det', 'value': 0.9625}], 'points': {'exterior': [[1766, 674], [1851, 990]], 'interior': []}}, {'classTitle': 'tie_det', 'description': '', 'tags': [{'name': 'confidence_det', 'value': 0.9773}], 'points': {'exterior': [[1966, 3737], [2350, 3922]], 'interior': []}}, {'classTitle': 'tie_det', 'description': '', 'tags': [{'name': 'confidence_det', 'value': 0.9807}], 'points': {'exterior': [[375, 2608], [733, 2866]], 'interior': []}}, {'classTitle': 'tie_det', 'description': '', 'tags': [{'name': 'confidence_det', 'value': 0.9827}], 'points': {'exterior': [[976, 908], [1050, 1086]], 'interior': []}}, {'classTitle': 'tie_det', 'description': '', 'tags': [{'name': 'confidence_det', 'value': 0.9957}], 'points': {'exterior': [[525, 4317], [774, 4807]], 'interior': []}}], 'success': True}
[28]:
# Deserialize the annotation using the model meta information that
# we received previously.
ann_det = sly.Annotation.from_json(ann_json, model_detection_meta)

# Render the inference results.
draw_labeled_image(img, ann_det)
../../../../../../_images/repo_help_jupyterlab_scripts_src_tutorials_09_detection_segmentation_pipeline_multi_gpu_detection_segmentation_pipeline_25_0.png

9. Apply Segmentation Model (sliding window)

[29]:
segmentation_sliding_window_config = {
    "name": "sliding_window",

    # Sliding window parameters.

    # Width and height in pixels.
    # Cannot be larger than the original image.
    "window": {
      "width": 2000,
      "height": 2000,
    },

    # Minimum overlap for each dimension. The last
    # window in every dimension may have higher overlap
    # with the previous one if necessary to fit the whole
    # window within the original image.
    "min_overlap": {
      "x": 400,
      "y": 400,
    },

    # Whether to save each sliding window instance as a
    # bounding box rectangle.
    #"save": False,

    # If saving the sliding window bounding boxes, which
    # class name to use.
    #"class_name": 'sliding_window_bbox',

    # Class renaming and filtering settings.
    # See "Full image inference" example for details.
    "model_classes": {
      "add_suffix": "_seg",
      "save_classes": ["person", "dog", "cat"]
    },

    #"model_tags": {
    #  "add_suffix": "",
    #  "save_names": "__all__"
    #}
}
[30]:
ann_json = api.model.inference(model_segmentation.id, img, mode=segmentation_sliding_window_config)
#print(ann_json)
[31]:
# Deserialize the annotation using the model meta information that
# we received previously.
ann_seg = sly.Annotation.from_json(ann_json, model_segmentation_meta)

# Render the inference results.
draw_labeled_image(img, ann_seg, draw_fill=True)
../../../../../../_images/repo_help_jupyterlab_scripts_src_tutorials_09_detection_segmentation_pipeline_multi_gpu_detection_segmentation_pipeline_29_0.png

10. Postprocessing Step: Keep only Ties that are on the Cats

[32]:
def _render_labels_for_class_name(labels, class_name, canvas):
    for label in labels:
        if label.obj_class.name == class_name:
            label.geometry.draw(canvas, True)

def safe_ratio(num, denom):
    return (num / denom) if denom != 0 else 0
[33]:
# rasterize all objects of class "cat"
mask_cat = np.full(ann_seg.img_size, False)
_render_labels_for_class_name(ann_seg.labels, "cat_seg", mask_cat)
draw_labeled_image(mask_cat, None)
../../../../../../_images/repo_help_jupyterlab_scripts_src_tutorials_09_detection_segmentation_pipeline_multi_gpu_detection_segmentation_pipeline_32_0.png
[34]:
filtered_ties = []
for label in ann_det.labels:
    if label.obj_class.name == "tie_det":
        temp_mask = np.full(ann_det.img_size, False)
        label.geometry.draw(temp_mask, True)

        # construct intersection and union bitmaps, calculate IoU metric
        intersection = mask_cat & temp_mask
        intersection_ratio = safe_ratio(intersection.sum(), temp_mask.sum())
        print(intersection_ratio)
        if intersection_ratio > 0.7:
            filtered_ties.append(label)
        #uncomment for debug visualization
        #draw_labeled_image(temp_mask, None)
0.0
0.0
0.9915681698411504
0.0
1.0
[35]:
# replace labels in detection results
ann_det_filtered = ann_det.clone(labels=filtered_ties)
[36]:
# Render the filtered results.
draw_labeled_image(img, ann_det_filtered)
../../../../../../_images/repo_help_jupyterlab_scripts_src_tutorials_09_detection_segmentation_pipeline_multi_gpu_detection_segmentation_pipeline_35_0.png