Supervisely Tutorial #10

In this tutorial we will show how to check that files are already uploaded to the server and how to upload only the new ones

Imports

[21]:
import os
import supervisely_lib as sly
[31]:
import pprint
[52]:
from datetime import datetime

Initialize API access with your credentials

[22]:
address = os.environ['SERVER_ADDRESS']
token = os.environ['API_TOKEN']

print("Server address: ", address)
#print("Your API token: ", token)
Server address:  https://app.supervise.ly/

Initialize the API access object

[23]:
api = sly.Api(address, token)

Script parameters

[29]:
team_name = "max"
workspace_name = "cats and dogs"
project_name = "project_x"

target_dir = 'images_to_upload'

Verify parameters

[25]:
team = api.team.get_info_by_name(team_name)
if team is None:
    raise RuntimeError("Team {!r} not found".format(team_name))

workspace = api.workspace.get_info_by_name(team.id, workspace_name)
if workspace is None:
    raise RuntimeError("Workspace {!r} not found".format(workspace_name))

print("Team: id={}, name={!r}".format(team.id, team.name))
print("Workspace: id={}, name={!r}".format(workspace.id, workspace.name))
Team: id=600, name='max'
Workspace: id=26846, name='cats and dogs'

Find project or create the new one

[26]:
project = api.project.get_info_by_name(workspace.id, project_name)
if project is None:
    project = api.project.create(workspace.id, project_name, 'project for uploading new images', change_name_if_conflict=False)
print("Project: id={}, name={!r}".format(project.id, project.name))
Project: id=60333, name='project_x'

Get paths of all images in a directory

[45]:
images_paths = sly.fs.list_files('images_to_upload')
pprint.pprint(images_paths)
['images_to_upload/adorable-animal-blur-cat-617278.jpg',
 'images_to_upload/image_02.jpeg',
 'images_to_upload/grey-and-white-short-fur-cat-104827.jpg',
 'images_to_upload/image_02_y.jpeg',
 'images_to_upload/image_01.jpeg',
 'images_to_upload/adult-black-pug-1851164.jpg',
 'images_to_upload/image_01_x.jpeg',
 'images_to_upload/IMG_0315.jpg',
 'images_to_upload/image_03.jpeg',
 'images_to_upload/adorable-animal-breed-canine-434090.jpg']

Calculate hashes for all images paths

[46]:
hash_to_image = {}
images_hashes = []

for idx, item in enumerate(images_paths):
    item_hash = sly.fs.get_file_hash(item)
    images_hashes.append(item_hash)
    hash_to_image[item_hash] = item
#pprint.pprint(images_hashes)
pprint.pprint(hash_to_image)
{'7bssCHBSspbfU5IXyAy89NE4ZtT13seIEyP46CvHw3I=': 'images_to_upload/image_03.jpeg',
 'QZXPQjcsXMuSSxHtK00qlDys+zo4o08tlNbwF8po1S8=': 'images_to_upload/image_02_y.jpeg',
 'RjO0Jfb0mKyoChaimr7yfAx5L9kQoWBr3Vd3vza/+Vc=': 'images_to_upload/image_01.jpeg',
 'TeHa9M1gQTz2F1hh4oQL7LWRaiWTEVP/gM+Ae9mTZaU=': 'images_to_upload/image_02.jpeg',
 'Y2H/xhCs61l8nwRr2howW71PJFtS/GGaRac3tOkxc/w=': 'images_to_upload/adorable-animal-blur-cat-617278.jpg',
 'Y9+aWgu2XXF3hdPV2nZauQoyLZxajAGAGPt0AZSEMqw=': 'images_to_upload/image_01_x.jpeg',
 'gNbD/201JoxZe2xGcxuAA6DEUcBIwQRGXBGehEYs8ZY=': 'images_to_upload/adorable-animal-breed-canine-434090.jpg',
 'om8cqJRQTV2ir/hh8/B7NkQvabuj8CUNcSuGeW7i8WE=': 'images_to_upload/grey-and-white-short-fur-cat-104827.jpg',
 'pB6T7Fn262CNnevD0YQFtfcJg4m53LZzLx8S9TmJ5No=': 'images_to_upload/adult-black-pug-1851164.jpg',
 'z0Muc7aDmSS6GuC+xrx9VBOJEfbKK5i/fCSDcsJFcaI=': 'images_to_upload/IMG_0315.jpg'}

Get hashes that are already on server

[47]:
remote_hashes = api.image.check_existing_hashes(images_hashes)
already_uploaded_images = {hh: hash_to_image[hh] for hh in remote_hashes}
pprint.pprint(already_uploaded_images)
{'7bssCHBSspbfU5IXyAy89NE4ZtT13seIEyP46CvHw3I=': 'images_to_upload/image_03.jpeg',
 'QZXPQjcsXMuSSxHtK00qlDys+zo4o08tlNbwF8po1S8=': 'images_to_upload/image_02_y.jpeg',
 'RjO0Jfb0mKyoChaimr7yfAx5L9kQoWBr3Vd3vza/+Vc=': 'images_to_upload/image_01.jpeg',
 'TeHa9M1gQTz2F1hh4oQL7LWRaiWTEVP/gM+Ae9mTZaU=': 'images_to_upload/image_02.jpeg',
 'Y9+aWgu2XXF3hdPV2nZauQoyLZxajAGAGPt0AZSEMqw=': 'images_to_upload/image_01_x.jpeg',
 'z0Muc7aDmSS6GuC+xrx9VBOJEfbKK5i/fCSDcsJFcaI=': 'images_to_upload/IMG_0315.jpg'}
[49]:
# Find hashes of images that should be uploaded
[50]:
# new_images - images that don't exist in the server storage
new_hashes = set(images_hashes) - set(remote_hashes)
new_images = {hh: hash_to_image[hh] for hh in new_hashes}
pprint.pprint(new_images)
{'Y2H/xhCs61l8nwRr2howW71PJFtS/GGaRac3tOkxc/w=': 'images_to_upload/adorable-animal-blur-cat-617278.jpg',
 'gNbD/201JoxZe2xGcxuAA6DEUcBIwQRGXBGehEYs8ZY=': 'images_to_upload/adorable-animal-breed-canine-434090.jpg',
 'om8cqJRQTV2ir/hh8/B7NkQvabuj8CUNcSuGeW7i8WE=': 'images_to_upload/grey-and-white-short-fur-cat-104827.jpg',
 'pB6T7Fn262CNnevD0YQFtfcJg4m53LZzLx8S9TmJ5No=': 'images_to_upload/adult-black-pug-1851164.jpg'}

Create dataset for new images

[73]:
new_dataset_name = "batch X: {}".format(datetime.now().strftime("%d-%m-%Y %H:%M:%S"))
dataset = api.dataset.create(project.id, new_dataset_name, description="new images", change_name_if_conflict=False)
print("Dataset: id={}, name={!r}".format(dataset.id, new_dataset_name))#dataset.name))
Dataset: id=275583, name='batch X: 13-02-2020 08:05:25'

Upload images to the created dataset

[65]:
paths_to_upload = list(new_images.values())
pprint.pprint(paths_to_upload)
['images_to_upload/grey-and-white-short-fur-cat-104827.jpg',
 'images_to_upload/adult-black-pug-1851164.jpg',
 'images_to_upload/adorable-animal-breed-canine-434090.jpg',
 'images_to_upload/adorable-animal-blur-cat-617278.jpg']
[67]:
names_to_upload = [sly.fs.get_file_name_with_ext(path)  for path in paths_to_upload]
pprint.pprint(names_to_upload)
['grey-and-white-short-fur-cat-104827.jpg',
 'adult-black-pug-1851164.jpg',
 'adorable-animal-breed-canine-434090.jpg',
 'adorable-animal-blur-cat-617278.jpg']
[62]:
uploaded_images_infos = api.image.upload_paths(dataset.id, names_to_upload, paths_to_upload)
[71]:
print("uploaded images:")
for info in uploaded_images_infos:
    print("id={}, name={!r}".format(info.id, info.name))
uploaded images:
id=131702135, name='grey-and-white-short-fur-cat-104827.jpg'
id=131702136, name='adult-black-pug-1851164.jpg'
id=131702137, name='adorable-animal-breed-canine-434090.jpg'
id=131702138, name='adorable-animal-blur-cat-617278.jpg'

Done!

[ ]: