Species classification API demo¶
Conservation biologists increasingly leverage citizen scientists to both take wildlife photos and label species in those photos. However, human labeling only has so much capacity, and individual volunteers only have so much expertise. Consequently, there is a huge opportunity for computer vision to contribution to wildlife population research by automatically labeling handheld photos.
In this notebook, we demonstrate our public Species Classification API, which classifies plant and animal species in handheld photos (we treat camera trap photos as a separate problem... but ask us about that too!).
For information, see http://aka.ms/speciesclassification.
The API is hosted via the AI for Earth API Platform.
Imports and constants¶
In [1]:
import requests
import base64
import species_config as cfg
from IPython.display import Image
from IPython.core.display import HTML
from azure.storage.blob import ContainerClient
%autosave 0
# Constants related to the Species Classification API
CONTENT_TYPE_KEY = 'Content-Type'
CONTENT_TYPE = 'application/octet-stream'
AUTHORIZATION_HEADER = 'Ocp-Apim-Subscription-Key'
BASE_URL = 'https://aiforearth.azure-api.net/'
CLASSIFY_FORMAT = '{0}/species-classification/v{1}/predict?topK={2}&predictMode={3}'
API_VERSION = '2.0'
PREDICT_MODE = 'classifyOnly'
# Constants related to pulling demo images from blob storage
BLOB_ACCOUNT_URL = 'https://speciesrecognitionblobs.blob.core.windows.net/'
BLOB_CONTAINER_NAME = 'speciesrecognitionblobcontainer'
# Constants relate to pulling demo images from Bing Image search
BING_SEARCH_URL = 'https://api.cognitive.microsoft.com/bing/v7.0/images/search'
# Subscription keys hidden via config file
BING_SUBSCRIPTION_KEY = cfg.BING_SUBSCRIPTION_KEY
SUBSCRIPTION_KEY = cfg.SUBSCRIPTION_KEY
BING_IMAGE_LICENSE = 'public'
BING_IMAGE_TYPE = 'photo'
BING_SAFE_SEARCH = 'Strict'
MAX_NUM_SEARCH_IMAGES = 3
blob_container_client = ContainerClient(account_url=BLOB_ACCOUNT_URL,
container_name=BLOB_CONTAINER_NAME,
credential=None)
Functions¶
In [2]:
def get_images(search_term=None):
if search_term is None:
return get_blob_images()
else:
return get_bing_images(search_term)
def get_bing_images(search_term):
headers = {"Ocp-Apim-Subscription-Key" : BING_SUBSCRIPTION_KEY}
params = {"q" : search_term, "license" : BING_IMAGE_LICENSE, "imageType" : BING_IMAGE_TYPE,
"safeSearch" : BING_SAFE_SEARCH}
response = requests.get(BING_SEARCH_URL, headers=headers, params=params)
response.raise_for_status()
search_results = response.json()
images_data = []
count = 0
for img in search_results["value"][:16]:
if(count == MAX_NUM_SEARCH_IMAGES): break
image_data = requests.get(img["thumbnailUrl"])
image_data.raise_for_status()
images_data.append({"url": img["thumbnailUrl"], "name" : search_term, "data": image_data.content})
count += 1
return images_data
def get_blob_images():
images_data = []
generator = blob_container_client.list_blobs()
for blob in generator:
blob_client = blob_container_client.get_blob_client(blob.name)
blob_content = blob_client.download_blob().readall()
imagebase64 = base64.b64encode(blob_content).decode('utf-8')
images_data.append({"url": "data:image/png;base64," + str(imagebase64), "name" : blob.name, "data": blob_content})
return images_data
def get_images_html_string(image_list):
html_string = ""
for i in range(len(image_list)):
html_string += "<img class='image' src='"+ image_list[i]['url'] + "'/>"
return html_string
def display_raw_images(images):
html_string = "<style>" \
"* { box-sizing: border-box;} " \
" .column {float: left;width: 50%;padding: 10px; overflow:visible} " \
" .column > img {height:50%}" \
" .row:after {content: "";display: table;clear: both;} " \
"</style>"
html_string += '<div class="row"><div class="column">'
#get images in even index
image_list = images[::2]
html_string += get_images_html_string(image_list)
html_string += "</div>"
html_string += '<div class="column">'
#get images in odd index
image_list = images[1::2]
html_string += get_images_html_string(image_list)
html_string += "</div></div>"
display(HTML(html_string))
def display_single_image(image):
html_string = "<style>" \
"div.output_subarea{overflow-x:hidden !important}" \
".container{margin:0 auto}" \
".single_image{width:50%} " \
"</style>"
html_string += "<div class='container'>" \
"<div class='image_container'>" \
"<img class='single_image' src='"+ image['url'] + "'/>"\
"</div>" \
"</div>"
display(HTML(html_string))
def display_classification_results(species, species_common, progress, is_first_item):
html_string = ""
if(is_first_item):
html_string = "<style>" \
".progress-container {margin:0 auto; min-height: 25px;margin:0;width:100%; margin-top:10px}" \
".progress-bar{background-color:#ffc107; padding:3px}" \
".progress-text{color:black; margin-top:5px;} " \
".species .species-common {" \
" color:black !important; font-family:'Helvetica Neue',Helvetica,Arial,sans-serif;" \
" font-size:14px;line-height:20px;}" \
"</style>"
progress = progress + "%"
style = "width:" + progress + ""
bing_search_link = 'https://bing.com/images/search?q=' + species
html_string += "<a style='color:black;' class='species' href='" + bing_search_link +"' target='_blank'>" + species + "</a>" \
"<span class='species-common'> ( " + species_common + " ) </span>" \
"<div class='progress progress-container'>" \
"<div class='progress-bar' style='" + style + "' >" \
"<span class='progress-text'>" + progress + "</span></div></div>" \
display(HTML(html_string))
def build_classify_url(topK=5, base_url=BASE_URL, version=API_VERSION, predictMode=PREDICT_MODE):
return CLASSIFY_FORMAT.format(base_url, version, topK, predictMode)
def get_api_headers(content_type):
return { CONTENT_TYPE_KEY: content_type, AUTHORIZATION_HEADER: SUBSCRIPTION_KEY }
def get_api_response(imgdata):
url = build_classify_url()
print('Running API...')
r = requests.post(url, headers=get_api_headers(CONTENT_TYPE), data=imgdata)
if(r.status_code != 200):
return r.json(), True
print('...done')
return r.json(), False
def classify_and_display_results(image_data):
result = get_api_response(image_data['data'])
if(result == None):
print ("Error occured while calling API...Please try again")
return
display_single_image(image_data)
predictions = result[0]['predictions']
is_first_item = True
for item in predictions:
species = item['species']
species_common = item['species_common']
prob = round(item['confidence'], 2)
display_classification_results(species, species_common, str(prob), is_first_item)
is_first_item = False
Retrieve and display images¶
In [6]:
# To pull images from Bing...
images = get_images('african penguin')
# To pull demo images from Blob storage
# images = get_blob_images()
display_raw_images(images)
Call API and show results¶
In [7]:
# Enter a number between 0 and MAX_NUM_SEARCH_IMAGES-1 (i.e., between 0 and 2)
classify_and_display_results(images[0])
# To explore or debug the result...
# result = get_api_response(images[0]['data'])
# print(result)