Species classification API demo

Conservation biologists increasingly leverage citizen scientists to both take wildlife photos and label species in those photos. However, human labeling only has so much capacity, and individual volunteers only have so much expertise. Consequently, there is a huge opportunity for computer vision to contribution to wildlife population research by automatically labeling handheld photos.

In this notebook, we demonstrate our public Species Classification API, which classifies plant and animal species in handheld photos (we treat camera trap photos as a separate problem... but ask us about that too!).

For information, see http://aka.ms/speciesclassification.

The API is hosted via the AI for Earth API Platform.

Download this notebook.

Imports and constants

In [1]:
import requests
import base64
import species_config as cfg
from IPython.display import Image
from IPython.core.display import HTML
from azure.storage.blob import ContainerClient

%autosave 0

# Constants related to the Species Classification API
CONTENT_TYPE_KEY = 'Content-Type'
CONTENT_TYPE = 'application/octet-stream'
AUTHORIZATION_HEADER = 'Ocp-Apim-Subscription-Key'
BASE_URL = 'https://aiforearth.azure-api.net/'
CLASSIFY_FORMAT = '{0}/species-classification/v{1}/predict?topK={2}&predictMode={3}'
API_VERSION = '2.0'
PREDICT_MODE = 'classifyOnly'

# Constants related to pulling demo images from blob storage
BLOB_ACCOUNT_URL = 'https://speciesrecognitionblobs.blob.core.windows.net/'
BLOB_CONTAINER_NAME = 'speciesrecognitionblobcontainer'

# Constants relate to pulling demo images from Bing Image search
BING_SEARCH_URL = 'https://api.cognitive.microsoft.com/bing/v7.0/images/search'

# Subscription keys hidden via config file
BING_SUBSCRIPTION_KEY = cfg.BING_SUBSCRIPTION_KEY
SUBSCRIPTION_KEY = cfg.SUBSCRIPTION_KEY

BING_IMAGE_LICENSE = 'public'
BING_IMAGE_TYPE = 'photo'
BING_SAFE_SEARCH = 'Strict'
MAX_NUM_SEARCH_IMAGES = 3

blob_container_client = ContainerClient(account_url=BLOB_ACCOUNT_URL, 
                                        container_name=BLOB_CONTAINER_NAME,
                                        credential=None)
Autosave disabled

Functions

In [2]:
def get_images(search_term=None):
    
    if search_term is None:
        return get_blob_images()
    else:
        return get_bing_images(search_term)
        
def get_bing_images(search_term):
    
    headers = {"Ocp-Apim-Subscription-Key" : BING_SUBSCRIPTION_KEY}
    params  = {"q" : search_term, "license" : BING_IMAGE_LICENSE, "imageType" : BING_IMAGE_TYPE, 
               "safeSearch" : BING_SAFE_SEARCH}
    response = requests.get(BING_SEARCH_URL, headers=headers, params=params)
    response.raise_for_status()
    search_results = response.json()

    images_data = []
    count = 0
    
    for img in search_results["value"][:16]:
        
        if(count == MAX_NUM_SEARCH_IMAGES): break
            
        image_data = requests.get(img["thumbnailUrl"])
        image_data.raise_for_status()
        images_data.append({"url": img["thumbnailUrl"], "name" : search_term, "data": image_data.content})
        count += 1
     
    return images_data   
  
def get_blob_images():
    
    images_data = []
    generator = blob_container_client.list_blobs()
    
    for blob in generator:                  
        blob_client = blob_container_client.get_blob_client(blob.name)
        blob_content = blob_client.download_blob().readall()
        imagebase64 = base64.b64encode(blob_content).decode('utf-8')
        images_data.append({"url": "data:image/png;base64," + str(imagebase64), "name" : blob.name, "data": blob_content})
        
    return images_data   

def get_images_html_string(image_list):
    
    html_string = ""
    for i in range(len(image_list)):
        html_string += "<img class='image' src='"+ image_list[i]['url'] + "'/>"
    return html_string

def display_raw_images(images): 
    
    html_string = "<style>" \
                  "* { box-sizing: border-box;} " \
                  " .column {float: left;width: 50%;padding: 10px; overflow:visible} " \
                  " .column > img {height:50%}" \
                  " .row:after {content: "";display: table;clear: both;} " \
                  "</style>"

    html_string += '<div class="row"><div class="column">' 

    #get images in even index
    image_list = images[::2]
    html_string += get_images_html_string(image_list)  
    html_string += "</div>"
    
    html_string += '<div class="column">'
    
    #get images in odd index
    image_list = images[1::2]
    html_string += get_images_html_string(image_list)
    html_string += "</div></div>"

    display(HTML(html_string))  

def display_single_image(image): 
    
    html_string = "<style>" \
                  "div.output_subarea{overflow-x:hidden !important}" \
                  ".container{margin:0 auto}" \
                  ".single_image{width:50%} " \
                  "</style>"
                
    html_string += "<div class='container'>" \
                       "<div class='image_container'>" \
                           "<img class='single_image' src='"+ image['url'] + "'/>"\
                       "</div>" \
                   "</div>"
 
   
    display(HTML(html_string))  
    
def display_classification_results(species, species_common, progress, is_first_item): 
    
    html_string = ""
    
    if(is_first_item):
        
        html_string = "<style>" \
                      ".progress-container {margin:0 auto; min-height: 25px;margin:0;width:100%; margin-top:10px}" \
                      ".progress-bar{background-color:#ffc107; padding:3px}" \
                      ".progress-text{color:black; margin-top:5px;} " \
                      ".species .species-common {" \
                      " color:black !important; font-family:'Helvetica Neue',Helvetica,Arial,sans-serif;" \
                      " font-size:14px;line-height:20px;}" \
                      "</style>"
                            
    progress = progress + "%"
    style = "width:" + progress + ""
    bing_search_link = 'https://bing.com/images/search?q=' + species
    
    html_string += "<a style='color:black;' class='species' href='" + bing_search_link +"' target='_blank'>" + species + "</a>" \
                   "<span class='species-common'>  ( " + species_common + " ) </span>" \
                   "<div class='progress progress-container'>" \
                   "<div class='progress-bar' style='" + style + "' >" \
                   "<span class='progress-text'>" + progress + "</span></div></div>" \
    
    display(HTML(html_string))
    
def build_classify_url(topK=5, base_url=BASE_URL, version=API_VERSION, predictMode=PREDICT_MODE):
    
    return CLASSIFY_FORMAT.format(base_url, version, topK, predictMode)

def get_api_headers(content_type):
    
    return { CONTENT_TYPE_KEY: content_type, AUTHORIZATION_HEADER: SUBSCRIPTION_KEY }

def get_api_response(imgdata):
    
    url = build_classify_url()
    
    print('Running API...')

    r = requests.post(url, headers=get_api_headers(CONTENT_TYPE), data=imgdata) 
    
    if(r.status_code != 200):
        return r.json(), True
    
    print('...done')
    
    return r.json(), False

def classify_and_display_results(image_data):
    
    result = get_api_response(image_data['data'])
        
    if(result == None):
        
        print ("Error occured while calling API...Please try again")
        return
    
    display_single_image(image_data)

    predictions = result[0]['predictions']
    
    is_first_item = True 
    
    for item in predictions:
        
        species = item['species']
        species_common = item['species_common']
        prob = round(item['confidence'], 2)
        
        display_classification_results(species, species_common,  str(prob), is_first_item)   
        is_first_item = False

Retrieve and display images

In [6]:
# To pull images from Bing...
images = get_images('african penguin')

# To pull demo images from Blob storage
# images = get_blob_images()

display_raw_images(images)

Call API and show results

In [7]:
# Enter a number between 0 and MAX_NUM_SEARCH_IMAGES-1 (i.e., between 0 and 2)
classify_and_display_results(images[0])

# To explore or debug the result...
# result = get_api_response(images[0]['data'])
# print(result)
Running API...
...done
Spheniscus demersus ( African Penguin )
55.58%
Spheniscus humboldti ( humboldt penguin )
30.78%
Pygoscelis antarcticus ( chinstrap penguin )
0.23%
Eudyptes chrysolophus ( macaroni penguin )
0.18%
Pygoscelis papua ( gentoo penguin )
0.18%