Detecting Labels

Label Detection detects broad sets of categories within an image, ranging from modes of transportation to animals.

This section demonstrates a few ways to detect labels in an image.

Detecting Labels in a Local Image

Here is an example of detecting labels in a local image file:

Protocol

Refer to the images:annotate API endpoint for complete details.

To perform label detection, make a POST request and provide the appropriate request body:

POST https://vision.googleapis.com/v1/images:annotate?key=YOUR_API_KEY
{
  "requests": [
    {
      "images": {
        "content": "/9j/7QBEUGhvdG9zaG9...base64-encoded-image-content...fXNWzvDEeYxxxzj/Coa6Bax//Z"
      },
      "features": [
        {
          "type": "LABEL_DETECTION"
        }
      ]
    }
  ]
}

See the AnnotateImageRequest reference documentation for more information on configuring the request body.

If the request is successful, the server returns a 200 OK HTTP status code and the response in JSON format:

{
  "responses": [
    {
      "labelAnnotations": [
        {
          "mid": "/m/01yrx",
          "description": "cat",
          "score": 0.92562944
        },
        {
          "mid": "/m/04rky",
          "description": "mammal",
          "score": 0.90815818
        },
        {
          "mid": "/m/01l7qd",
          "description": "whiskers",
          "score": 0.79939437
        },
        {
          "mid": "/m/07k6w8",
          "description": "small to medium sized cats",
          "score": 0.66373962
        },
        {
          "mid": "/m/0307l",
          "description": "cat like mammal",
          "score": 0.65950978
        }
      ]
    }
  ]
}

C#

For more on installing and creating a Cloud Vision API client, refer to Cloud Vision API Client Libraries.

private static object DetectLabels(string filePath)
{
    var client = ImageAnnotatorClient.Create();
    var image = Image.FromFile(filePath);
    var response = client.DetectLabels(image);
    foreach (var annotation in response)
    {
        if (annotation.Description != null)
            Console.WriteLine(annotation.Description);
    }
    return 0;
}

Go

For more on installing and creating a Cloud Vision API client, refer to Cloud Vision API Client Libraries.

// detectLabels gets labels from the Vision API for an image at the given file path.
func detectLabels(w io.Writer, file string) error {
	ctx := context.Background()

	client, err := vision.NewClient(ctx)
	if err != nil {
		return err
	}

	f, err := os.Open(file)
	if err != nil {
		return err
	}
	defer f.Close()

	image, err := vision.NewImageFromReader(f)
	if err != nil {
		return err
	}
	annotations, err := client.DetectLabels(ctx, image, 10)
	if err != nil {
		return err
	}

	if len(annotations) == 0 {
		fmt.Fprintln(w, "No labels found.")
	} else {
		fmt.Fprintln(w, "Labels:")
		for _, annotation := range annotations {
			fmt.Fprintln(w, annotation.Description)
		}
	}

	return nil
}

Java

For more on installing and creating a Cloud Vision API client, refer to Cloud Vision API Client Libraries.

public static void detectLabels(String filePath, PrintStream out) throws IOException {
  List<AnnotateImageRequest> requests = new ArrayList<>();

  ByteString imgBytes = ByteString.readFrom(new FileInputStream(filePath));

  Image img = Image.newBuilder().setContent(imgBytes).build();
  Feature feat = Feature.newBuilder().setType(Type.LABEL_DETECTION).build();
  AnnotateImageRequest request = AnnotateImageRequest.newBuilder()
      .addFeatures(feat)
      .setImage(img)
      .build();
  requests.add(request);

  BatchAnnotateImagesResponse response = visionApi.batchAnnotateImages(requests);
  List<AnnotateImageResponse> responses = response.getResponsesList();

  for (AnnotateImageResponse res : responses) {
    if (res.hasError()) {
      out.printf("Error: %s\n", res.getError().getMessage());
      return;
    }

    // For full list of available annotations, see http://g.co/cloud/vision/docs
    for (EntityAnnotation annotation : res.getLabelAnnotationsList()) {
      annotation.getAllFields().forEach((k, v)->out.printf("%s : %s\n", k, v.toString()));
    }
  }
}

Node.js

For more on installing and creating a Cloud Vision API client, refer to Cloud Vision API Client Libraries.

// Imports the Google Cloud client library
const Vision = require('@google-cloud/vision');

// Instantiates a client
const vision = Vision();

// The path to the local image file, e.g. "/path/to/image.png"
// const fileName = '/path/to/image.png';

// Performs label detection on the local file
vision.detectLabels(fileName)
  .then((results) => {
    const labels = results[0];

    console.log('Labels:');
    labels.forEach((label) => console.log(label));
  });

PHP

For more on installing and creating a Cloud Vision API client, refer to Cloud Vision API Client Libraries.

use Google\Cloud\Vision\VisionClient;


// $projectId = 'YOUR_PROJECT_ID';
// $path = 'path/to/your/image.jpg'

$vision = new VisionClient([
    'projectId' => $projectId,
]);

$image = $vision->image(file_get_contents($path), ['LABEL_DETECTION']);
$result = $vision->annotate($image);

print("LABELS:\n");
foreach ($result->labels() as $label) {
    print($label->description() . PHP_EOL);
}

Python

For more on installing and creating a Cloud Vision API client, refer to Cloud Vision API Client Libraries.

def detect_labels(path):
    """Detects labels in the file."""
    vision_client = vision.Client()

    with io.open(path, 'rb') as image_file:
        content = image_file.read()

    image = vision_client.image(content=content)

    labels = image.detect_labels()

    print('Labels:')
    for label in labels:
        print(label.description)

Detecting Labels in a Remote Image

For your convenience, the Cloud Vision API can perform label detection directly on an image file located in Google Cloud Storage without the need to send the contents of the image file in the body of your request.

Here is an example of performing label detection on a file located in Cloud Storage.

Protocol

Refer to the images:annotate API endpoint for complete details.

To perform label detection, make a POST request and provide the appropriate request body:

POST https://vision.googleapis.com/v1/images:annotate?key=YOUR_API_KEY
{
  "requests": [
    {
      "images": {
        "source": {
          "gcsImageUri": "gs://YOUR_BUCKET_NAME/YOUR_FILE_NAME"
        }
      },
      "features": [
        {
          "type": "LABEL_DETECTION"
        }
      ]
    }
  ]
}

See the AnnotateImageRequest reference documentation for more information on configuring the request body.

If the request is successful, the server returns a 200 OK HTTP status code and the response in JSON format:

{
  "responses": [
    {
      "labelAnnotations": [
        {
          "mid": "/m/01yrx",
          "description": "cat",
          "score": 0.92562944
        },
        {
          "mid": "/m/04rky",
          "description": "mammal",
          "score": 0.90815818
        },
        {
          "mid": "/m/01l7qd",
          "description": "whiskers",
          "score": 0.79939437
        },
        {
          "mid": "/m/07k6w8",
          "description": "small to medium sized cats",
          "score": 0.66373962
        },
        {
          "mid": "/m/0307l",
          "description": "cat like mammal",
          "score": 0.65950978
        }
      ]
    }
  ]
}

C#

For more on installing and creating a Cloud Vision API client, refer to Cloud Vision API Client Libraries.

private static object DetectLabels(string bucketName, string objectName)
{
    var client = ImageAnnotatorClient.Create();
    var image = Image.FromStorageUri($"gs://{bucketName}/{objectName}");
    var response = client.DetectLabels(image);
    foreach (var annotation in response)
    {
        if (annotation.Description != null)
            Console.WriteLine(annotation.Description);
    }
    return 0;
}

Go

For more on installing and creating a Cloud Vision API client, refer to Cloud Vision API Client Libraries.

// detectLabels gets labels from the Vision API for an image at the given file path.
func detectLabelsGCS(w io.Writer, file string) error {
	ctx := context.Background()

	client, err := vision.NewClient(ctx)
	if err != nil {
		return err
	}

	image := vision.NewImageFromGCS(file)
	annotations, err := client.DetectLabels(ctx, image, 10)
	if err != nil {
		return err
	}

	if len(annotations) == 0 {
		fmt.Fprintln(w, "No labels found.")
	} else {
		fmt.Fprintln(w, "Labels:")
		for _, annotation := range annotations {
			fmt.Fprintln(w, annotation.Description)
		}
	}

	return nil
}

Node.js

For more on installing and creating a Cloud Vision API client, refer to Cloud Vision API Client Libraries.

// Imports the Google Cloud client libraries
const Storage = require('@google-cloud/storage');
const Vision = require('@google-cloud/vision');

// Instantiates clients
const storage = Storage();
const vision = Vision();

// The name of the bucket where the file resides, e.g. "my-bucket"
// const bucketName = 'my-bucket';

// The path to the file within the bucket, e.g. "path/to/image.png"
// const fileName = 'path/to/image.png';

// Performs label detection on the remote file
vision.detectLabels(storage.bucket(bucketName).file(fileName))
  .then((results) => {
    const labels = results[0];

    console.log('Labels:');
    labels.forEach((label) => console.log(label));
  });

PHP

For more on installing and creating a Cloud Vision API client, refer to Cloud Vision API Client Libraries.

use Google\Cloud\ServiceBuilder;

// $projectId = 'YOUR_PROJECT_ID';
// $bucketName = 'your-bucket-name'
// $objectName = 'your-object-name'

$builder = new ServiceBuilder([
    'projectId' => $projectId,
]);
$vision = $builder->vision();
$storage = $builder->storage();

// fetch the storage object and annotate the image
$object = $storage->bucket($bucketName)->object($objectName);
$image = $vision->image($object, ['LABEL_DETECTION']);
$result = $vision->annotate($image);

// print the response
print("LABELS:\n");
foreach ($result->labels() as $label) {
    print($label->description() . PHP_EOL);
}

Python

For more on installing and creating a Cloud Vision API client, refer to Cloud Vision API Client Libraries.

def detect_labels_cloud_storage(uri):
    """Detects labels in the file located in Google Cloud Storage."""
    vision_client = vision.Client()
    image = vision_client.image(source_uri=uri)

    labels = image.detect_labels()

    print('Labels:')
    for label in labels:
        print(label.description)

Send feedback about...

Google Cloud Vision API Documentation