Inspect a string for sensitive data, omitting overlapping matches on domain and email

Omit matches on domain names that are part of email addresses in a DOMAIN_NAME detector scan.

Explore further

For detailed documentation that includes this code sample, see the following:

• Modifying infoType detectors to refine scan results

Code sample

C#

To learn how to install and use the client library for Sensitive Data Protection, see Sensitive Data Protection client libraries.

To authenticate to Sensitive Data Protection, set up Application Default Credentials. For more information, see Set up authentication for a local development environment.

using System.Linq;
using Google.Api.Gax.ResourceNames;
using Google.Cloud.Dlp.V2;
using static Google.Cloud.Dlp.V2.CustomInfoType.Types;

public class InspectStringWithoutOverlap
{
    public static InspectContentResponse Inspect(string projectId, string textToInspect)
    {
        // Initialize client that will be used to send requests. This client only needs to be created
        // once, and can be reused for multiple requests. After completing all of your requests, call
        // the "close" method on the client to safely clean up any remaining background resources.
        var dlp = DlpServiceClient.Create();

        // Specify the type and content to be inspected.
        var byteContentItem = new ByteContentItem
        {
            Type = ByteContentItem.Types.BytesType.TextUtf8,
            Data = Google.Protobuf.ByteString.CopyFromUtf8(textToInspect)
        };
        var contentItem = new ContentItem
        {
            ByteItem = byteContentItem
        };

        // Specify the type of info the inspection will look for.
        // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types.
        var infoTypes = new string[] { "DOMAIN_NAME", "EMAIL_ADDRESS" }.Select(it => new InfoType { Name = it });

        // Define a custom info type to exclude email addresses
        var customInfoType = new CustomInfoType
        {
            InfoType = new InfoType { Name = "EMAIL_ADDRESS" },
            ExclusionType = ExclusionType.Exclude
        };

        // Exclude EMAIL_ADDRESS matches
        var exclusionRule = new ExclusionRule
        {
            ExcludeInfoTypes = new ExcludeInfoTypes
            {
                InfoTypes = { new InfoType { Name = "EMAIL_ADDRESS" } }
            },
            MatchingType = MatchingType.PartialMatch
        };

        // Construct a ruleset that applies the exclusion rule to the DOMAIN_NAME infotype.
        // If a DOMAIN_NAME match is part of an EMAIL_ADDRESS match, the DOMAIN_NAME match will
        // be excluded.
        var ruleSet = new InspectionRuleSet
        {
            InfoTypes = { new InfoType { Name = "DOMAIN_NAME" } },
            Rules = { new InspectionRule { ExclusionRule = exclusionRule } }
        };

        // Construct the configuration for the Inspect request, including the ruleset.
        var config = new InspectConfig
        {
            InfoTypes = { infoTypes },
            CustomInfoTypes = { customInfoType },
            IncludeQuote = true,
            RuleSet = { ruleSet }
        };

        // Construct the Inspect request to be sent by the client.
        var request = new InspectContentRequest
        {
            Parent = new LocationName(projectId, "global").ToString(),
            Item = contentItem,
            InspectConfig = config
        };

        // Use the client to send the API request.
        var response = dlp.InspectContent(request);

        return response;
    }
}

Go

To learn how to install and use the client library for Sensitive Data Protection, see Sensitive Data Protection client libraries.

To authenticate to Sensitive Data Protection, set up Application Default Credentials. For more information, see Set up authentication for a local development environment.

import (
	"context"
	"fmt"
	"io"

	dlp "cloud.google.com/go/dlp/apiv2"
	"cloud.google.com/go/dlp/apiv2/dlppb"
)

// inspectStringWithoutOverlap inspects a string for sensitive data
// and omit overlapping matches on domain and email
func inspectStringWithoutOverlap(w io.Writer, projectID, textToInspect string) error {
	// projectID := "my-project-id"
	// textToInspect := "example.com is a domain, [e-mail] is an email."

	ctx := context.Background()

	// Initialize a client once and reuse it to send multiple requests. Clients
	// are safe to use across goroutines. When the client is no longer needed,
	// call the Close method to cleanup its resources.
	client, err := dlp.NewClient(ctx)
	if err != nil {
		return err
	}

	// Closing the client safely cleans up background resources.
	defer client.Close()

	// Specify the type and content to be inspected.
	contentItem := &dlppb.ContentItem{
		DataItem: &dlppb.ContentItem_ByteItem{
			ByteItem: &dlppb.ByteContentItem{
				Type: dlppb.ByteContentItem_TEXT_UTF8,
				Data: []byte(textToInspect),
			},
		},
	}

	// Specify the type of info the inspection will look for.
	// See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types.
	infoTypes := []*dlppb.InfoType{
		{Name: "DOMAIN_NAME"},
		{Name: "EMAIL_ADDRESS"},
	}

	// Define a custom info type to exclude email addresses
	customInfotype := &dlppb.CustomInfoType{
		InfoType: &dlppb.InfoType{
			Name: "EMAIL_ADDRESS",
		},
		ExclusionType: dlppb.CustomInfoType_EXCLUSION_TYPE_EXCLUDE,
	}

	// Exclude EMAIL_ADDRESS matches
	exclusionRule := &dlppb.ExclusionRule{
		Type: &dlppb.ExclusionRule_ExcludeInfoTypes{
			ExcludeInfoTypes: &dlppb.ExcludeInfoTypes{
				InfoTypes: []*dlppb.InfoType{
					{Name: "EMAIL_ADDRESS"},
				},
			},
		},
		MatchingType: dlppb.MatchingType_MATCHING_TYPE_PARTIAL_MATCH,
	}

	// Construct a ruleSet that applies the exclusion rule to the DOMAIN_NAME infoType.
	// If a DOMAIN_NAME match is part of an EMAIL_ADDRESS match, the DOMAIN_NAME match will
	// be excluded.
	ruleSet := &dlppb.InspectionRuleSet{
		InfoTypes: []*dlppb.InfoType{
			{Name: "DOMAIN_NAME"},
		},
		Rules: []*dlppb.InspectionRule{
			{
				Type: &dlppb.InspectionRule_ExclusionRule{
					ExclusionRule: exclusionRule,
				},
			},
		},
	}

	// Construct the Inspect request to be sent by the client.
	req := &dlppb.InspectContentRequest{
		Parent: fmt.Sprintf("projects/%s/locations/global", projectID),
		Item:   contentItem,
		// Construct the Inspect request to be sent by the client.
		InspectConfig: &dlppb.InspectConfig{
			InfoTypes: infoTypes,
			CustomInfoTypes: []*dlppb.CustomInfoType{
				customInfotype,
			},
			IncludeQuote: true,
			RuleSet: []*dlppb.InspectionRuleSet{
				ruleSet,
			},
		},
	}

	// Send the request.
	resp, err := client.InspectContent(ctx, req)
	if err != nil {
		return err
	}

	// Process the results.
	fmt.Fprintf(w, "Findings: %v\n", len(resp.Result.Findings))
	for _, v := range resp.GetResult().Findings {
		fmt.Fprintf(w, "Quote: %v\n", v.GetQuote())
		fmt.Fprintf(w, "Infotype Name: %v\n", v.GetInfoType().GetName())
		fmt.Fprintf(w, "Likelihood: %v\n", v.GetLikelihood())
	}
	return nil
}

Java

To learn how to install and use the client library for Sensitive Data Protection, see Sensitive Data Protection client libraries.

To authenticate to Sensitive Data Protection, set up Application Default Credentials. For more information, see Set up authentication for a local development environment.

import com.google.cloud.dlp.v2.DlpServiceClient;
import com.google.privacy.dlp.v2.ByteContentItem;
import com.google.privacy.dlp.v2.ByteContentItem.BytesType;
import com.google.privacy.dlp.v2.ContentItem;
import com.google.privacy.dlp.v2.CustomInfoType;
import com.google.privacy.dlp.v2.CustomInfoType.ExclusionType;
import com.google.privacy.dlp.v2.ExcludeInfoTypes;
import com.google.privacy.dlp.v2.ExclusionRule;
import com.google.privacy.dlp.v2.Finding;
import com.google.privacy.dlp.v2.InfoType;
import com.google.privacy.dlp.v2.InspectConfig;
import com.google.privacy.dlp.v2.InspectContentRequest;
import com.google.privacy.dlp.v2.InspectContentResponse;
import com.google.privacy.dlp.v2.InspectionRule;
import com.google.privacy.dlp.v2.InspectionRuleSet;
import com.google.privacy.dlp.v2.LocationName;
import com.google.privacy.dlp.v2.MatchingType;
import com.google.protobuf.ByteString;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

public class InspectStringWithoutOverlap {

  public static void main(String[] args) throws Exception {
    // TODO(developer): Replace these variables before running the sample.
    String projectId = "your-project-id";
    String textToInspect = "example.com is a domain, [e-mail] is an email.";
    inspectStringWithoutOverlap(projectId, textToInspect);
  }

  // Inspects the provided text, avoiding matches specified in the exclusion list.
  public static void inspectStringWithoutOverlap(String projectId, String textToInspect)
      throws IOException {
    // Initialize client that will be used to send requests. This client only needs to be created
    // once, and can be reused for multiple requests. After completing all of your requests, call
    // the "close" method on the client to safely clean up any remaining background resources.
    try (DlpServiceClient dlp = DlpServiceClient.create()) {
      // Specify the type and content to be inspected.
      ByteContentItem byteItem =
          ByteContentItem.newBuilder()
              .setType(BytesType.TEXT_UTF8)
              .setData(ByteString.copyFromUtf8(textToInspect))
              .build();
      ContentItem item = ContentItem.newBuilder().setByteItem(byteItem).build();

      // Specify the type of info the inspection will look for.
      // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types.
      List<InfoType> infoTypes = new ArrayList<>();
      for (String typeName : new String[] {"DOMAIN_NAME", "EMAIL_ADDRESS"}) {
        infoTypes.add(InfoType.newBuilder().setName(typeName).build());
      }

      // Define a custom info type to exclude email addresses
      CustomInfoType customInfoType =
          CustomInfoType.newBuilder()
              .setInfoType(InfoType.newBuilder().setName("EMAIL_ADDRESS"))
              .setExclusionType(ExclusionType.EXCLUSION_TYPE_EXCLUDE)
              .build();

      // Exclude EMAIL_ADDRESS matches
      ExclusionRule exclusionRule =
          ExclusionRule.newBuilder()
              .setExcludeInfoTypes(
                  ExcludeInfoTypes.newBuilder()
                      .addInfoTypes(InfoType.newBuilder().setName("EMAIL_ADDRESS")))
              .setMatchingType(MatchingType.MATCHING_TYPE_PARTIAL_MATCH)
              .build();

      // Construct a ruleset that applies the exclusion rule to the DOMAIN_NAME infotype.
      // If a DOMAIN_NAME match is part of an EMAIL_ADDRESS match, the DOMAIN_NAME match will
      // be excluded.
      InspectionRuleSet ruleSet =
          InspectionRuleSet.newBuilder()
              .addInfoTypes(InfoType.newBuilder().setName("DOMAIN_NAME"))
              .addRules(InspectionRule.newBuilder().setExclusionRule(exclusionRule))
              .build();

      // Construct the configuration for the Inspect request, including the ruleset.
      InspectConfig config =
          InspectConfig.newBuilder()
              .addAllInfoTypes(infoTypes)
              .addCustomInfoTypes(customInfoType)
              .setIncludeQuote(true)
              .addRuleSet(ruleSet)
              .build();

      // Construct the Inspect request to be sent by the client.
      InspectContentRequest request =
          InspectContentRequest.newBuilder()
              .setParent(LocationName.of(projectId, "global").toString())
              .setItem(item)
              .setInspectConfig(config)
              .build();

      // Use the client to send the API request.
      InspectContentResponse response = dlp.inspectContent(request);

      // Parse the response and process results
      System.out.println("Findings: " + response.getResult().getFindingsCount());
      for (Finding f : response.getResult().getFindingsList()) {
        System.out.println("\tQuote: " + f.getQuote());
        System.out.println("\tInfo type: " + f.getInfoType().getName());
        System.out.println("\tLikelihood: " + f.getLikelihood());
      }
    }
  }
}

Node.js

To learn how to install and use the client library for Sensitive Data Protection, see Sensitive Data Protection client libraries.

To authenticate to Sensitive Data Protection, set up Application Default Credentials. For more information, see Set up authentication for a local development environment.

// Imports the Google Cloud Data Loss Prevention library
const DLP = require('@google-cloud/dlp');

// Instantiates a client
const dlp = new DLP.DlpServiceClient();

// The project ID to run the API call under
// const projectId = 'my-project';

// The string to inspect
// const string = 'example.com is a domain, [e-mail] is an email.';

async function inspectStringWithoutOverlap() {
  // Construct item to inspect
  const item = {value: string};

  // The infoTypes of information to match
  const infoTypes = [{name: 'DOMAIN_NAME'}, {name: 'EMAIL_ADDRESS'}];

  // Define a custom info type to exclude email addresses
  const customInfoTypes = [
    {
      infoType: {name: 'EMAIL_ADDRESS'},
      exclusionType:
        DLP.protos.google.privacy.dlp.v2.CustomInfoType.ExclusionType
          .EXCLUSION_TYPE_EXCLUDE,
    },
  ];

  // Construct a exclusion rule
  const exclusionRule = {
    excludeInfoTypes: {
      infoTypes: [{name: 'EMAIL_ADDRESS'}],
    },
    matchingType:
      DLP.protos.google.privacy.dlp.v2.MatchingType
        .MATCHING_TYPE_PARTIAL_MATCH,
  };

  // Construct a rule set with exclusions
  const ruleSet = [
    {
      infoTypes: [{name: 'DOMAIN_NAME'}],
      rules: [
        {
          exclusionRule: exclusionRule,
        },
      ],
    },
  ];

  // Construct the inspect configuration
  const inspectConfig = {
    infoTypes: infoTypes,
    customInfoTypes: customInfoTypes,
    ruleSet: ruleSet,
    includeQuote: true,
  };

  // Combine configurations into a request for the service.
  const request = {
    parent: `projects/${projectId}/locations/global`,
    inspectConfig: inspectConfig,
    item: item,
  };

  // Run request
  const [response] = await dlp.inspectContent(request);

  // Print Findings
  const findings = response.result.findings;
  if (findings.length > 0) {
    console.log(`Findings: ${findings.length}\n`);
    findings.forEach(finding => {
      console.log(`InfoType: ${finding.infoType.name}`);
      console.log(`\tQuote: ${finding.quote}`);
      console.log(`\tLikelihood: ${finding.likelihood} \n`);
    });
  } else {
    console.log('No findings.');
  }
}
inspectStringWithoutOverlap();

PHP

To learn how to install and use the client library for Sensitive Data Protection, see Sensitive Data Protection client libraries.

To authenticate to Sensitive Data Protection, set up Application Default Credentials. For more information, see Set up authentication for a local development environment.

use Google\Cloud\Dlp\V2\Client\DlpServiceClient;
use Google\Cloud\Dlp\V2\ContentItem;
use Google\Cloud\Dlp\V2\CustomInfoType;
use Google\Cloud\Dlp\V2\CustomInfoType\ExclusionType;
use Google\Cloud\Dlp\V2\ExcludeInfoTypes;
use Google\Cloud\Dlp\V2\ExclusionRule;
use Google\Cloud\Dlp\V2\InfoType;
use Google\Cloud\Dlp\V2\InspectConfig;
use Google\Cloud\Dlp\V2\InspectContentRequest;
use Google\Cloud\Dlp\V2\InspectionRule;
use Google\Cloud\Dlp\V2\InspectionRuleSet;
use Google\Cloud\Dlp\V2\Likelihood;
use Google\Cloud\Dlp\V2\MatchingType;

/**
 * Inspect a string for sensitive data, omitting overlapping matches on domain and email
 * Omit matches on domain names that are part of email addresses in a DOMAIN_NAME detector scan.
 *
 * @param string $projectId         The Google Cloud project id to use as a parent resource.
 * @param string $textToInspect     The string to inspect.
 */
function inspect_string_without_overlap(
    // TODO(developer): Replace sample parameters before running the code.
    string $projectId,
    string $textToInspect = 'example.com is a domain, [e-mail] is an email.'
): void {
    // Instantiate a client.
    $dlp = new DlpServiceClient();

    $parent = "projects/$projectId/locations/global";

    // Specify what content you want the service to Inspect.
    $item = (new ContentItem())
        ->setValue($textToInspect);

    // Specify the type of info the inspection will look for.
    $domainName = (new InfoType())
        ->setName('DOMAIN_NAME');
    $emailAddress = (new InfoType())
        ->setName('EMAIL_ADDRESS');
    $infoTypes = [$domainName, $emailAddress];

    // Define a custom info type to exclude email addresses
    $customInfoType = (new CustomInfoType())
        ->setInfoType($emailAddress)
        ->setExclusionType(ExclusionType::EXCLUSION_TYPE_EXCLUDE);

    // Exclude EMAIL_ADDRESS matches
    $matchingType = MatchingType::MATCHING_TYPE_PARTIAL_MATCH;

    $exclusionRule = (new ExclusionRule())
        ->setMatchingType($matchingType)
        ->setExcludeInfoTypes((new ExcludeInfoTypes())
            ->setInfoTypes([$customInfoType->getInfoType()])
        );

    // Construct a ruleset that applies the exclusion rule to the DOMAIN_NAME infotype.
    // If a DOMAIN_NAME match is part of an EMAIL_ADDRESS match, the DOMAIN_NAME match will
    // be excluded.
    $inspectionRuleSet = (new InspectionRuleSet())
        ->setInfoTypes([$domainName])
        ->setRules([
            (new InspectionRule())
                ->setExclusionRule($exclusionRule),
        ]);

    // Construct the configuration for the Inspect request, including the ruleset.
    $inspectConfig = (new InspectConfig())
        ->setInfoTypes($infoTypes)
        ->setCustomInfoTypes([$customInfoType])
        ->setIncludeQuote(true)
        ->setRuleSet([$inspectionRuleSet]);

    // Run request
    $inspectContentRequest = (new InspectContentRequest())
        ->setParent($parent)
        ->setInspectConfig($inspectConfig)
        ->setItem($item);
    $response = $dlp->inspectContent($inspectContentRequest);

    // Print the results
    $findings = $response->getResult()->getFindings();
    if (count($findings) == 0) {
        printf('No findings.' . PHP_EOL);
    } else {
        printf('Findings:' . PHP_EOL);
        foreach ($findings as $finding) {
            printf('  Quote: %s' . PHP_EOL, $finding->getQuote());
            printf('  Info type: %s' . PHP_EOL, $finding->getInfoType()->getName());
            printf(
                '  Likelihood: %s' . PHP_EOL,
                Likelihood::name($finding->getLikelihood()));
        }
    }
}

Python

To learn how to install and use the client library for Sensitive Data Protection, see Sensitive Data Protection client libraries.

To authenticate to Sensitive Data Protection, set up Application Default Credentials. For more information, see Set up authentication for a local development environment.

import google.cloud.dlp


def inspect_string_without_overlap(project: str, content_string: str) -> None:
    """Matches EMAIL_ADDRESS and DOMAIN_NAME, but DOMAIN_NAME is omitted
    if it overlaps with EMAIL_ADDRESS

    Uses the Data Loss Prevention API to omit matches of one infotype
    that overlap with another.

    Args:
        project: The Google Cloud project id to use as a parent resource.
        content_string: The string to inspect.

    Returns:
        None; the response from the API is printed to the terminal.
    """

    # Instantiate a client.
    dlp = google.cloud.dlp_v2.DlpServiceClient()

    # Construct a list of infoTypes for DLP to locate in `content_string`. See
    # https://cloud.google.com/dlp/docs/concepts-infotypes for more information
    # about supported infoTypes.
    info_types_to_locate = [{"name": "DOMAIN_NAME"}, {"name": "EMAIL_ADDRESS"}]

    # Define a custom info type to exclude email addresses
    custom_info_types = [
        {
            "info_type": {"name": "EMAIL_ADDRESS"},
            "exclusion_type": google.cloud.dlp_v2.CustomInfoType.ExclusionType.EXCLUSION_TYPE_EXCLUDE,
        }
    ]

    # Construct a rule set that will exclude DOMAIN_NAME matches
    # that overlap with EMAIL_ADDRESS matches
    rule_set = [
        {
            "info_types": [{"name": "DOMAIN_NAME"}],
            "rules": [
                {
                    "exclusion_rule": {
                        "exclude_info_types": {
                            "info_types": [{"name": "EMAIL_ADDRESS"}]
                        },
                        "matching_type": google.cloud.dlp_v2.MatchingType.MATCHING_TYPE_PARTIAL_MATCH,
                    }
                }
            ],
        }
    ]

    # Construct the configuration dictionary
    inspect_config = {
        "info_types": info_types_to_locate,
        "custom_info_types": custom_info_types,
        "rule_set": rule_set,
        "include_quote": True,
    }

    # Construct the `item`.
    item = {"value": content_string}

    # Convert the project id into a full resource id.
    parent = f"projects/{project}"

    # Call the API.
    response = dlp.inspect_content(
        request={"parent": parent, "inspect_config": inspect_config, "item": item}
    )

    # Print out the results.
    if response.result.findings:
        for finding in response.result.findings:
            print(f"Quote: {finding.quote}")
            print(f"Info type: {finding.info_type.name}")
            print(f"Likelihood: {finding.likelihood}")
    else:
        print("No findings.")

What's next

To search and filter code samples for other Google Cloud products, see the Google Cloud sample browser.