Set the match likelihood of a table column

Set the match likelihood of an entire column of data. This approach is helpful, for example, if you want to exclude a column of data from inspection results.

Explore further

For detailed documentation that includes this code sample, see the following:

• Customizing match likelihood

Code sample

C#

To learn how to install and use the client library for Sensitive Data Protection, see Sensitive Data Protection client libraries.

To authenticate to Sensitive Data Protection, set up Application Default Credentials. For more information, see Set up authentication for a local development environment.

using Google.Api.Gax.ResourceNames;
using Google.Cloud.Dlp.V2;
using System;
using System.Collections.Generic;
using static Google.Cloud.Dlp.V2.CustomInfoType.Types;

public class InspectTableWithCustomHotwords
{
    public static InspectResult InspectTable(
        string projectId,
        Table tableToInspect = null,
        IEnumerable<InfoType> infoTypes = null)
    {
        // Instantiate the dlp client.
        var dlp = DlpServiceClient.Create();

        // Construct the table if null.
        if (tableToInspect == null)
        {
            var row1 = new Value[]
            {
                new Value{ StringValue = "111-11-1111" },
                new Value { StringValue = "222-22-2222" }
            };
            tableToInspect = new Table
            {
                Headers =
                {
                    new FieldId { Name = "Fake Social Security Number" },
                    new FieldId { Name = "Real Social Security Number" }
                },
                Rows =
                {
                    new Table.Types.Row { Values = { row1 } }
                }
            };
        }

        // Specify the table and construct the content item.
        var contentItem = new ContentItem { Table = tableToInspect };

        // Specify the type of info to be inspected.
        var infotypes = infoTypes ?? new InfoType[] { new InfoType { Name = "US_SOCIAL_SECURITY_NUMBER" } };

        // Construct the Inspection Rule Set by specifying the hotword rule as detection rule.
        var ruleSet = new InspectionRuleSet[]
        {
            new InspectionRuleSet
            {
                InfoTypes = { infotypes },
                Rules =
                {
                    new InspectionRule
                    {
                        HotwordRule = new DetectionRule.Types.HotwordRule
                        {
                            HotwordRegex = new Regex
                            {
                                Pattern = "(Fake Social Security Number)"
                            },
                            LikelihoodAdjustment = new DetectionRule.Types.LikelihoodAdjustment
                            {
                                FixedLikelihood = Likelihood.VeryUnlikely
                            },
                            Proximity = new DetectionRule.Types.Proximity
                            {
                                WindowBefore = 1
                            }
                        }
                    }
                }
            }
        };

        // Construct the request.
        var request = new InspectContentRequest
        {
            InspectConfig = new InspectConfig
            {
                InfoTypes = { infotypes },
                IncludeQuote = true,
                MinLikelihood = Likelihood.Possible,
                RuleSet = { ruleSet }
            },
            ParentAsLocationName = new LocationName(projectId, "global"),
            Item = contentItem
        };

        // Call the API.
        InspectContentResponse response = dlp.InspectContent(request);

        // Inspect the results.
        var resultFindings = response.Result.Findings;

        Console.WriteLine($"Findings: {resultFindings.Count}");

        foreach (var f in resultFindings)
        {
            Console.WriteLine("\tQuote: " + f.Quote);
            Console.WriteLine("\tInfo type: " + f.InfoType.Name);
            Console.WriteLine("\tLikelihood: " + f.Likelihood);
        }

        return response.Result;
    }
}

Go

To learn how to install and use the client library for Sensitive Data Protection, see Sensitive Data Protection client libraries.

To authenticate to Sensitive Data Protection, set up Application Default Credentials. For more information, see Set up authentication for a local development environment.

import (
	"context"
	"fmt"
	"io"

	dlp "cloud.google.com/go/dlp/apiv2"
	"cloud.google.com/go/dlp/apiv2/dlppb"
)

// inspectTableWithCustomHotword Sets the match likelihood of a table column to customize data inspection results.
// This example showcases how you can adjust the match likelihood for an entire column of data, enabling the
// exclusion of specific columns from inspection if needed.
func inspectTableWithCustomHotword(w io.Writer, projectID, hotwordRegexPattern string) error {
	// projectID := "your-project-id"
	// hotwordRegexPattern := "(Fake Social Security Number)"

	tableToInspect := &dlppb.Table{
		Headers: []*dlppb.FieldId{
			{Name: "Fake Social Security Number"},
			{Name: "Real Social Security Number"},
		},
		Rows: []*dlppb.Table_Row{
			{
				Values: []*dlppb.Value{
					{
						Type: &dlppb.Value_StringValue{StringValue: "111-11-1111"},
					},
					{
						Type: &dlppb.Value_StringValue{StringValue: "222-22-2222"},
					},
				},
			},
		},
	}

	ctx := context.Background()

	// Initialize a client once and reuse it to send multiple requests. Clients
	// are safe to use across goroutines. When the client is no longer needed,
	// call the Close method to cleanup its resources.
	client, err := dlp.NewClient(ctx)
	if err != nil {
		return err
	}

	// Closing the client safely cleans up background resources.
	defer client.Close()

	// Specify what content you want the service to de-identify.
	contentItem := &dlppb.ContentItem_Table{
		Table: tableToInspect,
	}

	// Specify the likelihood adjustment to adjust the match likelihood for your detection rule
	// based on your needs and desired level of sensitivity in data analysis.
	likelihoodAdjustment := &dlppb.CustomInfoType_DetectionRule_LikelihoodAdjustment{
		Adjustment: &dlppb.CustomInfoType_DetectionRule_LikelihoodAdjustment_FixedLikelihood{
			FixedLikelihood: dlppb.Likelihood_VERY_UNLIKELY,
		},
	}

	// Specify the type of info the inspection will look for.
	// See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types.
	infoTypes := []*dlppb.InfoType{
		{Name: "US_SOCIAL_SECURITY_NUMBER"},
	}

	// Specify the proximity so that It helps identify sensitive information
	// occurring near other data points, enabling more context-aware analysis.
	proximity := &dlppb.CustomInfoType_DetectionRule_Proximity{
		WindowBefore: 5,
	}

	// Construct hotWord rule.
	hotwordRule := &dlppb.CustomInfoType_DetectionRule_HotwordRule{
		HotwordRegex: &dlppb.CustomInfoType_Regex{
			Pattern: hotwordRegexPattern,
		},
		Proximity:            proximity,
		LikelihoodAdjustment: likelihoodAdjustment,
	}

	// Construct rule set for the inspect config.
	inspectionRuleSet := &dlppb.InspectionRuleSet{
		InfoTypes: infoTypes,
		Rules: []*dlppb.InspectionRule{
			{
				Type: &dlppb.InspectionRule_HotwordRule{
					HotwordRule: hotwordRule,
				},
			},
		},
	}

	// Construct the configuration for the Inspect request.
	config := &dlppb.InspectConfig{
		IncludeQuote:  true,
		InfoTypes:     infoTypes,
		MinLikelihood: dlppb.Likelihood_POSSIBLE,
		RuleSet: []*dlppb.InspectionRuleSet{
			inspectionRuleSet,
		},
	}

	// Construct the Inspect request to be sent by the client.
	req := &dlppb.InspectContentRequest{
		Parent: fmt.Sprintf("projects/%s/locations/global", projectID),
		Item: &dlppb.ContentItem{
			DataItem: contentItem,
		},
		InspectConfig: config,
	}
	// Send the request.
	resp, err := client.InspectContent(ctx, req)
	if err != nil {
		return err
	}

	// Parse the response and process results.
	fmt.Fprintf(w, "Findings: %v\n", len(resp.Result.Findings))
	for _, v := range resp.GetResult().Findings {
		fmt.Fprintf(w, "Quote: %v\n", v.GetQuote())
		fmt.Fprintf(w, "Infotype Name: %v\n", v.GetInfoType().GetName())
		fmt.Fprintf(w, "Likelihood: %v\n", v.GetLikelihood())
	}
	return nil
}

Java

To learn how to install and use the client library for Sensitive Data Protection, see Sensitive Data Protection client libraries.

To authenticate to Sensitive Data Protection, set up Application Default Credentials. For more information, see Set up authentication for a local development environment.

import com.google.cloud.dlp.v2.DlpServiceClient;
import com.google.privacy.dlp.v2.ContentItem;
import com.google.privacy.dlp.v2.CustomInfoType;
import com.google.privacy.dlp.v2.FieldId;
import com.google.privacy.dlp.v2.Finding;
import com.google.privacy.dlp.v2.InfoType;
import com.google.privacy.dlp.v2.InspectConfig;
import com.google.privacy.dlp.v2.InspectContentRequest;
import com.google.privacy.dlp.v2.InspectContentResponse;
import com.google.privacy.dlp.v2.InspectionRule;
import com.google.privacy.dlp.v2.InspectionRuleSet;
import com.google.privacy.dlp.v2.Likelihood;
import com.google.privacy.dlp.v2.LocationName;
import com.google.privacy.dlp.v2.Table;
import com.google.privacy.dlp.v2.Value;
import java.io.IOException;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;

public class InspectTableWithCustomHotword {

  public static void main(String[] args) throws IOException {
    // TODO(developer): Replace these variables before running the sample.
    // The Google Cloud project id to use as a parent resource.
    String projectId = "your-project-id";
    // Specify the table to be considered for de-identification.
    Table tableToDeIdentify =
        Table.newBuilder()
            .addHeaders(FieldId.newBuilder().setName("Some Social Security Number").build())
            .addHeaders(FieldId.newBuilder().setName("Real Social Security Number").build())
            .addRows(
                Table.Row.newBuilder()
                    .addValues(Value.newBuilder().setStringValue("111-11-1111").build())
                    .addValues(Value.newBuilder().setStringValue("222-22-2222").build())
                    .build())
            .build();
    // Specify the regex pattern to be detected.
    // Refer https://github.com/google/re2/wiki/Syntax for creating regular expression.
    String hotwordRegexPattern = "Some Social Security Number";
    inspectDemotingFindingsWithHotwords(projectId, tableToDeIdentify, hotwordRegexPattern);
  }

  //  Inspects the provided table, excluding the findings of entire column matching regular
  // expression.
  public static void inspectDemotingFindingsWithHotwords(
      String projectId, Table tableToDeIdentify, String hotwordRegexPattern) throws IOException {

    // Initialize client that will be used to send requests. This client only needs to be created
    // once, and can be reused for multiple requests. After completing all of your requests, call
    // the "close" method on the client to safely clean up any remaining background resources.
    try (DlpServiceClient dlp = DlpServiceClient.create()) {
      // Specify what content you want the service to de-identify.
      ContentItem contentItem = ContentItem.newBuilder().setTable(tableToDeIdentify).build();

      CustomInfoType.DetectionRule.LikelihoodAdjustment likelihoodAdjustment =
          CustomInfoType.DetectionRule.LikelihoodAdjustment.newBuilder()
              .setFixedLikelihood(Likelihood.VERY_UNLIKELY)
              .build();

      // Specify the type of info the inspection will look for.
      // See https://cloud.google.com/dlp/docs/infotypes-reference for complete list of info types
      List<InfoType> infoTypes =
          Stream.of("US_SOCIAL_SECURITY_NUMBER")
              .map(it -> InfoType.newBuilder().setName(it).build())
              .collect(Collectors.toList());

      CustomInfoType.DetectionRule.Proximity proximity =
          CustomInfoType.DetectionRule.Proximity.newBuilder().setWindowBefore(1).build();

      // Construct hotword rule.
      CustomInfoType.DetectionRule.HotwordRule hotwordRule =
          CustomInfoType.DetectionRule.HotwordRule.newBuilder()
              .setHotwordRegex(
                  CustomInfoType.Regex.newBuilder().setPattern(hotwordRegexPattern).build())
              .setLikelihoodAdjustment(likelihoodAdjustment)
              .setProximity(proximity)
              .build();

      // Construct rule set for the inspect config.
      InspectionRuleSet inspectionRuleSet =
          InspectionRuleSet.newBuilder()
              .addAllInfoTypes(infoTypes)
              .addRules(InspectionRule.newBuilder().setHotwordRule(hotwordRule))
              .build();

      // Construct the configuration for the Inspect request.
      InspectConfig config =
          InspectConfig.newBuilder()
              .setIncludeQuote(true)
              .setMinLikelihood(Likelihood.POSSIBLE)
              .addRuleSet(inspectionRuleSet)
              .addAllInfoTypes(infoTypes)
              .build();

      // Construct the Inspect request to be sent by the client.
      InspectContentRequest request =
          InspectContentRequest.newBuilder()
              .setParent(LocationName.of(projectId, "global").toString())
              .setItem(contentItem)
              .setInspectConfig(config)
              .build();

      InspectContentResponse response = dlp.inspectContent(request);
      // Parse the response and process results.
      System.out.println("Findings: " + response.getResult().getFindingsCount());
      for (Finding f : response.getResult().getFindingsList()) {
        System.out.println("\tQuote: " + f.getQuote());
        System.out.println("\tInfo type: " + f.getInfoType().getName());
        System.out.println("\tLikelihood: " + f.getLikelihood());
      }
    }
  }
}

Node.js

To learn how to install and use the client library for Sensitive Data Protection, see Sensitive Data Protection client libraries.

To authenticate to Sensitive Data Protection, set up Application Default Credentials. For more information, see Set up authentication for a local development environment.

// Imports the Google Cloud Data Loss Prevention library
const DLP = require('@google-cloud/dlp');

// Instantiates a client
const dlp = new DLP.DlpServiceClient();

// The project ID to run the API call under.
// const projectId = "your-project-id";

// Table to inspect
const tableToInspect = {
  headers: [
    {name: 'Fake Social Security Number'},
    {name: 'Real Social Security Number'},
  ],
  rows: [
    {
      values: [{stringValue: '111-11-1111'}, {stringValue: '222-22-2222'}],
    },
  ],
};

async function inspectWithCustomHotwords() {
  // Specify the regex pattern to be detected.
  const hotwordRegexPattern = '(Fake Social Security Number)';

  // Specify what content you want the service to de-identify.
  const contentItem = {
    table: tableToInspect,
  };

  // Specify the type of info the inspection will look for.
  const infoTypes = [{name: 'US_SOCIAL_SECURITY_NUMBER'}];

  // Construct hotword rule.
  const hotwordRule = {
    hotwordRegex: {
      pattern: hotwordRegexPattern,
    },
    likelihoodAdjustment: {
      fixedLikelihood:
        DLP.protos.google.privacy.dlp.v2.Likelihood.VERY_UNLIKELY,
    },
    proximity: {
      windowBefore: 1,
    },
  };

  // Construct rule set for the inspect configuration.
  const inspectionRuleSet = {
    infoTypes: infoTypes,
    rules: [
      {
        hotwordRule: hotwordRule,
      },
    ],
  };

  // Construct the configuration for the Inspect request.
  const config = {
    infoTypes: infoTypes,
    ruleSet: [inspectionRuleSet],
    minLikelihood: DLP.protos.google.privacy.dlp.v2.Likelihood.POSSIBLE,
    includeQuote: true,
  };

  // Construct the Inspect request to be sent by the client.
  const request = {
    parent: `projects/${projectId}/locations/global`,
    item: contentItem,
    inspectConfig: config,
  };

  // Use the client to send the API request.
  const [response] = await dlp.inspectContent(request);

  // Print Findings.
  const findings = response.result.findings;
  if (findings.length > 0) {
    console.log(`Findings: ${findings.length}\n`);
    findings.forEach(finding => {
      console.log(`InfoType: ${finding.infoType.name}`);
      console.log(`\tQuote: ${finding.quote}`);
      console.log(`\tLikelihood: ${finding.likelihood} \n`);
    });
  } else {
    console.log('No findings.');
  }
}
inspectWithCustomHotwords();

PHP

To learn how to install and use the client library for Sensitive Data Protection, see Sensitive Data Protection client libraries.

To authenticate to Sensitive Data Protection, set up Application Default Credentials. For more information, see Set up authentication for a local development environment.

use Google\Cloud\Dlp\V2\Client\DlpServiceClient;
use Google\Cloud\Dlp\V2\ContentItem;
use Google\Cloud\Dlp\V2\CustomInfoType\DetectionRule\HotwordRule;
use Google\Cloud\Dlp\V2\CustomInfoType\DetectionRule\LikelihoodAdjustment;
use Google\Cloud\Dlp\V2\CustomInfoType\DetectionRule\Proximity;
use Google\Cloud\Dlp\V2\CustomInfoType\Regex;
use Google\Cloud\Dlp\V2\FieldId;
use Google\Cloud\Dlp\V2\InfoType;
use Google\Cloud\Dlp\V2\InspectConfig;
use Google\Cloud\Dlp\V2\InspectContentRequest;
use Google\Cloud\Dlp\V2\InspectionRule;
use Google\Cloud\Dlp\V2\InspectionRuleSet;
use Google\Cloud\Dlp\V2\Likelihood;
use Google\Cloud\Dlp\V2\Table;
use Google\Cloud\Dlp\V2\Table\Row;
use Google\Cloud\Dlp\V2\Value;

/**
 * Hotword example: Set the match likelihood of a table column.
 * This example demonstrates how you can set the match likelihood of an entire column of data.
 * This approach is helpful, for example, if you want to exclude a column of data from inspection
 * results.
 *
 * @param string $projectId         The Google Cloud project id to use as a parent resource.
 */
function inspect_column_values_w_custom_hotwords(string $projectId): void
{
    // Instantiate a client.
    $dlp = new DlpServiceClient();

    $parent = "projects/$projectId/locations/global";

    // Specify the table to be inspected.
    $tableToDeIdentify = (new Table())
        ->setHeaders([
            (new FieldId())
                ->setName('Fake Social Security Number'),
            (new FieldId())
                ->setName('Real Social Security Number'),
        ])
        ->setRows([
            (new Row())->setValues([
                (new Value())
                    ->setStringValue('111-11-1111'),
                (new Value())
                    ->setStringValue('222-22-2222')
            ])
        ]);

    $item = (new ContentItem())
        ->setTable($tableToDeIdentify);

    // Specify the regex pattern the inspection will look for.
    $hotwordRegexPattern = 'Fake Social Security Number';

    // Specify hotword likelihood adjustment.
    $likelihoodAdjustment = (new LikelihoodAdjustment())
        ->setFixedLikelihood(Likelihood::VERY_UNLIKELY);

    // Specify a window around a finding to apply a detection rule.
    $proximity = (new Proximity())
        ->setWindowBefore(1);

    // Construct the hotword rule.
    $hotwordRule = (new HotwordRule())
        ->setHotwordRegex((new Regex())
            ->setPattern($hotwordRegexPattern))
        ->setLikelihoodAdjustment($likelihoodAdjustment)
        ->setProximity($proximity);

    // Construct rule set for the inspect config.
    $infotype = (new InfoType())
        ->setName('US_SOCIAL_SECURITY_NUMBER');
    $inspectionRuleSet = (new InspectionRuleSet())
        ->setInfoTypes([$infotype])
        ->setRules([
            (new InspectionRule())
                ->setHotwordRule($hotwordRule)
        ]);

    // Construct the configuration for the Inspect request.
    $inspectConfig = (new InspectConfig())
        ->setInfoTypes([$infotype])
        ->setIncludeQuote(true)
        ->setRuleSet([$inspectionRuleSet])
        ->setMinLikelihood(Likelihood::POSSIBLE);

    // Run request.
    $inspectContentRequest = (new InspectContentRequest())
        ->setParent($parent)
        ->setInspectConfig($inspectConfig)
        ->setItem($item);
    $response = $dlp->inspectContent($inspectContentRequest);

    // Print the results.
    $findings = $response->getResult()->getFindings();
    if (count($findings) == 0) {
        printf('No findings.' . PHP_EOL);
    } else {
        printf('Findings:' . PHP_EOL);
        foreach ($findings as $finding) {
            printf('  Quote: %s' . PHP_EOL, $finding->getQuote());
            printf('  Info type: %s' . PHP_EOL, $finding->getInfoType()->getName());
            printf('  Likelihood: %s' . PHP_EOL, Likelihood::name($finding->getLikelihood()));
        }
    }
}

Python

To learn how to install and use the client library for Sensitive Data Protection, see Sensitive Data Protection client libraries.

To authenticate to Sensitive Data Protection, set up Application Default Credentials. For more information, see Set up authentication for a local development environment.

from typing import List

import google.cloud.dlp


def inspect_column_values_w_custom_hotwords(
    project: str,
    table_header: List[str],
    table_rows: List[List[str]],
    info_types: List[str],
    custom_hotword: str,
) -> None:
    """Uses the Data Loss Prevention API to inspect table data using built-in
    infoType detectors, excluding columns that match a custom hot-word.
    Args:
        project: The Google Cloud project id to use as a parent resource.
        table_header: List of strings representing table field names.
        table_rows: List of rows representing table values.
        info_types: The infoType for which hot-word rule is applied.
        custom_hotword: The custom regular expression used for likelihood boosting.
    """

    # Instantiate a client
    dlp = google.cloud.dlp_v2.DlpServiceClient()

    # Construct the `table`. For more details on the table schema, please see
    # https://cloud.google.com/dlp/docs/reference/rest/v2/ContentItem#Table
    headers = [{"name": val} for val in table_header]
    rows = []
    for row in table_rows:
        rows.append({"values": [{"string_value": cell_val} for cell_val in row]})
    table = {"headers": headers, "rows": rows}

    # Construct the `item` for table to be inspected.
    item = {"table": table}

    # Prepare info_types by converting the list of strings into a list of
    # dictionaries.
    info_types = [{"name": info_type} for info_type in info_types]

    # Construct a rule set with caller provided hot-word, with a likelihood
    # boost to VERY_UNLIKELY when the hot-word are present
    hotword_rule = {
        "hotword_regex": {"pattern": custom_hotword},
        "likelihood_adjustment": {
            "fixed_likelihood": google.cloud.dlp_v2.Likelihood.VERY_UNLIKELY
        },
        "proximity": {"window_before": 1},
    }

    rule_set = [
        {
            "info_types": info_types,
            "rules": [{"hotword_rule": hotword_rule}],
        }
    ]

    # Construct the configuration dictionary, which defines the entire inspect content task.
    inspect_config = {
        "info_types": info_types,
        "rule_set": rule_set,
        "min_likelihood": google.cloud.dlp_v2.Likelihood.POSSIBLE,
        "include_quote": True,
    }

    # Convert the project id into a full resource id.
    parent = f"projects/{project}/locations/global"

    # Call the API
    response = dlp.inspect_content(
        request={
            "parent": parent,
            "inspect_config": inspect_config,
            "item": item,
        }
    )

    # Print out the results.
    if response.result.findings:
        for finding in response.result.findings:
            try:
                if finding.quote:
                    print(f"Quote: {finding.quote}")
            except AttributeError:
                pass
            print(f"Info type: {finding.info_type.name}")
            print(f"Likelihood: {finding.likelihood}")
    else:
        print("No findings.")

What's next

To search and filter code samples for other Google Cloud products, see the Google Cloud sample browser.