De-identify data using table bucketing

Transform a column without inspection. To transform a column in which the content is already known, you can skip inspection and specify a transformation directly.

Explore further

For detailed documentation that includes this code sample, see the following:

• Examples of de-identifying tabular data

Code sample

C#

To learn how to install and use the client library for Sensitive Data Protection, see Sensitive Data Protection client libraries.

To authenticate to Sensitive Data Protection, set up Application Default Credentials. For more information, see Set up authentication for a local development environment.

using System;
using Google.Api.Gax.ResourceNames;
using Google.Cloud.Dlp.V2;

public class DeidentifyUsingTableBucketing
{
    public static Table DeidentifyData(
        string projectId,
        Table tableToInspect = null)
    {
        // Instantiate dlp client.
        var dlp = DlpServiceClient.Create();

        // Construct the table if null.
        if (tableToInspect == null)
        {
            var row1 = new Value[]
            {
                new Value { StringValue = "101" },
                new Value { StringValue = "Charles Dickens" },
                new Value { StringValue = "95" }
            };
            var row2 = new Value[]
            {
                new Value { StringValue = "22" },
                new Value { StringValue = "Jane Austin" },
                new Value { StringValue = "21" }
            };
            var row3 = new Value[]
            {
                new Value { StringValue = "55" },
                new Value { StringValue = "Mark Twain" },
                new Value { StringValue = "75" }
            };

            tableToInspect = new Table
            {
                Headers =
                {
                    new FieldId { Name = "AGE" },
                    new FieldId { Name = "PATIENT" },
                    new FieldId { Name = "HAPPINESS SCORE" }
                },
                Rows =
                {
                    new Table.Types.Row { Values = { row1 } },
                    new Table.Types.Row { Values = { row2 } },
                    new Table.Types.Row { Values = { row3 } }
                }
            };
        }

        // Construct the table content item.
        var contentItem = new ContentItem { Table = tableToInspect };

        // Specify how the content should be de-identified.
        var fixedSizeBucketingConfig = new FixedSizeBucketingConfig
        {
            BucketSize = 10,
            LowerBound = new Value { IntegerValue = 0 },
            UpperBound = new Value { IntegerValue = 100 },
        };

        // Specify the fields to be encrypted.
        var fields = new FieldId[] { new FieldId { Name = "HAPPINESS SCORE" } };

        // Associate the encryption with the specified field.
        var fieldTransformation = new FieldTransformation
        {
            PrimitiveTransformation = new PrimitiveTransformation
            {
                FixedSizeBucketingConfig = fixedSizeBucketingConfig
            },
            Fields = { fields }
        };

        // Construct the deidentify config.
        var deidentifyConfig = new DeidentifyConfig
        {
            RecordTransformations = new RecordTransformations
            {
                FieldTransformations = { fieldTransformation }
            }
        };

        // Construct the request.
        var request = new DeidentifyContentRequest
        {
            ParentAsLocationName = new LocationName(projectId, "global"),
            DeidentifyConfig = deidentifyConfig,
            Item = contentItem,
        };

        // Call the API.
        var response = dlp.DeidentifyContent(request);

        // Inspect the response.
        Console.WriteLine(response.Item.Table);

        return response.Item.Table;
    }
}

Go

To learn how to install and use the client library for Sensitive Data Protection, see Sensitive Data Protection client libraries.

To authenticate to Sensitive Data Protection, set up Application Default Credentials. For more information, see Set up authentication for a local development environment.

import (
	"context"
	"fmt"
	"io"

	dlp "cloud.google.com/go/dlp/apiv2"
	"cloud.google.com/go/dlp/apiv2/dlppb"
)

// deIdentifyTableBucketing de-identifies data using table bucketing
func deIdentifyTableBucketing(w io.Writer, projectID string) error {
	// projectId := "your-project-id"
	// table := "your-table-value"

	row1 := &dlppb.Table_Row{
		Values: []*dlppb.Value{
			{Type: &dlppb.Value_StringValue{StringValue: "22"}},
			{Type: &dlppb.Value_StringValue{StringValue: "Jane Austen"}},
			{Type: &dlppb.Value_StringValue{StringValue: "21"}},
		},
	}

	row2 := &dlppb.Table_Row{
		Values: []*dlppb.Value{
			{Type: &dlppb.Value_StringValue{StringValue: "55"}},
			{Type: &dlppb.Value_StringValue{StringValue: "Mark Twain"}},
			{Type: &dlppb.Value_StringValue{StringValue: "75"}},
		},
	}

	row3 := &dlppb.Table_Row{
		Values: []*dlppb.Value{
			{Type: &dlppb.Value_StringValue{StringValue: "101"}},
			{Type: &dlppb.Value_StringValue{StringValue: "Charles Dickens"}},
			{Type: &dlppb.Value_StringValue{StringValue: "95"}},
		},
	}

	table := &dlppb.Table{
		Headers: []*dlppb.FieldId{
			{Name: "AGE"},
			{Name: "PATIENT"},
			{Name: "HAPPINESS SCORE"},
		},
		Rows: []*dlppb.Table_Row{
			{Values: row1.Values},
			{Values: row2.Values},
			{Values: row3.Values},
		},
	}

	ctx := context.Background()

	// Initialize a client once and reuse it to send multiple requests. Clients
	// are safe to use across goroutines. When the client is no longer needed,
	// call the Close method to cleanup its resources.
	client, err := dlp.NewClient(ctx)
	if err != nil {
		return err
	}

	// Closing the client safely cleans up background resources.
	defer client.Close()

	// Specify what content you want the service to de-identify.
	contentItem := &dlppb.ContentItem{
		DataItem: &dlppb.ContentItem_Table{
			Table: table,
		},
	}

	// Specify how the content should be de-identified.
	fixedSizeBucketingConfig := &dlppb.FixedSizeBucketingConfig{
		BucketSize: 10,
		LowerBound: &dlppb.Value{
			Type: &dlppb.Value_IntegerValue{
				IntegerValue: 0,
			},
		},
		UpperBound: &dlppb.Value{
			Type: &dlppb.Value_IntegerValue{
				IntegerValue: 100,
			},
		},
	}
	primitiveTransformation := &dlppb.PrimitiveTransformation_FixedSizeBucketingConfig{
		FixedSizeBucketingConfig: fixedSizeBucketingConfig,
	}

	// Specify field to be encrypted.
	fieldId := &dlppb.FieldId{
		Name: "HAPPINESS SCORE",
	}

	// Associate the encryption with the specified field.
	fieldTransformation := &dlppb.FieldTransformation{
		Transformation: &dlppb.FieldTransformation_PrimitiveTransformation{
			PrimitiveTransformation: &dlppb.PrimitiveTransformation{
				Transformation: primitiveTransformation,
			},
		},
		Fields: []*dlppb.FieldId{
			fieldId,
		},
	}

	recordTransformations := &dlppb.RecordTransformations{
		FieldTransformations: []*dlppb.FieldTransformation{
			fieldTransformation,
		},
	}

	// Construct the de-identification request to be sent by the client.
	req := &dlppb.DeidentifyContentRequest{
		Parent: fmt.Sprintf("projects/%s/locations/global", projectID),
		DeidentifyConfig: &dlppb.DeidentifyConfig{
			Transformation: &dlppb.DeidentifyConfig_RecordTransformations{
				RecordTransformations: recordTransformations,
			},
		},
		Item: contentItem,
	}
	// Send the request.
	resp, err := client.DeidentifyContent(ctx, req)
	if err != nil {
		return err
	}

	// Print the results.
	fmt.Fprintf(w, "Table after de-identification : %v", resp.GetItem().GetTable())
	return nil
}

Java

To learn how to install and use the client library for Sensitive Data Protection, see Sensitive Data Protection client libraries.

To authenticate to Sensitive Data Protection, set up Application Default Credentials. For more information, see Set up authentication for a local development environment.

import com.google.cloud.dlp.v2.DlpServiceClient;
import com.google.privacy.dlp.v2.ContentItem;
import com.google.privacy.dlp.v2.DeidentifyConfig;
import com.google.privacy.dlp.v2.DeidentifyContentRequest;
import com.google.privacy.dlp.v2.DeidentifyContentResponse;
import com.google.privacy.dlp.v2.FieldId;
import com.google.privacy.dlp.v2.FieldTransformation;
import com.google.privacy.dlp.v2.FixedSizeBucketingConfig;
import com.google.privacy.dlp.v2.LocationName;
import com.google.privacy.dlp.v2.PrimitiveTransformation;
import com.google.privacy.dlp.v2.RecordTransformations;
import com.google.privacy.dlp.v2.Table;
import com.google.privacy.dlp.v2.Table.Row;
import com.google.privacy.dlp.v2.Value;
import java.io.IOException;

public class DeIdentifyTableBucketing {

  public static void main(String[] args) throws IOException {
    // TODO(developer): Replace these variables before running the sample.
    String projectId = "your-project-id";
    Table tableToDeIdentify =
        Table.newBuilder()
            .addHeaders(FieldId.newBuilder().setName("AGE").build())
            .addHeaders(FieldId.newBuilder().setName("PATIENT").build())
            .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build())
            .addRows(
                Row.newBuilder()
                    .addValues(Value.newBuilder().setStringValue("101").build())
                    .addValues(Value.newBuilder().setStringValue("Charles Dickens").build())
                    .addValues(Value.newBuilder().setStringValue("95").build())
                    .build())
            .addRows(
                Row.newBuilder()
                    .addValues(Value.newBuilder().setStringValue("22").build())
                    .addValues(Value.newBuilder().setStringValue("Jane Austen").build())
                    .addValues(Value.newBuilder().setStringValue("21").build())
                    .build())
            .addRows(
                Row.newBuilder()
                    .addValues(Value.newBuilder().setStringValue("55").build())
                    .addValues(Value.newBuilder().setStringValue("Mark Twain").build())
                    .addValues(Value.newBuilder().setStringValue("75").build())
                    .build())
            .build();

    deIdentifyTableBucketing(projectId, tableToDeIdentify);
  }

  public static Table deIdentifyTableBucketing(String projectId, Table tableToDeIdentify)
      throws IOException {
    // Initialize client that will be used to send requests. This client only needs to be created
    // once, and can be reused for multiple requests. After completing all of your requests, call
    // the "close" method on the client to safely clean up any remaining background resources.
    try (DlpServiceClient dlp = DlpServiceClient.create()) {
      // Specify what content you want the service to de-identify.
      ContentItem contentItem = ContentItem.newBuilder().setTable(tableToDeIdentify).build();

      // Specify how the content should be de-identified.
      FixedSizeBucketingConfig fixedSizeBucketingConfig =
          FixedSizeBucketingConfig.newBuilder()
              .setBucketSize(10)
              .setLowerBound(Value.newBuilder().setIntegerValue(0).build())
              .setUpperBound(Value.newBuilder().setIntegerValue(100).build())
              .build();
      PrimitiveTransformation primitiveTransformation =
          PrimitiveTransformation.newBuilder()
              .setFixedSizeBucketingConfig(fixedSizeBucketingConfig)
              .build();

      // Specify field to be encrypted.
      FieldId fieldId = FieldId.newBuilder().setName("HAPPINESS SCORE").build();

      // Associate the encryption with the specified field.
      FieldTransformation fieldTransformation =
          FieldTransformation.newBuilder()
              .setPrimitiveTransformation(primitiveTransformation)
              .addFields(fieldId)
              .build();
      RecordTransformations transformations =
          RecordTransformations.newBuilder().addFieldTransformations(fieldTransformation).build();

      DeidentifyConfig deidentifyConfig =
          DeidentifyConfig.newBuilder().setRecordTransformations(transformations).build();

      // Combine configurations into a request for the service.
      DeidentifyContentRequest request =
          DeidentifyContentRequest.newBuilder()
              .setParent(LocationName.of(projectId, "global").toString())
              .setItem(contentItem)
              .setDeidentifyConfig(deidentifyConfig)
              .build();

      // Send the request and receive response from the service.
      DeidentifyContentResponse response = dlp.deidentifyContent(request);

      // Print the results.
      System.out.println("Table after de-identification: " + response.getItem().getTable());

      return response.getItem().getTable();
    }
  }
}

Node.js

To learn how to install and use the client library for Sensitive Data Protection, see Sensitive Data Protection client libraries.

To authenticate to Sensitive Data Protection, set up Application Default Credentials. For more information, see Set up authentication for a local development environment.

// Imports the Google Cloud Data Loss Prevention library
const DLP = require('@google-cloud/dlp');

// Initialize google DLP Client
const dlp = new DLP.DlpServiceClient();

// The project ID to run the API call under
// const projectId = 'my-project';

// Construct the tabular data
const tablularData = {
  headers: [{name: 'AGE'}, {name: 'PATIENT'}, {name: 'HAPPINESS SCORE'}],
  rows: [
    {
      values: [
        {integerValue: 101},
        {stringValue: 'Charles Dickens'},
        {integerValue: 95},
      ],
    },
    {
      values: [
        {integerValue: 22},
        {stringValue: 'Jane Austen'},
        {integerValue: 21},
      ],
    },
    {
      values: [
        {integerValue: 55},
        {stringValue: 'Mark Twain'},
        {integerValue: 75},
      ],
    },
  ],
};
async function deIdentifyTableBucketing() {
  // Specify field to be de-identified.
  const targetColumn = {name: 'HAPPINESS SCORE'};

  // Specify how the content should be de-identified.
  const bucketingConfig = {
    bucketSize: 10,
    lowerBound: {
      integerValue: 0,
    },
    upperBound: {
      integerValue: 100,
    },
  };

  const primitiveTransformation = {
    fixedSizeBucketingConfig: bucketingConfig,
  };

  // Combine configurations into a request for the service.
  const request = {
    parent: `projects/${projectId}/locations/global`,
    item: {
      table: tablularData,
    },
    deidentifyConfig: {
      recordTransformations: {
        fieldTransformations: [
          {
            fields: [targetColumn],
            primitiveTransformation,
          },
        ],
      },
    },
  };
  // Send the request and receive response from the service
  const [response] = await dlp.deidentifyContent(request);

  // Print the results.
  console.log(
    `Table after de-identification: ${JSON.stringify(response.item.table)}`
  );
}

deIdentifyTableBucketing();

PHP

To learn how to install and use the client library for Sensitive Data Protection, see Sensitive Data Protection client libraries.

To authenticate to Sensitive Data Protection, set up Application Default Credentials. For more information, see Set up authentication for a local development environment.

use Google\Cloud\Dlp\V2\Client\DlpServiceClient;
use Google\Cloud\Dlp\V2\ContentItem;
use Google\Cloud\Dlp\V2\DeidentifyConfig;
use Google\Cloud\Dlp\V2\DeidentifyContentRequest;
use Google\Cloud\Dlp\V2\FieldId;
use Google\Cloud\Dlp\V2\FieldTransformation;
use Google\Cloud\Dlp\V2\FixedSizeBucketingConfig;
use Google\Cloud\Dlp\V2\PrimitiveTransformation;
use Google\Cloud\Dlp\V2\RecordTransformations;
use Google\Cloud\Dlp\V2\Table;
use Google\Cloud\Dlp\V2\Table\Row;
use Google\Cloud\Dlp\V2\Value;

/**
 * De-identify data using table bucketing
 * Transform a column without inspection. To transform a column in which the content is
 * already known, you can skip inspection and specify a transformation directly.
 *
 * @param string $callingProjectId      The Google Cloud project id to use as a parent resource.
 * @param string $inputCsvFile          The input file(csv) path  to deidentify
 * @param string $outputCsvFile         The oupt file path to save deidentify content
 *
 */
function deidentify_table_bucketing(
    // TODO(developer): Replace sample parameters before running the code.
    string $callingProjectId,
    string $inputCsvFile = './test/data/table2.csv',
    string $outputCsvFile = './test/data/deidentify_table_bucketing_output.csv'
): void {
    // Instantiate a client.
    $dlp = new DlpServiceClient();

    // Read a CSV file
    $csvLines = file($inputCsvFile, FILE_IGNORE_NEW_LINES);
    $csvHeaders = explode(',', $csvLines[0]);
    $csvRows = array_slice($csvLines, 1);

    // Convert CSV file into protobuf objects
    $tableHeaders = array_map(function ($csvHeader) {
        return (new FieldId)
            ->setName($csvHeader);
    }, $csvHeaders);

    $tableRows = array_map(function ($csvRow) {
        $rowValues = array_map(function ($csvValue) {
            return (new Value())
                ->setStringValue($csvValue);
        }, explode(',', $csvRow));
        return (new Row())
            ->setValues($rowValues);
    }, $csvRows);

    // Construct the table object
    $tableToDeIdentify = (new Table())
        ->setHeaders($tableHeaders)
        ->setRows($tableRows);

    // Specify what content you want the service to de-identify.
    $contentItem = (new ContentItem())
        ->setTable($tableToDeIdentify);

    // Specify how the content should be de-identified.
    $fixedSizeBucketingConfig = (new FixedSizeBucketingConfig())
        ->setBucketSize(10)
        ->setLowerBound((new Value())
            ->setIntegerValue(10))
        ->setUpperBound((new Value())
            ->setIntegerValue(100));

    $primitiveTransformation = (new PrimitiveTransformation())
        ->setFixedSizeBucketingConfig($fixedSizeBucketingConfig);

    // Specify the field to to apply bucketing transform on
    $fieldId = (new FieldId())
        ->setName('HAPPINESS_SCORE');

    // Associate the encryption with the specified field.
    $fieldTransformation = (new FieldTransformation())
        ->setPrimitiveTransformation($primitiveTransformation)
        ->setFields([$fieldId]);

    $recordTransformations = (new RecordTransformations())
        ->setFieldTransformations([$fieldTransformation]);

    // Create the deidentification configuration object
    $deidentifyConfig = (new DeidentifyConfig())
        ->setRecordTransformations($recordTransformations);

    $parent = "projects/$callingProjectId/locations/global";

    // Run request
    $deidentifyContentRequest = (new DeidentifyContentRequest())
        ->setParent($parent)
        ->setDeidentifyConfig($deidentifyConfig)
        ->setItem($contentItem);
    $response = $dlp->deidentifyContent($deidentifyContentRequest);

    // Print results
    $csvRef = fopen($outputCsvFile, 'w');
    fputcsv($csvRef, $csvHeaders);
    foreach ($response->getItem()->getTable()->getRows() as $tableRow) {
        $values = array_map(function ($tableValue) {
            return $tableValue->getStringValue();
        }, iterator_to_array($tableRow->getValues()));
        fputcsv($csvRef, $values);
    };
    printf($outputCsvFile);
}

Python

To learn how to install and use the client library for Sensitive Data Protection, see Sensitive Data Protection client libraries.

To authenticate to Sensitive Data Protection, set up Application Default Credentials. For more information, see Set up authentication for a local development environment.

from typing import Dict, List, Union

import google.cloud.dlp
from google.cloud.dlp_v2 import types


def deidentify_table_bucketing(
    project: str,
    table_data: Dict[str, Union[List[str], List[List[str]]]],
    deid_content_list: List[str],
    bucket_size: int,
    bucketing_lower_bound: int,
    bucketing_upper_bound: int,
) -> types.dlp.Table:
    """Uses the Data Loss Prevention API to de-identify sensitive data in a
    table by replacing them with fixed size bucket ranges.
    Args:
        project: The Google Cloud project id to use as a parent resource.
        table_data: Dictionary representing table data.
        deid_content_list: A list of fields in table to de-identify.
        bucket_size: Size of each bucket for fixed sized bucketing
            (except for minimum and maximum buckets). So if ``bucketing_lower_bound`` = 10,
            ``bucketing_upper_bound`` = 89, and ``bucket_size`` = 10, then the
            following buckets would be used: -10, 10-20, 20-30, 30-40,
            40-50, 50-60, 60-70, 70-80, 80-89, 89+.
       bucketing_lower_bound: Lower bound value of buckets.
       bucketing_upper_bound:  Upper bound value of buckets.

    Returns:
       De-identified table is returned;
       the response from the API is also printed to the terminal.

    Example:
    >> $ python deidentify_table_bucketing.py \
        '{"header": ["email", "phone number", "age"],
        "rows": [["[e-mail]", "4232342345", "35"],
        ["[e-mail]", "4253458383", "68"]]}' \
        ["age"] 10 0 100
        >>  '{"header": ["email", "phone number", "age"],
            "rows": [["[e-mail]", "4232342345", "30:40"],
            ["[e-mail]", "4253458383", "60:70"]]}'
    """

    # Instantiate a client.
    dlp = google.cloud.dlp_v2.DlpServiceClient()

    # Convert the project id into a full resource id.
    parent = f"projects/{project}/locations/global"

    # Construct the `table`. For more details on the table schema, please see
    # https://cloud.google.com/dlp/docs/reference/rest/v2/ContentItem#Table
    headers = [{"name": val} for val in table_data["header"]]
    rows = []
    for row in table_data["rows"]:
        rows.append({"values": [{"string_value": cell_val} for cell_val in row]})

    table = {"headers": headers, "rows": rows}

    # Construct the `item`.
    item = {"table": table}

    # Construct fixed sized bucketing configuration
    fixed_size_bucketing_config = {
        "bucket_size": bucket_size,
        "lower_bound": {"integer_value": bucketing_lower_bound},
        "upper_bound": {"integer_value": bucketing_upper_bound},
    }

    # Specify fields to be de-identified
    deid_content_list = [{"name": _i} for _i in deid_content_list]

    # Construct Deidentify Config
    deidentify_config = {
        "record_transformations": {
            "field_transformations": [
                {
                    "fields": deid_content_list,
                    "primitive_transformation": {
                        "fixed_size_bucketing_config": fixed_size_bucketing_config
                    },
                }
            ]
        }
    }

    # Call the API.
    response = dlp.deidentify_content(
        request={"parent": parent, "deidentify_config": deidentify_config, "item": item}
    )

    # Print the results.
    print(f"Table after de-identification: {response.item.table}")

    # Return the response.
    return response.item.table

What's next

To search and filter code samples for other Google Cloud products, see the Google Cloud sample browser.