Code sample to demonstrate lifecycle of different Dataplex Catalog resources and their interactions. Create aspect type, entry type, entry group and entry, retrieve entry and clean up created resources.
Code sample
Java
Before trying this sample, follow the Java setup instructions in the Dataplex quickstart using client libraries. For more information, see the Dataplex Java API reference documentation.
To authenticate to Dataplex, set up Application Default Credentials. For more information, see Set up authentication for a local development environment.
import com.google.cloud.dataplex.v1.Aspect;
import com.google.cloud.dataplex.v1.AspectType;
import com.google.cloud.dataplex.v1.CatalogServiceClient;
import com.google.cloud.dataplex.v1.Entry;
import com.google.cloud.dataplex.v1.EntryGroup;
import com.google.cloud.dataplex.v1.EntryGroupName;
import com.google.cloud.dataplex.v1.EntryName;
import com.google.cloud.dataplex.v1.EntrySource;
import com.google.cloud.dataplex.v1.EntryType;
import com.google.cloud.dataplex.v1.EntryView;
import com.google.cloud.dataplex.v1.GetEntryRequest;
import com.google.cloud.dataplex.v1.LocationName;
import com.google.cloud.dataplex.v1.SearchEntriesRequest;
import com.google.cloud.dataplex.v1.SearchEntriesResult;
import com.google.protobuf.Struct;
import com.google.protobuf.Value;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
public class Quickstart {
public static void main(String[] args) {
// TODO(developer): Replace these variables before running the sample.
String projectId = "MY_PROJECT_ID";
// Available locations: https://cloud.google.com/dataplex/docs/locations
String location = "MY_LOCATION";
// Variables below can be replaced with custom values or defaults can be kept
String aspectTypeId = "dataplex-quickstart-aspect-type";
String entryTypeId = "dataplex-quickstart-entry-type";
String entryGroupId = "dataplex-quickstart-entry-group";
String entryId = "dataplex-quickstart-entry";
quickstart(projectId, location, aspectTypeId, entryTypeId, entryGroupId, entryId);
}
// Method to demonstrate lifecycle of different Dataplex resources and their interactions.
// Method creates Aspect Type, Entry Type, Entry Group and Entry, retrieves Entry
// and cleans up created resources.
public static void quickstart(
String projectId,
String location,
String aspectTypeId,
String entryTypeId,
String entryGroupId,
String entryId) {
// Initialize client that will be used to send requests. This client only needs to be created
// once, and can be reused for multiple requests.
try (CatalogServiceClient client = CatalogServiceClient.create()) {
// 0) Prepare variables used in following steps
LocationName globalLocationName = LocationName.of(projectId, "global");
LocationName specificLocationName = LocationName.of(projectId, location);
// 1) Create Aspect Type that will be attached to Entry Type
AspectType.MetadataTemplate aspectField =
AspectType.MetadataTemplate.newBuilder()
// The name must follow regex ^(([a-zA-Z]{1})([\\w\\-_]{0,62}))$
// That means name must only contain alphanumeric character or dashes or underscores,
// start with an alphabet, and must be less than 63 characters.
.setName("example_field")
// Metadata Template is recursive structure,
// primitive types such as "string" or "integer" indicate leaf node,
// complex types such as "record" or "array" would require nested Metadata Template
.setType("string")
.setIndex(1)
.setAnnotations(
AspectType.MetadataTemplate.Annotations.newBuilder()
.setDescription("example field to be filled during entry creation")
.build())
.setConstraints(
AspectType.MetadataTemplate.Constraints.newBuilder()
// Specifies if field will be required in Aspect Type.
.setRequired(true)
.build())
.build();
AspectType aspectType =
AspectType.newBuilder()
.setDescription("aspect type for dataplex quickstart")
.setMetadataTemplate(
AspectType.MetadataTemplate.newBuilder()
.setName("example_template")
.setType("record")
// Aspect Type fields, that themselves are Metadata Templates
.addAllRecordFields(List.of(aspectField))
.build())
.build();
AspectType createdAspectType =
client
.createAspectTypeAsync(
// Aspect Type is created in "global" location to highlight, that resources from
// "global" region can be attached to Entry created in specific location
globalLocationName, aspectType, aspectTypeId)
.get();
System.out.println("Step 1: Created aspect type -> " + createdAspectType.getName());
// 2) Create Entry Type, of which type Entry will be created
EntryType entryType =
EntryType.newBuilder()
.setDescription("entry type for dataplex quickstart")
.addRequiredAspects(
EntryType.AspectInfo.newBuilder()
// Aspect Type created in step 1
.setType(
String.format(
"projects/%s/locations/global/aspectTypes/%s",
projectId, aspectTypeId))
.build())
.build();
EntryType createdEntryType =
client
// Entry Type is created in "global" location to highlight, that resources from
// "global" region can be attached to Entry created in specific location
.createEntryTypeAsync(globalLocationName, entryType, entryTypeId)
.get();
System.out.println("Step 2: Created entry type -> " + createdEntryType.getName());
// 3) Create Entry Group in which Entry will be located
EntryGroup entryGroup =
EntryGroup.newBuilder().setDescription("entry group for dataplex quickstart").build();
EntryGroup createdEntryGroup =
client
// Entry Group is created for specific location
.createEntryGroupAsync(specificLocationName, entryGroup, entryGroupId)
.get();
System.out.println("Step 3: Created entry group -> " + createdEntryGroup.getName());
// 4) Create Entry
// Wait 30 seconds to allow previously created resources to propagate
Thread.sleep(30000);
String aspectKey = String.format("%s.global.%s", projectId, aspectTypeId);
Entry entry =
Entry.newBuilder()
.setEntryType(
// Entry is an instance of Entry Type created in step 2
String.format(
"projects/%s/locations/global/entryTypes/%s", projectId, entryTypeId))
.setEntrySource(
EntrySource.newBuilder().setDescription("entry for dataplex quickstart").build())
.putAllAspects(
Map.of(
// Attach Aspect that is an instance of Aspect Type created in step 1
aspectKey,
Aspect.newBuilder()
.setAspectType(
String.format(
"projects/%s/locations/global/aspectTypes/%s",
projectId, aspectTypeId))
.setData(
Struct.newBuilder()
.putFields(
"example_field",
Value.newBuilder()
.setStringValue("example value for the field")
.build())
.build())
.build()))
.build();
Entry createdEntry =
client.createEntry(
// Entry is created in specific location, but it is still possible to link it with
// resources (Aspect Type and Entry Type) from "global" location
EntryGroupName.of(projectId, location, entryGroupId), entry, entryId);
System.out.println("Step 4: Created entry -> " + createdEntry.getName());
// 5) Retrieve created Entry
GetEntryRequest getEntryRequest =
GetEntryRequest.newBuilder()
.setName(EntryName.of(projectId, location, entryGroupId, entryId).toString())
.setView(EntryView.FULL)
.build();
Entry retrievedEntry = client.getEntry(getEntryRequest);
System.out.println("Step 5: Retrieved entry -> " + retrievedEntry.getName());
retrievedEntry
.getAspectsMap()
.values()
.forEach(
retrievedAspect -> {
System.out.println("Retrieved aspect for entry:");
System.out.println(" * aspect type -> " + retrievedAspect.getAspectType());
System.out.println(
" * aspect field value -> "
+ retrievedAspect
.getData()
.getFieldsMap()
.get("example_field")
.getStringValue());
});
// 6) Use Search capabilities to find Entry
// Wait 30 seconds to allow resources to propagate to Search
System.out.println("Step 6: Waiting for resources to propagate to Search...");
Thread.sleep(30000);
SearchEntriesRequest searchEntriesRequest =
SearchEntriesRequest.newBuilder()
.setName(globalLocationName.toString())
.setQuery("name:dataplex-quickstart-entry")
.build();
CatalogServiceClient.SearchEntriesPagedResponse searchEntriesResponse =
client.searchEntries(searchEntriesRequest);
List<Entry> entriesFromSearch =
searchEntriesResponse.getPage().getResponse().getResultsList().stream()
.map(SearchEntriesResult::getDataplexEntry)
.collect(Collectors.toList());
System.out.println("Entries found in Search:");
// Please note in output that Entry Group and Entry Type are also represented as Entries
entriesFromSearch.forEach(
entryFromSearch -> System.out.println(" * " + entryFromSearch.getName()));
// 7) Clean created resources
client
.deleteEntryGroupAsync(
String.format(
"projects/%s/locations/%s/entryGroups/%s", projectId, location, entryGroupId))
.get();
client
.deleteEntryTypeAsync(
String.format("projects/%s/locations/global/entryTypes/%s", projectId, entryTypeId))
.get();
client
.deleteAspectTypeAsync(
String.format("projects/%s/locations/global/aspectTypes/%s", projectId, aspectTypeId))
.get();
System.out.println("Step 7: Successfully cleaned up resources");
} catch (IOException | InterruptedException | ExecutionException e) {
System.err.println("Error during quickstart execution: " + e);
}
}
}Python
Before trying this sample, follow the Python setup instructions in the Dataplex quickstart using client libraries. For more information, see the Dataplex Python API reference documentation.
To authenticate to Dataplex, set up Application Default Credentials. For more information, see Set up authentication for a local development environment.
import time
from google.cloud import dataplex_v1
from google.protobuf import struct_pb2
# Method to demonstrate lifecycle of different Dataplex resources and their interactions.
# Method creates Aspect Type, Entry Type, Entry Group and Entry, retrieves Entry
# and cleans up created resources.
def quickstart(
project_id: str,
location: str,
aspect_type_id: str,
entry_type_id: str,
entry_group_id: str,
entry_id: str,
) -> None:
# Initialize client that will be used to send requests across threads. This
# client only needs to be created once, and can be reused for multiple requests.
# After completing all of your requests, call the "__exit__()" method to safely
# clean up any remaining background resources. Alternatively, use the client as
# a context manager.
with dataplex_v1.CatalogServiceClient() as client:
# 0) Prepare variables used in following steps
global_parent = f"projects/{project_id}/locations/global"
specific_location_parent = f"projects/{project_id}/locations/{location}"
# 1) Create Aspect Type that will be attached to Entry Type
aspect_field = dataplex_v1.AspectType.MetadataTemplate(
# The name must follow regex ^(([a-zA-Z]{1})([\\w\\-_]{0,62}))$
# That means name must only contain alphanumeric character or dashes or underscores,
# start with an alphabet, and must be less than 63 characters.
name="example_field",
# Metadata Template is recursive structure,
# primitive types such as "string" or "integer" indicate leaf node,
# complex types such as "record" or "array" would require nested Metadata Template
type="string",
index=1,
annotations=dataplex_v1.AspectType.MetadataTemplate.Annotations(
description="example field to be filled during entry creation"
),
constraints=dataplex_v1.AspectType.MetadataTemplate.Constraints(
# Specifies if field will be required in Aspect Type.
required=True
),
)
aspect_type = dataplex_v1.AspectType(
description="aspect type for dataplex quickstart",
metadata_template=dataplex_v1.AspectType.MetadataTemplate(
name="example_template",
type="record",
# Aspect Type fields, that themselves are Metadata Templates.
record_fields=[aspect_field],
),
)
aspect_type_create_operation = client.create_aspect_type(
# Aspect Type is created in "global" location to highlight, that resources from
# "global" region can be attached to Entry created in specific location
parent=global_parent,
aspect_type=aspect_type,
aspect_type_id=aspect_type_id,
)
created_aspect_type = aspect_type_create_operation.result(60)
print(f"Step 1: Created aspect type -> {created_aspect_type.name}")
# 2) Create Entry Type, of which type Entry will be created
entry_type = dataplex_v1.EntryType(
description="entry type for dataplex quickstart",
required_aspects=[
dataplex_v1.EntryType.AspectInfo(
# Aspect Type created in step 1
type=f"projects/{project_id}/locations/global/aspectTypes/{aspect_type_id}"
)
],
)
entry_type_create_operation = client.create_entry_type(
# Entry Type is created in "global" location to highlight, that resources from
# "global" region can be attached to Entry created in specific location
parent=global_parent,
entry_type=entry_type,
entry_type_id=entry_type_id,
)
created_entry_type = entry_type_create_operation.result(60)
print(f"Step 2: Created entry type -> {created_entry_type.name}")
# 3) Create Entry Group in which Entry will be located
entry_group = dataplex_v1.EntryGroup(
description="entry group for dataplex quickstart"
)
entry_group_create_operation = client.create_entry_group(
# Entry Group is created for specific location
parent=specific_location_parent,
entry_group=entry_group,
entry_group_id=entry_group_id,
)
created_entry_group = entry_group_create_operation.result(60)
print(f"Step 3: Created entry group -> {created_entry_group.name}")
# 4) Create Entry
# Wait 10 second to allow previously created resources to propagate
time.sleep(10)
aspect_key = f"{project_id}.global.{aspect_type_id}"
entry = dataplex_v1.Entry(
# Entry is an instance of Entry Type created in step 2
entry_type=f"projects/{project_id}/locations/global/entryTypes/{entry_type_id}",
entry_source=dataplex_v1.EntrySource(
description="entry for dataplex quickstart"
),
aspects={
# Attach Aspect that is an instance of Aspect Type created in step 1
aspect_key: dataplex_v1.Aspect(
aspect_type=f"projects/{project_id}/locations/global/aspectTypes/{aspect_type_id}",
data=struct_pb2.Struct(
fields={
"example_field": struct_pb2.Value(
string_value="example value for the field"
),
}
),
)
},
)
created_entry = client.create_entry(
# Entry is created in specific location, but it is still possible to link it with
# resources (Aspect Type and Entry Type) from "global" location
parent=f"projects/{project_id}/locations/{location}/entryGroups/{entry_group_id}",
entry=entry,
entry_id=entry_id,
)
print(f"Step 4: Created entry -> {created_entry.name}")
# 5) Retrieve created Entry
get_entry_request = dataplex_v1.GetEntryRequest(
name=f"projects/{project_id}/locations/{location}/entryGroups/{entry_group_id}/entries/{entry_id}",
view=dataplex_v1.EntryView.FULL,
)
retrieved_entry = client.get_entry(request=get_entry_request)
print(f"Step 5: Retrieved entry -> {retrieved_entry.name}")
for retrieved_aspect in retrieved_entry.aspects.values():
print("Retrieved aspect for entry:")
print(f" * aspect type -> {retrieved_aspect.aspect_type}")
print(f" * aspect field value -> {retrieved_aspect.data['example_field']}")
# 6) Use Search capabilities to find Entry
# Wait 30 second to allow resources to propagate to Search
print("Step 6: Waiting for resources to propagate to Search...")
time.sleep(30)
search_entries_request = dataplex_v1.SearchEntriesRequest(
name=global_parent, query="name:dataplex-quickstart-entry"
)
results = client.search_entries(search_entries_request)
search_entries_response = results._response
entries_from_search = [
result.dataplex_entry for result in search_entries_response.results
]
print("Entries found in Search:")
# Please note in output that Entry Group and Entry Type are also represented as Entries
for entry_from_search in entries_from_search:
print(f" * {entry_from_search.name}")
# 7) Clean created resources
client.delete_entry_group(
name=f"projects/{project_id}/locations/{location}/entryGroups/{entry_group_id}"
)
client.delete_entry_type(
name=f"projects/{project_id}/locations/global/entryTypes/{entry_type_id}"
)
client.delete_aspect_type(
name=f"projects/{project_id}/locations/global/aspectTypes/{aspect_type_id}"
)
print("Step 7: Successfully cleaned up resources")
if __name__ == "__main__":
# TODO(developer): Replace these variables before running the sample.
project_id = "MY_PROJECT_ID"
# Available locations: https://cloud.google.com/dataplex/docs/locations
location = "MY_LOCATION"
# Variables below can be replaced with custom values or defaults can be kept
aspect_type_id = "dataplex-quickstart-aspect-type"
entry_type_id = "dataplex-quickstart-entry-type"
entry_group_id = "dataplex-quickstart-entry-group"
entry_id = "dataplex-quickstart-entry"
quickstart(
project_id, location, aspect_type_id, entry_type_id, entry_group_id, entry_id
)What's next
To search and filter code samples for other Google Cloud products, see the Google Cloud sample browser.