Index
CatalogService(interface)ContentService(interface)DataScanService(interface)DataTaxonomyService(interface)DataplexService(interface)MetadataService(interface)Action(message)Action.Category(enum)Action.FailedSecurityPolicyApply(message)Action.IncompatibleDataSchema(message)Action.IncompatibleDataSchema.SchemaChange(enum)Action.InvalidDataFormat(message)Action.InvalidDataOrganization(message)Action.InvalidDataPartition(message)Action.InvalidDataPartition.PartitionStructure(enum)Action.MissingData(message)Action.MissingResource(message)Action.UnauthorizedResource(message)Aspect(message)AspectSource(message)AspectType(message)AspectType.Authorization(message)AspectType.MetadataTemplate(message)AspectType.MetadataTemplate.Annotations(message)AspectType.MetadataTemplate.Constraints(message)AspectType.MetadataTemplate.EnumValue(message)Asset(message)Asset.DiscoverySpec(message)Asset.DiscoverySpec.CsvOptions(message)Asset.DiscoverySpec.JsonOptions(message)Asset.DiscoveryStatus(message)Asset.DiscoveryStatus.State(enum)Asset.DiscoveryStatus.Stats(message)Asset.ResourceSpec(message)Asset.ResourceSpec.AccessMode(enum)Asset.ResourceSpec.Type(enum)Asset.ResourceStatus(message)Asset.ResourceStatus.State(enum)Asset.SecurityStatus(message)Asset.SecurityStatus.State(enum)AssetStatus(message)CancelJobRequest(message)CancelMetadataJobRequest(message)Content(message)Content.Notebook(message)Content.Notebook.KernelType(enum)Content.SqlScript(message)Content.SqlScript.QueryEngine(enum)CreateAspectTypeRequest(message)CreateAssetRequest(message)CreateContentRequest(message)CreateDataAttributeBindingRequest(message)CreateDataAttributeRequest(message)CreateDataScanRequest(message)CreateDataTaxonomyRequest(message)CreateEntityRequest(message)CreateEntryGroupRequest(message)CreateEntryRequest(message)CreateEntryTypeRequest(message)CreateEnvironmentRequest(message)CreateLakeRequest(message)CreateMetadataJobRequest(message)CreatePartitionRequest(message)CreateTaskRequest(message)CreateZoneRequest(message)DataAccessSpec(message)DataAttribute(message)DataAttributeBinding(message)DataAttributeBinding.Path(message)DataProfileResult(message)DataProfileResult.PostScanActionsResult(message)DataProfileResult.PostScanActionsResult.BigQueryExportResult(message)DataProfileResult.PostScanActionsResult.BigQueryExportResult.State(enum)DataProfileResult.Profile(message)DataProfileResult.Profile.Field(message)DataProfileResult.Profile.Field.ProfileInfo(message)DataProfileResult.Profile.Field.ProfileInfo.DoubleFieldInfo(message)DataProfileResult.Profile.Field.ProfileInfo.IntegerFieldInfo(message)DataProfileResult.Profile.Field.ProfileInfo.StringFieldInfo(message)DataProfileResult.Profile.Field.ProfileInfo.TopNValue(message)DataProfileSpec(message)DataProfileSpec.PostScanActions(message)DataProfileSpec.PostScanActions.BigQueryExport(message)DataProfileSpec.SelectedFields(message)DataQualityColumnResult(message)DataQualityDimension(message)DataQualityDimensionResult(message)DataQualityResult(message)DataQualityResult.PostScanActionsResult(message)DataQualityResult.PostScanActionsResult.BigQueryExportResult(message)DataQualityResult.PostScanActionsResult.BigQueryExportResult.State(enum)DataQualityRule(message)DataQualityRule.NonNullExpectation(message)DataQualityRule.RangeExpectation(message)DataQualityRule.RegexExpectation(message)DataQualityRule.RowConditionExpectation(message)DataQualityRule.SetExpectation(message)DataQualityRule.SqlAssertion(message)DataQualityRule.StatisticRangeExpectation(message)DataQualityRule.StatisticRangeExpectation.ColumnStatistic(enum)DataQualityRule.TableConditionExpectation(message)DataQualityRule.UniquenessExpectation(message)DataQualityRuleResult(message)DataQualityScanRuleResult(message)DataQualityScanRuleResult.EvaluationType(enum)DataQualityScanRuleResult.Result(enum)DataQualityScanRuleResult.RuleType(enum)DataQualitySpec(message)DataQualitySpec.PostScanActions(message)DataQualitySpec.PostScanActions.BigQueryExport(message)DataQualitySpec.PostScanActions.JobEndTrigger(message)DataQualitySpec.PostScanActions.JobFailureTrigger(message)DataQualitySpec.PostScanActions.NotificationReport(message)DataQualitySpec.PostScanActions.Recipients(message)DataQualitySpec.PostScanActions.ScoreThresholdTrigger(message)DataScan(message)DataScan.ExecutionSpec(message)DataScan.ExecutionStatus(message)DataScanEvent(message)DataScanEvent.DataProfileAppliedConfigs(message)DataScanEvent.DataProfileResult(message)DataScanEvent.DataQualityAppliedConfigs(message)DataScanEvent.DataQualityResult(message)DataScanEvent.PostScanActionsResult(message)DataScanEvent.PostScanActionsResult.BigQueryExportResult(message)DataScanEvent.PostScanActionsResult.BigQueryExportResult.State(enum)DataScanEvent.ScanType(enum)DataScanEvent.Scope(enum)DataScanEvent.State(enum)DataScanEvent.Trigger(enum)DataScanJob(message)DataScanJob.State(enum)DataScanType(enum)DataSource(message)DataTaxonomy(message)DeleteAspectTypeRequest(message)DeleteAssetRequest(message)DeleteContentRequest(message)DeleteDataAttributeBindingRequest(message)DeleteDataAttributeRequest(message)DeleteDataScanRequest(message)DeleteDataTaxonomyRequest(message)DeleteEntityRequest(message)DeleteEntryGroupRequest(message)DeleteEntryRequest(message)DeleteEntryTypeRequest(message)DeleteEnvironmentRequest(message)DeleteLakeRequest(message)DeletePartitionRequest(message)DeleteTaskRequest(message)DeleteZoneRequest(message)DiscoveryEvent(message)DiscoveryEvent.ActionDetails(message)DiscoveryEvent.ConfigDetails(message)DiscoveryEvent.EntityDetails(message)DiscoveryEvent.EntityType(enum)DiscoveryEvent.EventType(enum)DiscoveryEvent.PartitionDetails(message)Entity(message)Entity.CompatibilityStatus(message)Entity.CompatibilityStatus.Compatibility(message)Entity.Type(enum)Entry(message)EntryGroup(message)EntrySource(message)EntrySource.Ancestor(message)EntryType(message)EntryType.AspectInfo(message)EntryType.Authorization(message)EntryView(enum)Environment(message)Environment.Endpoints(message)Environment.InfrastructureSpec(message)Environment.InfrastructureSpec.ComputeResources(message)Environment.InfrastructureSpec.OsImageRuntime(message)Environment.SessionSpec(message)Environment.SessionStatus(message)GenerateDataQualityRulesRequest(message)GenerateDataQualityRulesResponse(message)GetAspectTypeRequest(message)GetAssetRequest(message)GetContentRequest(message)GetContentRequest.ContentView(enum)GetDataAttributeBindingRequest(message)GetDataAttributeRequest(message)GetDataScanJobRequest(message)GetDataScanJobRequest.DataScanJobView(enum)GetDataScanRequest(message)GetDataScanRequest.DataScanView(enum)GetDataTaxonomyRequest(message)GetEntityRequest(message)GetEntityRequest.EntityView(enum)GetEntryGroupRequest(message)GetEntryRequest(message)GetEntryTypeRequest(message)GetEnvironmentRequest(message)GetJobRequest(message)GetLakeRequest(message)GetMetadataJobRequest(message)GetPartitionRequest(message)GetTaskRequest(message)GetZoneRequest(message)GovernanceEvent(message)GovernanceEvent.Entity(message)GovernanceEvent.Entity.EntityType(enum)GovernanceEvent.EventType(enum)ImportItem(message)Job(message)Job.Service(enum)Job.State(enum)Job.Trigger(enum)JobEvent(message)JobEvent.ExecutionTrigger(enum)JobEvent.Service(enum)JobEvent.State(enum)JobEvent.Type(enum)Lake(message)Lake.Metastore(message)Lake.MetastoreStatus(message)Lake.MetastoreStatus.State(enum)ListActionsResponse(message)ListAspectTypesRequest(message)ListAspectTypesResponse(message)ListAssetActionsRequest(message)ListAssetsRequest(message)ListAssetsResponse(message)ListContentRequest(message)ListContentResponse(message)ListDataAttributeBindingsRequest(message)ListDataAttributeBindingsResponse(message)ListDataAttributesRequest(message)ListDataAttributesResponse(message)ListDataScanJobsRequest(message)ListDataScanJobsResponse(message)ListDataScansRequest(message)ListDataScansResponse(message)ListDataTaxonomiesRequest(message)ListDataTaxonomiesResponse(message)ListEntitiesRequest(message)ListEntitiesRequest.EntityView(enum)ListEntitiesResponse(message)ListEntriesRequest(message)ListEntriesResponse(message)ListEntryGroupsRequest(message)ListEntryGroupsResponse(message)ListEntryTypesRequest(message)ListEntryTypesResponse(message)ListEnvironmentsRequest(message)ListEnvironmentsResponse(message)ListJobsRequest(message)ListJobsResponse(message)ListLakeActionsRequest(message)ListLakesRequest(message)ListLakesResponse(message)ListMetadataJobsRequest(message)ListMetadataJobsResponse(message)ListPartitionsRequest(message)ListPartitionsResponse(message)ListSessionsRequest(message)ListSessionsResponse(message)ListTasksRequest(message)ListTasksResponse(message)ListZoneActionsRequest(message)ListZonesRequest(message)ListZonesResponse(message)LookupEntryRequest(message)MetadataJob(message)MetadataJob.ImportJobResult(message)MetadataJob.ImportJobSpec(message)MetadataJob.ImportJobSpec.ImportJobScope(message)MetadataJob.ImportJobSpec.LogLevel(enum)MetadataJob.ImportJobSpec.SyncMode(enum)MetadataJob.Status(message)MetadataJob.Status.State(enum)MetadataJob.Type(enum)OperationMetadata(message)Partition(message)ResourceAccessSpec(message)RunDataScanRequest(message)RunDataScanResponse(message)RunTaskRequest(message)RunTaskResponse(message)ScannedData(message)ScannedData.IncrementalField(message)Schema(message)Schema.Mode(enum)Schema.PartitionField(message)Schema.PartitionStyle(enum)Schema.SchemaField(message)Schema.Type(enum)SearchEntriesRequest(message)SearchEntriesResponse(message)SearchEntriesResult(message)SearchEntriesResult.Snippets(message) (deprecated)Session(message)SessionEvent(message)SessionEvent.EventType(enum)SessionEvent.QueryDetail(message)SessionEvent.QueryDetail.Engine(enum)State(enum)StorageAccess(message)StorageAccess.AccessMode(enum)StorageFormat(message)StorageFormat.CompressionFormat(enum)StorageFormat.CsvOptions(message)StorageFormat.Format(enum)StorageFormat.IcebergOptions(message)StorageFormat.JsonOptions(message)StorageSystem(enum)Task(message)Task.ExecutionSpec(message)Task.ExecutionStatus(message)Task.InfrastructureSpec(message)Task.InfrastructureSpec.BatchComputeResources(message)Task.InfrastructureSpec.ContainerImageRuntime(message)Task.InfrastructureSpec.VpcNetwork(message)Task.NotebookTaskConfig(message)Task.SparkTaskConfig(message)Task.TriggerSpec(message)Task.TriggerSpec.Type(enum)TransferStatus(enum)Trigger(message)Trigger.OnDemand(message)Trigger.Schedule(message)UpdateAspectTypeRequest(message)UpdateAssetRequest(message)UpdateContentRequest(message)UpdateDataAttributeBindingRequest(message)UpdateDataAttributeRequest(message)UpdateDataScanRequest(message)UpdateDataTaxonomyRequest(message)UpdateEntityRequest(message)UpdateEntryGroupRequest(message)UpdateEntryRequest(message)UpdateEntryTypeRequest(message)UpdateEnvironmentRequest(message)UpdateLakeRequest(message)UpdateTaskRequest(message)UpdateZoneRequest(message)Zone(message)Zone.DiscoverySpec(message)Zone.DiscoverySpec.CsvOptions(message)Zone.DiscoverySpec.JsonOptions(message)Zone.ResourceSpec(message)Zone.ResourceSpec.LocationType(enum)Zone.Type(enum)
CatalogService
The primary resources offered by this service are EntryGroups, EntryTypes, AspectTypes, and Entries. They collectively let data administrators organize, manage, secure, and catalog data located across cloud projects in their organization in a variety of storage systems, including Cloud Storage and BigQuery.
| CancelMetadataJob |
|---|
|
Cancels a metadata job. If you cancel a metadata import job that is in progress, the changes in the job might be partially applied. We recommend that you reset the state of the entry groups in your project by running another metadata job that reverts the changes from the canceled job.
|
| CreateAspectType |
|---|
|
Creates an AspectType.
|
| CreateEntry |
|---|
|
Creates an Entry.
|
| CreateEntryGroup |
|---|
|
Creates an EntryGroup.
|
| CreateEntryType |
|---|
|
Creates an EntryType.
|
| CreateMetadataJob |
|---|
|
Creates a metadata job. For example, use a metadata job to import Dataplex Catalog entries and aspects from a third-party system into Dataplex.
|
| DeleteAspectType |
|---|
|
Deletes an AspectType.
|
| DeleteEntry |
|---|
|
Deletes an Entry.
|
| DeleteEntryGroup |
|---|
|
Deletes an EntryGroup.
|
| DeleteEntryType |
|---|
|
Deletes an EntryType.
|
| GetAspectType |
|---|
|
Gets an AspectType.
|
| GetEntry |
|---|
|
Gets an Entry. Caution: The BigQuery metadata that is stored in Dataplex Catalog is changing. For more information, see Changes to BigQuery metadata stored in Dataplex Catalog.
|
| GetEntryGroup |
|---|
|
Gets an EntryGroup.
|
| GetEntryType |
|---|
|
Gets an EntryType.
|
| GetMetadataJob |
|---|
|
Gets a metadata job.
|
| ListAspectTypes |
|---|
|
Lists AspectType resources in a project and location.
|
| ListEntries |
|---|
|
Lists Entries within an EntryGroup.
|
| ListEntryGroups |
|---|
|
Lists EntryGroup resources in a project and location.
|
| ListEntryTypes |
|---|
|
Lists EntryType resources in a project and location.
|
| ListMetadataJobs |
|---|
|
Lists metadata jobs.
|
| LookupEntry |
|---|
|
Looks up a single Entry by name using the permission on the source system. Caution: The BigQuery metadata that is stored in Dataplex Catalog is changing. For more information, see Changes to BigQuery metadata stored in Dataplex Catalog.
|
| SearchEntries |
|---|
|
Searches for Entries matching the given query and scope.
|
| UpdateAspectType |
|---|
|
Updates an AspectType.
|
| UpdateEntry |
|---|
|
Updates an Entry.
|
| UpdateEntryGroup |
|---|
|
Updates an EntryGroup.
|
| UpdateEntryType |
|---|
|
Updates an EntryType.
|
ContentService
ContentService manages Notebook and SQL Scripts for Dataplex.
| CreateContent |
|---|
|
Create a content.
|
| DeleteContent |
|---|
|
Delete a content.
|
| GetContent |
|---|
|
Get a content resource.
|
| GetIamPolicy |
|---|
|
Gets the access control policy for a contentitem resource. A Caller must have Google IAM
|
| ListContent |
|---|
|
List content.
|
| SetIamPolicy |
|---|
|
Sets the access control policy on the specified contentitem resource. Replaces any existing policy. Caller must have Google IAM
|
| TestIamPermissions |
|---|
|
Returns the caller's permissions on a resource. If the resource does not exist, an empty set of permissions is returned (a A caller is not required to have Google IAM permission to make this request. Note: This operation is designed to be used for building permission-aware UIs and command-line tools, not for authorization checking. This operation may "fail open" without warning.
|
| UpdateContent |
|---|
|
Update a content. Only supports full resource update.
|
DataScanService
DataScanService manages DataScan resources which can be configured to run various types of data scanning workload and generate enriched metadata (e.g. Data Profile, Data Quality) for the data source.
| CreateDataScan |
|---|
|
Creates a DataScan resource.
|
| DeleteDataScan |
|---|
|
Deletes a DataScan resource.
|
| GenerateDataQualityRules |
|---|
|
Generates recommended data quality rules based on the results of a data profiling scan. Use the recommendations to build rules for a data quality scan.
|
| GetDataScan |
|---|
|
Gets a DataScan resource.
|
| GetDataScanJob |
|---|
|
Gets a DataScanJob resource.
|
| ListDataScanJobs |
|---|
|
Lists DataScanJobs under the given DataScan.
|
| ListDataScans |
|---|
|
Lists DataScans.
|
| RunDataScan |
|---|
|
Runs an on-demand execution of a DataScan
|
| UpdateDataScan |
|---|
|
Updates a DataScan resource.
|
DataTaxonomyService
DataTaxonomyService enables attribute-based governance. The resources currently offered include DataTaxonomy and DataAttribute.
| CreateDataAttribute |
|---|
|
Create a DataAttribute resource.
|
| CreateDataAttributeBinding |
|---|
|
Create a DataAttributeBinding resource.
|
| CreateDataTaxonomy |
|---|
|
Create a DataTaxonomy resource.
|
| DeleteDataAttribute |
|---|
|
Deletes a Data Attribute resource.
|
| DeleteDataAttributeBinding |
|---|
|
Deletes a DataAttributeBinding resource. All attributes within the DataAttributeBinding must be deleted before the DataAttributeBinding can be deleted.
|
| DeleteDataTaxonomy |
|---|
|
Deletes a DataTaxonomy resource. All attributes within the DataTaxonomy must be deleted before the DataTaxonomy can be deleted.
|
| GetDataAttribute |
|---|
|
Retrieves a Data Attribute resource.
|
| GetDataAttributeBinding |
|---|
|
Retrieves a DataAttributeBinding resource.
|
| GetDataTaxonomy |
|---|
|
Retrieves a DataTaxonomy resource.
|
| ListDataAttributeBindings |
|---|
|
Lists DataAttributeBinding resources in a project and location.
|
| ListDataAttributes |
|---|
|
Lists Data Attribute resources in a DataTaxonomy.
|
| ListDataTaxonomies |
|---|
|
Lists DataTaxonomy resources in a project and location.
|
| UpdateDataAttribute |
|---|
|
Updates a DataAttribute resource.
|
| UpdateDataAttributeBinding |
|---|
|
Updates a DataAttributeBinding resource.
|
| UpdateDataTaxonomy |
|---|
|
Updates a DataTaxonomy resource.
|
DataplexService
Dataplex service provides data lakes as a service. The primary resources offered by this service are Lakes, Zones and Assets which collectively allow a data administrator to organize, manage, secure and catalog data across their organization located across cloud projects in a variety of storage systems including Cloud Storage and BigQuery.
| CancelJob |
|---|
|
Cancel jobs running for the task resource.
|
| CreateAsset |
|---|
|
Creates an asset resource.
|
| CreateEnvironment |
|---|
|
Create an environment resource.
|
| CreateLake |
|---|
|
Creates a lake resource.
|
| CreateTask |
|---|
|
Creates a task resource within a lake.
|
| CreateZone |
|---|
|
Creates a zone resource within a lake.
|
| DeleteAsset |
|---|
|
Deletes an asset resource. The referenced storage resource is detached (default) or deleted based on the associated Lifecycle policy.
|
| DeleteEnvironment |
|---|
|
Delete the environment resource. All the child resources must have been deleted before environment deletion can be initiated.
|
| DeleteLake |
|---|
|
Deletes a lake resource. All zones within the lake must be deleted before the lake can be deleted.
|
| DeleteTask |
|---|
|
Delete the task resource.
|
| DeleteZone |
|---|
|
Deletes a zone resource. All assets within a zone must be deleted before the zone can be deleted.
|
| GetAsset |
|---|
|
Retrieves an asset resource.
|
| GetEnvironment |
|---|
|
Get environment resource.
|
| GetJob |
|---|
|
Get job resource.
|
| GetLake |
|---|
|
Retrieves a lake resource.
|
| GetTask |
|---|
|
Get task resource.
|
| GetZone |
|---|
|
Retrieves a zone resource.
|
| ListAssetActions |
|---|
|
Lists action resources in an asset.
|
| ListAssets |
|---|
|
Lists asset resources in a zone.
|
| ListEnvironments |
|---|
|
Lists environments under the given lake.
|
| ListJobs |
|---|
|
Lists Jobs under the given task.
|
| ListLakeActions |
|---|
|
Lists action resources in a lake.
|
| ListLakes |
|---|
|
Lists lake resources in a project and location.
|
| ListSessions |
|---|
|
Lists session resources in an environment.
|
| ListTasks |
|---|
|
Lists tasks under the given lake.
|
| ListZoneActions |
|---|
|
Lists action resources in a zone.
|
| ListZones |
|---|
|
Lists zone resources in a lake.
|
| RunTask |
|---|
|
Run an on demand execution of a Task.
|
| UpdateAsset |
|---|
|
Updates an asset resource.
|
| UpdateEnvironment |
|---|
|
Update the environment resource.
|
| UpdateLake |
|---|
|
Updates a lake resource.
|
| UpdateTask |
|---|
|
Update the task resource.
|
| UpdateZone |
|---|
|
Updates a zone resource.
|
MetadataService
Metadata service manages metadata resources such as tables, filesets and partitions.
| CreateEntity |
|---|
|
Create a metadata entity.
|
| CreatePartition |
|---|
|
Create a metadata partition.
|
| DeleteEntity |
|---|
|
Delete a metadata entity.
|
| DeletePartition |
|---|
|
Delete a metadata partition.
|
| GetEntity |
|---|
|
Get a metadata entity.
|
| GetPartition |
|---|
|
Get a metadata partition of an entity.
|
| ListEntities |
|---|
|
List metadata entities in a zone.
|
| ListPartitions |
|---|
|
List metadata partitions of an entity.
|
| UpdateEntity |
|---|
|
Update a metadata entity. Only supports full resource update.
|
Action
Action represents an issue requiring administrator action for resolution.
| Fields | |
|---|---|
category |
The category of issue associated with the action. |
issue |
Detailed description of the issue requiring action. |
detect_time |
The time that the issue was detected. |
name |
Output only. The relative resource name of the action, of the form: |
lake |
Output only. The relative resource name of the lake, of the form: |
zone |
Output only. The relative resource name of the zone, of the form: |
asset |
Output only. The relative resource name of the asset, of the form: |
data_locations[] |
The list of data locations associated with this action. Cloud Storage locations are represented as URI paths(E.g. |
Union field details. Additional details about the action based on the action category. details can be only one of the following: |
|
invalid_data_format |
Details for issues related to invalid or unsupported data formats. |
incompatible_data_schema |
Details for issues related to incompatible schemas detected within data. |
invalid_data_partition |
Details for issues related to invalid or unsupported data partition structure. |
missing_data |
Details for issues related to absence of data within managed resources. |
missing_resource |
Details for issues related to absence of a managed resource. |
unauthorized_resource |
Details for issues related to lack of permissions to access data resources. |
failed_security_policy_apply |
Details for issues related to applying security policy. |
invalid_data_organization |
Details for issues related to invalid data arrangement. |
Category
The category of issues.
| Enums | |
|---|---|
CATEGORY_UNSPECIFIED |
Unspecified category. |
RESOURCE_MANAGEMENT |
Resource management related issues. |
SECURITY_POLICY |
Security policy related issues. |
DATA_DISCOVERY |
Data and discovery related issues. |
FailedSecurityPolicyApply
Failed to apply security policy to the managed resource(s) under a lake, zone or an asset. For a lake or zone resource, one or more underlying assets has a failure applying security policy to the associated managed resource.
| Fields | |
|---|---|
asset |
Resource name of one of the assets with failing security policy application. Populated for a lake or zone resource only. |
IncompatibleDataSchema
Action details for incompatible schemas detected by discovery.
| Fields | |
|---|---|
table |
The name of the table containing invalid data. |
existing_schema |
The existing and expected schema of the table. The schema is provided as a JSON formatted structure listing columns and data types. |
new_schema |
The new and incompatible schema within the table. The schema is provided as a JSON formatted structured listing columns and data types. |
sampled_data_locations[] |
The list of data locations sampled and used for format/schema inference. |
schema_change |
Whether the action relates to a schema that is incompatible or modified. |
SchemaChange
Whether the action relates to a schema that is incompatible or modified.
| Enums | |
|---|---|
SCHEMA_CHANGE_UNSPECIFIED |
Schema change unspecified. |
INCOMPATIBLE |
Newly discovered schema is incompatible with existing schema. |
MODIFIED |
Newly discovered schema has changed from existing schema for data in a curated zone. |
InvalidDataFormat
Action details for invalid or unsupported data files detected by discovery.
| Fields | |
|---|---|
sampled_data_locations[] |
The list of data locations sampled and used for format/schema inference. |
expected_format |
The expected data format of the entity. |
new_format |
The new unexpected data format within the entity. |
InvalidDataOrganization
This type has no fields.
Action details for invalid data arrangement.
InvalidDataPartition
Action details for invalid or unsupported partitions detected by discovery.
| Fields | |
|---|---|
expected_structure |
The issue type of InvalidDataPartition. |
PartitionStructure
The expected partition structure.
| Enums | |
|---|---|
PARTITION_STRUCTURE_UNSPECIFIED |
PartitionStructure unspecified. |
CONSISTENT_KEYS |
Consistent hive-style partition definition (both raw and curated zone). |
HIVE_STYLE_KEYS |
Hive style partition definition (curated zone only). |
MissingData
This type has no fields.
Action details for absence of data detected by discovery.
MissingResource
This type has no fields.
Action details for resource references in assets that cannot be located.
Aspect
An aspect is a single piece of metadata describing an entry.
| Fields | |
|---|---|
aspect_type |
Output only. The resource name of the type used to create this Aspect. |
path |
Output only. The path in the entry under which the aspect is attached. |
create_time |
Output only. The time when the Aspect was created. |
update_time |
Output only. The time when the Aspect was last updated. |
data |
Required. The content of the aspect, according to its aspect type schema. The maximum size of the field is 120KB (encoded as UTF-8). |
aspect_source |
Optional. Information related to the source system of the aspect. |
AspectSource
Information related to the source system of the aspect.
| Fields | |
|---|---|
create_time |
The time the aspect was created in the source system. |
update_time |
The time the aspect was last updated in the source system. |
data_version |
The version of the data format used to produce this data. This field is used to indicated when the underlying data format changes (e.g., schema modifications, changes to the source URL format definition, etc). |
AspectType
AspectType is a template for creating Aspects, and represents the JSON-schema for a given Entry, for example, BigQuery Table Schema.
| Fields | |
|---|---|
name |
Output only. The relative resource name of the AspectType, of the form: projects/{project_number}/locations/{location_id}/aspectTypes/{aspect_type_id}. |
uid |
Output only. System generated globally unique ID for the AspectType. If you delete and recreate the AspectType with the same name, then this ID will be different. |
create_time |
Output only. The time when the AspectType was created. |
update_time |
Output only. The time when the AspectType was last updated. |
description |
Optional. Description of the AspectType. |
display_name |
Optional. User friendly display name. |
labels |
Optional. User-defined labels for the AspectType. |
etag |
The service computes this checksum. The client may send it on update and delete requests to ensure it has an up-to-date value before proceeding. |
authorization |
Immutable. Defines the Authorization for this type. |
metadata_template |
Required. MetadataTemplate of the aspect. |
transfer_status |
Output only. Denotes the transfer status of the Aspect Type. It is unspecified for Aspect Types created from Dataplex API. |
Authorization
Autorization for an AspectType.
| Fields | |
|---|---|
alternate_use_permission |
Immutable. The IAM permission grantable on the EntryGroup to allow access to instantiate Aspects of Dataplex owned AspectTypes, only settable for Dataplex owned Types. |
MetadataTemplate
MetadataTemplate definition for an AspectType.
| Fields | |
|---|---|
index |
Optional. Index is used to encode Template messages. The value of index can range between 1 and 2,147,483,647. Index must be unique within all fields in a Template. (Nested Templates can reuse indexes). Once a Template is defined, the index cannot be changed, because it identifies the field in the actual storage format. Index is a mandatory field, but it is optional for top level fields, and map/array "values" definitions. |
name |
Required. The name of the field. |
type |
Required. The datatype of this field. The following values are supported: Primitive types:
Complex types:
|
record_fields[] |
Optional. Field definition. You must specify it if the type is record. It defines the nested fields. |
enum_values[] |
Optional. The list of values for an enum type. You must define it if the type is enum. |
map_items |
Optional. If the type is map, set map_items. map_items can refer to a primitive field or a complex (record only) field. To specify a primitive field, you only need to set name and type in the nested MetadataTemplate. The recommended value for the name field is item, as this isn't used in the actual payload. |
array_items |
Optional. If the type is array, set array_items. array_items can refer to a primitive field or a complex (record only) field. To specify a primitive field, you only need to set name and type in the nested MetadataTemplate. The recommended value for the name field is item, as this isn't used in the actual payload. |
type_id |
Optional. You can use type id if this definition of the field needs to be reused later. The type id must be unique across the entire template. You can only specify it if the field type is record. |
type_ref |
Optional. A reference to another field definition (not an inline definition). The value must be equal to the value of an id field defined elsewhere in the MetadataTemplate. Only fields with record type can refer to other fields. |
constraints |
Optional. Specifies the constraints on this field. |
annotations |
Optional. Specifies annotations on this field. |
Annotations
Definition of the annotations of a field.
| Fields | |
|---|---|
deprecated |
Optional. Marks a field as deprecated. You can include a deprecation message. |
display_name |
Optional. Display name for a field. |
description |
Optional. Description for a field. |
display_order |
Optional. Display order for a field. You can use this to reorder where a field is rendered. |
string_type |
Optional. You can use String Type annotations to specify special meaning to string fields. The following values are supported:
|
string_values[] |
Optional. Suggested hints for string fields. You can use them to suggest values to users through console. |
Constraints
Definition of the constraints of a field.
| Fields | |
|---|---|
required |
Optional. Marks this field as optional or required. |
EnumValue
Definition of Enumvalue, to be used for enum fields.
| Fields | |
|---|---|
index |
Required. Index for the enum value. It can't be modified. |
name |
Required. Name of the enumvalue. This is the actual value that the aspect can contain. |
deprecated |
Optional. You can set this message if you need to deprecate an enum value. |
Asset
An asset represents a cloud resource that is being managed within a lake as a member of a zone.
| Fields | |
|---|---|
name |
Output only. The relative resource name of the asset, of the form: |
display_name |
Optional. User friendly display name. |
uid |
Output only. System generated globally unique ID for the asset. This ID will be different if the asset is deleted and re-created with the same name. |
create_time |
Output only. The time when the asset was created. |
update_time |
Output only. The time when the asset was last updated. |
labels |
Optional. User defined labels for the asset. |
description |
Optional. Description of the asset. |
state |
Output only. Current state of the asset. |
resource_spec |
Required. Specification of the resource that is referenced by this asset. |
resource_status |
Output only. Status of the resource referenced by this asset. |
security_status |
Output only. Status of the security policy applied to resource referenced by this asset. |
discovery_spec |
Optional. Specification of the discovery feature applied to data referenced by this asset. When this spec is left unset, the asset will use the spec set on the parent zone. |
discovery_status |
Output only. Status of the discovery feature applied to data referenced by this asset. |
DiscoverySpec
Settings to manage the metadata discovery and publishing for an asset.
| Fields | |
|---|---|
enabled |
Optional. Whether discovery is enabled. |
include_patterns[] |
Optional. The list of patterns to apply for selecting data to include during discovery if only a subset of the data should considered. For Cloud Storage bucket assets, these are interpreted as glob patterns used to match object names. For BigQuery dataset assets, these are interpreted as patterns to match table names. |
exclude_patterns[] |
Optional. The list of patterns to apply for selecting data to exclude during discovery. For Cloud Storage bucket assets, these are interpreted as glob patterns used to match object names. For BigQuery dataset assets, these are interpreted as patterns to match table names. |
csv_options |
Optional. Configuration for CSV data. |
json_options |
Optional. Configuration for Json data. |
Union field trigger. Determines when discovery is triggered. trigger can be only one of the following: |
|
schedule |
Optional. Cron schedule (https://en.wikipedia.org/wiki/Cron) for running discovery periodically. Successive discovery runs must be scheduled at least 60 minutes apart. The default value is to run discovery every 60 minutes. To explicitly set a timezone to the cron tab, apply a prefix in the cron tab: "CRON_TZ=${IANA_TIME_ZONE}" or TZ=${IANA_TIME_ZONE}". The ${IANA_TIME_ZONE} may only be a valid string from IANA time zone database. For example, |
CsvOptions
Describe CSV and similar semi-structured data formats.
| Fields | |
|---|---|
header_rows |
Optional. The number of rows to interpret as header rows that should be skipped when reading data rows. |
delimiter |
Optional. The delimiter being used to separate values. This defaults to ','. |
encoding |
Optional. The character encoding of the data. The default is UTF-8. |
disable_type_inference |
Optional. Whether to disable the inference of data type for CSV data. If true, all columns will be registered as strings. |
JsonOptions
Describe JSON data format.
| Fields | |
|---|---|
encoding |
Optional. The character encoding of the data. The default is UTF-8. |
disable_type_inference |
Optional. Whether to disable the inference of data type for Json data. If true, all columns will be registered as their primitive types (strings, number or boolean). |
DiscoveryStatus
Status of discovery for an asset.
| Fields | |
|---|---|
state |
The current status of the discovery feature. |
message |
Additional information about the current state. |
update_time |
Last update time of the status. |
last_run_time |
The start time of the last discovery run. |
stats |
Data Stats of the asset reported by discovery. |
last_run_duration |
The duration of the last discovery run. |
State
Current state of discovery.
| Enums | |
|---|---|
STATE_UNSPECIFIED |
State is unspecified. |
SCHEDULED |
Discovery for the asset is scheduled. |
IN_PROGRESS |
Discovery for the asset is running. |
PAUSED |
Discovery for the asset is currently paused (e.g. due to a lack of available resources). It will be automatically resumed. |
DISABLED |
Discovery for the asset is disabled. |
Stats
The aggregated data statistics for the asset reported by discovery.
| Fields | |
|---|---|
data_items |
The count of data items within the referenced resource. |
data_size |
The number of stored data bytes within the referenced resource. |
tables |
The count of table entities within the referenced resource. |
filesets |
The count of fileset entities within the referenced resource. |
ResourceSpec
Identifies the cloud resource that is referenced by this asset.
| Fields | |
|---|---|
name |
Immutable. Relative name of the cloud resource that contains the data that is being managed within a lake. For example: |
type |
Required. Immutable. Type of resource. |
read_access_mode |
Optional. Determines how read permissions are handled for each asset and their associated tables. Only available to storage buckets assets. |
AccessMode
Access Mode determines how data stored within the resource is read. This is only applicable to storage bucket assets.
| Enums | |
|---|---|
ACCESS_MODE_UNSPECIFIED |
Access mode unspecified. |
DIRECT |
Default. Data is accessed directly using storage APIs. |
MANAGED |
Data is accessed through a managed interface using BigQuery APIs. |
Type
Type of resource.
| Enums | |
|---|---|
TYPE_UNSPECIFIED |
Type not specified. |
STORAGE_BUCKET |
Cloud Storage bucket. |
BIGQUERY_DATASET |
BigQuery dataset. |
ResourceStatus
Status of the resource referenced by an asset.
| Fields | |
|---|---|
state |
The current state of the managed resource. |
message |
Additional information about the current state. |
update_time |
Last update time of the status. |
managed_access_identity |
Output only. Service account associated with the BigQuery Connection. |
State
The state of a resource.
| Enums | |
|---|---|
STATE_UNSPECIFIED |
State unspecified. |
READY |
Resource does not have any errors. |
ERROR |
Resource has errors. |
SecurityStatus
Security policy status of the asset. Data security policy, i.e., readers, writers & owners, should be specified in the lake/zone/asset IAM policy.
| Fields | |
|---|---|
state |
The current state of the security policy applied to the attached resource. |
message |
Additional information about the current state. |
update_time |
Last update time of the status. |
State
The state of the security policy.
| Enums | |
|---|---|
STATE_UNSPECIFIED |
State unspecified. |
READY |
Security policy has been successfully applied to the attached resource. |
APPLYING |
Security policy is in the process of being applied to the attached resource. |
ERROR |
Security policy could not be applied to the attached resource due to errors. |
AssetStatus
Aggregated status of the underlying assets of a lake or zone.
| Fields | |
|---|---|
update_time |
Last update time of the status. |
active_assets |
Number of active assets. |
security_policy_applying_assets |
Number of assets that are in process of updating the security policy on attached resources. |
CancelJobRequest
Cancel task jobs.
| Fields | |
|---|---|
name |
Required. The resource name of the job: Authorization requires the following IAM permission on the specified resource
|
CancelMetadataJobRequest
Cancel metadata job request.
| Fields | |
|---|---|
name |
Required. The resource name of the job, in the format Authorization requires the following IAM permission on the specified resource
|
Content
Content represents a user-visible notebook or a sql script
| Fields | |
|---|---|
name |
Output only. The relative resource name of the content, of the form: projects/{project_id}/locations/{location_id}/lakes/{lake_id}/content/{content_id} |
uid |
Output only. System generated globally unique ID for the content. This ID will be different if the content is deleted and re-created with the same name. |
path |
Required. The path for the Content file, represented as directory structure. Unique within a lake. Limited to alphanumerics, hyphens, underscores, dots and slashes. |
create_time |
Output only. Content creation time. |
update_time |
Output only. The time when the content was last updated. |
labels |
Optional. User defined labels for the content. |
description |
Optional. Description of the content. |
Union field data. Only returned in GetContent requests and not in ListContent request. data can be only one of the following: |
|
data_text |
Required. Content data in string format. |
Union field content. Types of content content can be only one of the following: |
|
sql_script |
Sql Script related configurations. |
notebook |
Notebook related configurations. |
Notebook
Configuration for Notebook content.
| Fields | |
|---|---|
kernel_type |
Required. Kernel Type of the notebook. |
KernelType
Kernel Type of the Jupyter notebook.
| Enums | |
|---|---|
KERNEL_TYPE_UNSPECIFIED |
Kernel Type unspecified. |
PYTHON3 |
Python 3 Kernel. |
SqlScript
Configuration for the Sql Script content.
| Fields | |
|---|---|
engine |
Required. Query Engine to be used for the Sql Query. |
QueryEngine
Query Engine Type of the SQL Script.
| Enums | |
|---|---|
QUERY_ENGINE_UNSPECIFIED |
Value was unspecified. |
SPARK |
Spark SQL Query. |
CreateAspectTypeRequest
Create AspectType Request.
| Fields | |
|---|---|
parent |
Required. The resource name of the AspectType, of the form: projects/{project_number}/locations/{location_id} where Authorization requires the following IAM permission on the specified resource
|
aspect_type_id |
Required. AspectType identifier. |
aspect_type |
Required. AspectType Resource. |
validate_only |
Optional. The service validates the request without performing any mutations. The default is false. |
CreateAssetRequest
Create asset request.
| Fields | |
|---|---|
parent |
Required. The resource name of the parent zone: Authorization requires the following IAM permission on the specified resource
|
asset_id |
Required. Asset identifier. This ID will be used to generate names such as table names when publishing metadata to Hive Metastore and BigQuery. * Must contain only lowercase letters, numbers and hyphens. * Must start with a letter. * Must end with a number or a letter. * Must be between 1-63 characters. * Must be unique within the zone. |
asset |
Required. Asset resource. |
validate_only |
Optional. Only validate the request, but do not perform mutations. The default is false. |
CreateContentRequest
Create content request.
| Fields | |
|---|---|
parent |
Required. The resource name of the parent lake: projects/{project_id}/locations/{location_id}/lakes/{lake_id} Authorization requires the following IAM permission on the specified resource
|
content |
Required. Content resource. |
validate_only |
Optional. Only validate the request, but do not perform mutations. The default is false. |
CreateDataAttributeBindingRequest
Create DataAttributeBinding request.
| Fields | |
|---|---|
parent |
Required. The resource name of the parent data taxonomy projects/{project_number}/locations/{location_id} Authorization requires the following IAM permission on the specified resource
|
data_attribute_binding_id |
Required. DataAttributeBinding identifier. * Must contain only lowercase letters, numbers and hyphens. * Must start with a letter. * Must be between 1-63 characters. * Must end with a number or a letter. * Must be unique within the Location. |
data_attribute_binding |
Required. DataAttributeBinding resource. |
validate_only |
Optional. Only validate the request, but do not perform mutations. The default is false. |
CreateDataAttributeRequest
Create DataAttribute request.
| Fields | |
|---|---|
parent |
Required. The resource name of the parent data taxonomy projects/{project_number}/locations/{location_id}/dataTaxonomies/{data_taxonomy_id} Authorization requires the following IAM permission on the specified resource
|
data_attribute_id |
Required. DataAttribute identifier. * Must contain only lowercase letters, numbers and hyphens. * Must start with a letter. * Must be between 1-63 characters. * Must end with a number or a letter. * Must be unique within the DataTaxonomy. |
data_attribute |
Required. DataAttribute resource. |
validate_only |
Optional. Only validate the request, but do not perform mutations. The default is false. |
CreateDataScanRequest
Create dataScan request.
| Fields | |
|---|---|
parent |
Required. The resource name of the parent location: Authorization requires the following IAM permission on the specified resource
|
data_scan |
Required. DataScan resource. |
data_scan_id |
Required. DataScan identifier.
|
validate_only |
Optional. Only validate the request, but do not perform mutations. The default is |
CreateDataTaxonomyRequest
Create DataTaxonomy request.
| Fields | |
|---|---|
parent |
Required. The resource name of the data taxonomy location, of the form: projects/{project_number}/locations/{location_id} where Authorization requires the following IAM permission on the specified resource
|
data_taxonomy_id |
Required. DataTaxonomy identifier. * Must contain only lowercase letters, numbers and hyphens. * Must start with a letter. * Must be between 1-63 characters. * Must end with a number or a letter. * Must be unique within the Project. |
data_taxonomy |
Required. DataTaxonomy resource. |
validate_only |
Optional. Only validate the request, but do not perform mutations. The default is false. |
CreateEntityRequest
Create a metadata entity request.
| Fields | |
|---|---|
parent |
Required. The resource name of the parent zone: Authorization requires the following IAM permission on the specified resource
|
entity |
Required. Entity resource. |
validate_only |
Optional. Only validate the request, but do not perform mutations. The default is false. |
CreateEntryGroupRequest
Create EntryGroup Request.
| Fields | |
|---|---|
parent |
Required. The resource name of the entryGroup, of the form: projects/{project_number}/locations/{location_id} where Authorization requires the following IAM permission on the specified resource
|
entry_group_id |
Required. EntryGroup identifier. |
entry_group |
Required. EntryGroup Resource. |
validate_only |
Optional. The service validates the request without performing any mutations. The default is false. |
CreateEntryRequest
Create Entry request.
| Fields | |
|---|---|
parent |
Required. The resource name of the parent Entry Group: |
entry_id |
Required. Entry identifier. It has to be unique within an Entry Group. Entries corresponding to Google Cloud resources use an Entry ID format based on full resource names. The format is a full resource name of the resource without the prefix double slashes in the API service name part of the full resource name. This allows retrieval of entries using their associated resource name. For example, if the full resource name of a resource is It is also suggested to follow the same convention for entries corresponding to resources from providers or systems other than Google Cloud. The maximum size of the field is 4000 characters. |
entry |
Required. Entry resource. |
CreateEntryTypeRequest
Create EntryType Request.
| Fields | |
|---|---|
parent |
Required. The resource name of the EntryType, of the form: projects/{project_number}/locations/{location_id} where Authorization requires the following IAM permission on the specified resource
|
entry_type_id |
Required. EntryType identifier. |
entry_type |
Required. EntryType Resource. |
validate_only |
Optional. The service validates the request without performing any mutations. The default is false. |
CreateEnvironmentRequest
Create environment request.
| Fields | |
|---|---|
parent |
Required. The resource name of the parent lake: Authorization requires the following IAM permission on the specified resource
|
environment_id |
Required. Environment identifier. * Must contain only lowercase letters, numbers and hyphens. * Must start with a letter. * Must be between 1-63 characters. * Must end with a number or a letter. * Must be unique within the lake. |
environment |
Required. Environment resource. |
validate_only |
Optional. Only validate the request, but do not perform mutations. The default is false. |
CreateLakeRequest
Create lake request.
| Fields | |
|---|---|
parent |
Required. The resource name of the lake location, of the form: projects/{project_number}/locations/{location_id} where Authorization requires the following IAM permission on the specified resource
|
lake_id |
Required. Lake identifier. This ID will be used to generate names such as database and dataset names when publishing metadata to Hive Metastore and BigQuery. * Must contain only lowercase letters, numbers and hyphens. * Must start with a letter. * Must end with a number or a letter. * Must be between 1-63 characters. * Must be unique within the customer project / location. |
lake |
Required. Lake resource |
validate_only |
Optional. Only validate the request, but do not perform mutations. The default is false. |
CreateMetadataJobRequest
Create metadata job request.
| Fields | |
|---|---|
parent |
Required. The resource name of the parent location, in the format Authorization requires the following IAM permission on the specified resource
|
metadata_job |
Required. The metadata job resource. |
metadata_job_id |
Optional. The metadata job ID. If not provided, a unique ID is generated with the prefix |
CreatePartitionRequest
Create metadata partition request.
| Fields | |
|---|---|
parent |
Required. The resource name of the parent zone: Authorization requires the following IAM permission on the specified resource
|
partition |
Required. Partition resource. |
validate_only |
Optional. Only validate the request, but do not perform mutations. The default is false. |
CreateTaskRequest
Create task request.
| Fields | |
|---|---|
parent |
Required. The resource name of the parent lake: Authorization requires the following IAM permission on the specified resource
|
task_id |
Required. Task identifier. |
task |
Required. Task resource. |
validate_only |
Optional. Only validate the request, but do not perform mutations. The default is false. |
CreateZoneRequest
Create zone request.
| Fields | |
|---|---|
parent |
Required. The resource name of the parent lake: Authorization requires the following IAM permission on the specified resource
|
zone_id |
Required. Zone identifier. This ID will be used to generate names such as database and dataset names when publishing metadata to Hive Metastore and BigQuery. * Must contain only lowercase letters, numbers and hyphens. * Must start with a letter. * Must end with a number or a letter. * Must be between 1-63 characters. * Must be unique across all lakes from all locations in a project. * Must not be one of the reserved IDs (i.e. "default", "global-temp") |
zone |
Required. Zone resource. |
validate_only |
Optional. Only validate the request, but do not perform mutations. The default is false. |
DataAccessSpec
DataAccessSpec holds the access control configuration to be enforced on data stored within resources (eg: rows, columns in BigQuery Tables). When associated with data, the data is only accessible to principals explicitly granted access through the DataAccessSpec. Principals with access to the containing resource are not implicitly granted access.
| Fields | |
|---|---|
readers[] |
Optional. The format of strings follows the pattern followed by IAM in the bindings. user:{email}, serviceAccount:{email} group:{email}. The set of principals to be granted reader role on data stored within resources. |
DataAttribute
Denotes one dataAttribute in a dataTaxonomy, for example, PII. DataAttribute resources can be defined in a hierarchy. A single dataAttribute resource can contain specs of multiple types
PII
- ResourceAccessSpec :
- readers :foo@bar.com
- DataAccessSpec :
- readers :bar@foo.com
| Fields | |
|---|---|
name |
Output only. The relative resource name of the dataAttribute, of the form: projects/{project_number}/locations/{location_id}/dataTaxonomies/{dataTaxonomy}/attributes/{data_attribute_id}. |
uid |
Output only. System generated globally unique ID for the DataAttribute. This ID will be different if the DataAttribute is deleted and re-created with the same name. |
create_time |
Output only. The time when the DataAttribute was created. |
update_time |
Output only. The time when the DataAttribute was last updated. |
description |
Optional. Description of the DataAttribute. |
display_name |
Optional. User friendly display name. |
labels |
Optional. User-defined labels for the DataAttribute. |
parent_id |
Optional. The ID of the parent DataAttribute resource, should belong to the same data taxonomy. Circular dependency in parent chain is not valid. Maximum depth of the hierarchy allowed is 4. [a -> b -> c -> d -> e, depth = 4] |
attribute_count |
Output only. The number of child attributes present for this attribute. |
etag |
This checksum is computed by the server based on the value of other fields, and may be sent on update and delete requests to ensure the client has an up-to-date value before proceeding. |
resource_access_spec |
Optional. Specified when applied to a resource (eg: Cloud Storage bucket, BigQuery dataset, BigQuery table). |
data_access_spec |
Optional. Specified when applied to data stored on the resource (eg: rows, columns in BigQuery Tables). |
DataAttributeBinding
DataAttributeBinding represents binding of attributes to resources. Eg: Bind 'CustomerInfo' entity with 'PII' attribute.
| Fields | |
|---|---|
name |
Output only. The relative resource name of the Data Attribute Binding, of the form: projects/{project_number}/locations/{location}/dataAttributeBindings/{data_attribute_binding_id} |
uid |
Output only. System generated globally unique ID for the DataAttributeBinding. This ID will be different if the DataAttributeBinding is deleted and re-created with the same name. |
create_time |
Output only. The time when the DataAttributeBinding was created. |
update_time |
Output only. The time when the DataAttributeBinding was last updated. |
description |
Optional. Description of the DataAttributeBinding. |
display_name |
Optional. User friendly display name. |
labels |
Optional. User-defined labels for the DataAttributeBinding. |
etag |
This checksum is computed by the server based on the value of other fields, and may be sent on update and delete requests to ensure the client has an up-to-date value before proceeding. Etags must be used when calling the DeleteDataAttributeBinding and the UpdateDataAttributeBinding method. |
attributes[] |
Optional. List of attributes to be associated with the resource, provided in the form: projects/{project}/locations/{location}/dataTaxonomies/{dataTaxonomy}/attributes/{data_attribute_id} |
paths[] |
Optional. The list of paths for items within the associated resource (eg. columns and partitions within a table) along with attribute bindings. |
Union field resource_reference. The reference to the resource that is associated to attributes, or the query to match resources and associate attributes. resource_reference can be only one of the following: |
|
resource |
Optional. Immutable. The resource name of the resource that is associated to attributes. Presently, only entity resource is supported in the form: projects/{project}/locations/{location}/lakes/{lake}/zones/{zone}/entities/{entity_id} Must belong in the same project and region as the attribute binding, and there can only exist one active binding for a resource. |
Path
Represents a subresource of the given resource, and associated bindings with it. Currently supported subresources are column and partition schema fields within a table.
| Fields | |
|---|---|
name |
Required. The name identifier of the path. Nested columns should be of the form: 'address.city'. |
attributes[] |
Optional. List of attributes to be associated with the path of the resource, provided in the form: projects/{project}/locations/{location}/dataTaxonomies/{dataTaxonomy}/attributes/{data_attribute_id} |
DataProfileResult
DataProfileResult defines the output of DataProfileScan. Each field of the table will have field type specific profile result.
| Fields | |
|---|---|
row_count |
The count of rows scanned. |
profile |
The profile information per field. |
scanned_data |
The data scanned for this result. |
post_scan_actions_result |
Output only. The result of post scan actions. |
PostScanActionsResult
The result of post scan actions of DataProfileScan job.
| Fields | |
|---|---|
bigquery_export_result |
Output only. The result of BigQuery export post scan action. |
BigQueryExportResult
The result of BigQuery export post scan action.
| Fields | |
|---|---|
state |
Output only. Execution state for the BigQuery exporting. |
message |
Output only. Additional information about the BigQuery exporting. |
State
Execution state for the exporting.
| Enums | |
|---|---|
STATE_UNSPECIFIED |
The exporting state is unspecified. |
SUCCEEDED |
The exporting completed successfully. |
FAILED |
The exporting is no longer running due to an error. |
SKIPPED |
The exporting is skipped due to no valid scan result to export (usually caused by scan failed). |
Profile
Contains name, type, mode and field type specific profile information.
| Fields | |
|---|---|
fields[] |
List of fields with structural and profile information for each field. |
Field
A field within a table.
| Fields | |
|---|---|
name |
The name of the field. |
type |
The data type retrieved from the schema of the data source. For instance, for a BigQuery native table, it is the BigQuery Table Schema. For a Dataplex Entity, it is the Entity Schema. |
mode |
The mode of the field. Possible values include:
|
profile |
Profile information for the corresponding field. |
ProfileInfo
The profile information for each field type.
| Fields | |
|---|---|
null_ratio |
Ratio of rows with null value against total scanned rows. |
distinct_ratio |
Ratio of rows with distinct values against total scanned rows. Not available for complex non-groupable field type, including RECORD, ARRAY, GEOGRAPHY, and JSON, as well as fields with REPEATABLE mode. |
top_n_values[] |
The list of top N non-null values, frequency and ratio with which they occur in the scanned data. N is 10 or equal to the number of distinct values in the field, whichever is smaller. Not available for complex non-groupable field type, including RECORD, ARRAY, GEOGRAPHY, and JSON, as well as fields with REPEATABLE mode. |
Union field field_info. Structural and profile information for specific field type. Not available, if mode is REPEATABLE. field_info can be only one of the following: |
|
string_profile |
String type field information. |
integer_profile |
Integer type field information. |
double_profile |
Double type field information. |
DoubleFieldInfo
The profile information for a double type field.
| Fields | |
|---|---|
average |
Average of non-null values in the scanned data. NaN, if the field has a NaN. |
standard_deviation |
Standard deviation of non-null values in the scanned data. NaN, if the field has a NaN. |
min |
Minimum of non-null values in the scanned data. NaN, if the field has a NaN. |
quartiles[] |
A quartile divides the number of data points into four parts, or quarters, of more-or-less equal size. Three main quartiles used are: The first quartile (Q1) splits off the lowest 25% of data from the highest 75%. It is also known as the lower or 25th empirical quartile, as 25% of the data is below this point. The second quartile (Q2) is the median of a data set. So, 50% of the data lies below this point. The third quartile (Q3) splits off the highest 25% of data from the lowest 75%. It is known as the upper or 75th empirical quartile, as 75% of the data lies below this point. Here, the quartiles is provided as an ordered list of quartile values for the scanned data, occurring in order Q1, median, Q3. |
max |
Maximum of non-null values in the scanned data. NaN, if the field has a NaN. |
IntegerFieldInfo
The profile information for an integer type field.
| Fields | |
|---|---|
average |
Average of non-null values in the scanned data. NaN, if the field has a NaN. |
standard_deviation |
Standard deviation of non-null values in the scanned data. NaN, if the field has a NaN. |
min |
Minimum of non-null values in the scanned data. NaN, if the field has a NaN. |
quartiles[] |
A quartile divides the number of data points into four parts, or quarters, of more-or-less equal size. Three main quartiles used are: The first quartile (Q1) splits off the lowest 25% of data from the highest 75%. It is also known as the lower or 25th empirical quartile, as 25% of the data is below this point. The second quartile (Q2) is the median of a data set. So, 50% of the data lies below this point. The third quartile (Q3) splits off the highest 25% of data from the lowest 75%. It is known as the upper or 75th empirical quartile, as 75% of the data lies below this point. Here, the quartiles is provided as an ordered list of approximate quartile values for the scanned data, occurring in order Q1, median, Q3. |
max |
Maximum of non-null values in the scanned data. NaN, if the field has a NaN. |
StringFieldInfo
The profile information for a string type field.
| Fields | |
|---|---|
min_length |
Minimum length of non-null values in the scanned data. |
max_length |
Maximum length of non-null values in the scanned data. |
average_length |
Average length of non-null values in the scanned data. |
TopNValue
Top N non-null values in the scanned data.
| Fields | |
|---|---|
value |
String value of a top N non-null value. |
count |
Count of the corresponding value in the scanned data. |
ratio |
Ratio of the corresponding value in the field against the total number of rows in the scanned data. |
DataProfileSpec
DataProfileScan related setting.
| Fields | |
|---|---|
sampling_percent |
Optional. The percentage of the records to be selected from the dataset for DataScan.
|
row_filter |
Optional. A filter applied to all rows in a single DataScan job. The filter needs to be a valid SQL expression for a WHERE clause in BigQuery standard SQL syntax. Example: col1 >= 0 AND col2 < 10 |
post_scan_actions |
Optional. Actions to take upon job completion.. |
include_fields |
Optional. The fields to include in data profile. If not specified, all fields at the time of profile scan job execution are included, except for ones listed in |
exclude_fields |
Optional. The fields to exclude from data profile. If specified, the fields will be excluded from data profile, regardless of |
PostScanActions
The configuration of post scan actions of DataProfileScan job.
| Fields | |
|---|---|
bigquery_export |
Optional. If set, results will be exported to the provided BigQuery table. |
BigQueryExport
The configuration of BigQuery export post scan action.
| Fields | |
|---|---|
results_table |
Optional. The BigQuery table to export DataProfileScan results to. Format: //bigquery.googleapis.com/projects/PROJECT_ID/datasets/DATASET_ID/tables/TABLE_ID |
SelectedFields
The specification for fields to include or exclude in data profile scan.
| Fields | |
|---|---|
field_names[] |
Optional. Expected input is a list of fully qualified names of fields as in the schema. Only top-level field names for nested fields are supported. For instance, if 'x' is of nested field type, listing 'x' is supported but 'x.y.z' is not supported. Here 'y' and 'y.z' are nested fields of 'x'. |
DataQualityColumnResult
DataQualityColumnResult provides a more detailed, per-column view of the results.
| Fields | |
|---|---|
column |
Output only. The column specified in the DataQualityRule. |
score |
Output only. The column-level data quality score for this data scan job if and only if the 'column' field is set. The score ranges between between [0, 100] (up to two decimal points). |
DataQualityDimension
A dimension captures data quality intent about a defined subset of the rules specified.
| Fields | |
|---|---|
name |
The dimension name a rule belongs to. Supported dimensions are ["COMPLETENESS", "ACCURACY", "CONSISTENCY", "VALIDITY", "UNIQUENESS", "FRESHNESS", "VOLUME"] |
DataQualityDimensionResult
DataQualityDimensionResult provides a more detailed, per-dimension view of the results.
| Fields | |
|---|---|
dimension |
Output only. The dimension config specified in the DataQualitySpec, as is. |
passed |
Whether the dimension passed or failed. |
score |
Output only. The dimension-level data quality score for this data scan job if and only if the 'dimension' field is set. The score ranges between [0, 100] (up to two decimal points). |
DataQualityResult
The output of a DataQualityScan.
| Fields | |
|---|---|
passed |
Overall data quality result -- |
dimensions[] |
A list of results at the dimension level. A dimension will have a corresponding |
columns[] |
Output only. A list of results at the column level. A column will have a corresponding |
rules[] |
A list of all the rules in a job, and their results. |
row_count |
The count of rows processed. |
scanned_data |
The data scanned for this result. |
post_scan_actions_result |
Output only. The result of post scan actions. |
score |
Output only. The overall data quality score. The score ranges between [0, 100] (up to two decimal points). |
PostScanActionsResult
The result of post scan actions of DataQualityScan job.
| Fields | |
|---|---|
bigquery_export_result |
Output only. The result of BigQuery export post scan action. |
BigQueryExportResult
The result of BigQuery export post scan action.
| Fields | |
|---|---|
state |
Output only. Execution state for the BigQuery exporting. |
message |
Output only. Additional information about the BigQuery exporting. |
State
Execution state for the exporting.
| Enums | |
|---|---|
STATE_UNSPECIFIED |
The exporting state is unspecified. |
SUCCEEDED |
The exporting completed successfully. |
FAILED |
The exporting is no longer running due to an error. |
SKIPPED |
The exporting is skipped due to no valid scan result to export (usually caused by scan failed). |
DataQualityRule
A rule captures data quality intent about a data source.
| Fields | |
|---|---|
column |
Optional. The unnested column which this rule is evaluated against. |
ignore_null |
Optional. Rows with This field is only valid for the following type of rules:
|
dimension |
Required. The dimension a rule belongs to. Results are also aggregated at the dimension level. Supported dimensions are ["COMPLETENESS", "ACCURACY", "CONSISTENCY", "VALIDITY", "UNIQUENESS", "FRESHNESS", "VOLUME"] |
threshold |
Optional. The minimum ratio of passing_rows / total_rows required to pass this rule, with a range of [0.0, 1.0]. 0 indicates default value (i.e. 1.0). This field is only valid for row-level type rules. |
name |
Optional. A mutable name for the rule.
|
description |
Optional. Description of the rule.
|
suspended |
Optional. Whether the Rule is active or suspended. Default is false. |
Union field rule_type. The rule-specific configuration. rule_type can be only one of the following: |
|
range_expectation |
Row-level rule which evaluates whether each column value lies between a specified range. |
non_null_expectation |
Row-level rule which evaluates whether each column value is null. |
set_expectation |
Row-level rule which evaluates whether each column value is contained by a specified set. |
regex_expectation |
Row-level rule which evaluates whether each column value matches a specified regex. |
uniqueness_expectation |
Row-level rule which evaluates whether each column value is unique. |
statistic_range_expectation |
Aggregate rule which evaluates whether the column aggregate statistic lies between a specified range. |
row_condition_expectation |
Row-level rule which evaluates whether each row in a table passes the specified condition. |
table_condition_expectation |
Aggregate rule which evaluates whether the provided expression is true for a table. |
sql_assertion |
Aggregate rule which evaluates the number of rows returned for the provided statement. If any rows are returned, this rule fails. |
NonNullExpectation
This type has no fields.
Evaluates whether each column value is null.
RangeExpectation
Evaluates whether each column value lies between a specified range.
| Fields | |
|---|---|
min_value |
Optional. The minimum column value allowed for a row to pass this validation. At least one of |
max_value |
Optional. The maximum column value allowed for a row to pass this validation. At least one of |
strict_min_enabled |
Optional. Whether each value needs to be strictly greater than ('>') the minimum, or if equality is allowed. Only relevant if a |
strict_max_enabled |
Optional. Whether each value needs to be strictly lesser than ('<') the maximum, or if equality is allowed. Only relevant if a |
RegexExpectation
Evaluates whether each column value matches a specified regex.
| Fields | |
|---|---|
regex |
Optional. A regular expression the column value is expected to match. |
RowConditionExpectation
Evaluates whether each row passes the specified condition.
The SQL expression needs to use BigQuery standard SQL syntax and should produce a boolean value per row as the result.
Example: col1 >= 0 AND col2 < 10
| Fields | |
|---|---|
sql_expression |
Optional. The SQL expression. |
SetExpectation
Evaluates whether each column value is contained by a specified set.
| Fields | |
|---|---|
values[] |
Optional. Expected values for the column value. |
SqlAssertion
A SQL statement that is evaluated to return rows that match an invalid state. If any rows are are returned, this rule fails.
The SQL statement must use BigQuery standard SQL syntax, and must not contain any semicolons.
You can use the data reference parameter ${data()} to reference the source table with all of its precondition filters applied. Examples of precondition filters include row filters, incremental data filters, and sampling. For more information, see Data reference parameter.
Example: SELECT * FROM ${data()} WHERE price < 0
| Fields | |
|---|---|
sql_statement |
Optional. The SQL statement. |
StatisticRangeExpectation
Evaluates whether the column aggregate statistic lies between a specified range.
| Fields | |
|---|---|
statistic |
Optional. The aggregate metric to evaluate. |
min_value |
Optional. The minimum column statistic value allowed for a row to pass this validation. At least one of |
max_value |
Optional. The maximum column statistic value allowed for a row to pass this validation. At least one of |
strict_min_enabled |
Optional. Whether column statistic needs to be strictly greater than ('>') the minimum, or if equality is allowed. Only relevant if a |
strict_max_enabled |
Optional. Whether column statistic needs to be strictly lesser than ('<') the maximum, or if equality is allowed. Only relevant if a |
ColumnStatistic
The list of aggregate metrics a rule can be evaluated against.
| Enums | |
|---|---|
STATISTIC_UNDEFINED |
Unspecified statistic type |
MEAN |
Evaluate the column mean |
MIN |
Evaluate the column min |
MAX |
Evaluate the column max |
TableConditionExpectation
Evaluates whether the provided expression is true.
The SQL expression needs to use BigQuery standard SQL syntax and should produce a scalar boolean result.
Example: MIN(col1) >= 0
| Fields | |
|---|---|
sql_expression |
Optional. The SQL expression. |
UniquenessExpectation
This type has no fields.
Evaluates whether the column has duplicates.
DataQualityRuleResult
DataQualityRuleResult provides a more detailed, per-rule view of the results.
| Fields | |
|---|---|
rule |
The rule specified in the DataQualitySpec, as is. |
passed |
Whether the rule passed or failed. |
evaluated_count |
The number of rows a rule was evaluated against. This field is only valid for row-level type rules. Evaluated count can be configured to either
|
passed_count |
The number of rows which passed a rule evaluation. This field is only valid for row-level type rules. |
null_count |
The number of rows with null values in the specified column. |
pass_ratio |
The ratio of passed_count / evaluated_count. This field is only valid for row-level type rules. |
failing_rows_query |
The query to find rows that did not pass this rule. This field is only valid for row-level type rules. |
assertion_row_count |
Output only. The number of rows returned by the SQL statement in a SQL assertion rule. This field is only valid for SQL assertion rules. |
DataQualityScanRuleResult
Information about the result of a data quality rule for data quality scan. The monitored resource is 'DataScan'.
| Fields | |
|---|---|
job_id |
Identifier of the specific data scan job this log entry is for. |
data_source |
The data source of the data scan (e.g. BigQuery table name). |
column |
The column which this rule is evaluated against. |
rule_name |
The name of the data quality rule. |
rule_type |
The type of the data quality rule. |
evalution_type |
The evaluation type of the data quality rule. |
rule_dimension |
The dimension of the data quality rule. |
threshold_percent |
The passing threshold ([0.0, 100.0]) of the data quality rule. |
result |
The result of the data quality rule. |
evaluated_row_count |
The number of rows evaluated against the data quality rule. This field is only valid for rules of PER_ROW evaluation type. |
passed_row_count |
The number of rows which passed a rule evaluation. This field is only valid for rules of PER_ROW evaluation type. |
null_row_count |
The number of rows with null values in the specified column. |
assertion_row_count |
The number of rows returned by the SQL statement in a SQL assertion rule. This field is only valid for SQL assertion rules. |
EvaluationType
The evaluation type of the data quality rule.
| Enums | |
|---|---|
EVALUATION_TYPE_UNSPECIFIED |
An unspecified evaluation type. |
PER_ROW |
The rule evaluation is done at per row level. |
AGGREGATE |
The rule evaluation is done for an aggregate of rows. |
Result
Whether the data quality rule passed or failed.
| Enums | |
|---|---|
RESULT_UNSPECIFIED |
An unspecified result. |
PASSED |
The data quality rule passed. |
FAILED |
The data quality rule failed. |
RuleType
The type of the data quality rule.
| Enums | |
|---|---|
RULE_TYPE_UNSPECIFIED |
An unspecified rule type. |
NON_NULL_EXPECTATION |
See DataQualityRule.NonNullExpectation. |
RANGE_EXPECTATION |
See DataQualityRule.RangeExpectation. |
REGEX_EXPECTATION |
See DataQualityRule.RegexExpectation. |
ROW_CONDITION_EXPECTATION |
See DataQualityRule.RowConditionExpectation. |
SET_EXPECTATION |
See DataQualityRule.SetExpectation. |
STATISTIC_RANGE_EXPECTATION |
See DataQualityRule.StatisticRangeExpectation. |
TABLE_CONDITION_EXPECTATION |
See DataQualityRule.TableConditionExpectation. |
UNIQUENESS_EXPECTATION |
See DataQualityRule.UniquenessExpectation. |
SQL_ASSERTION |
See DataQualityRule.SqlAssertion. |
DataQualitySpec
DataQualityScan related setting.
| Fields | |
|---|---|
rules[] |
Required. The list of rules to evaluate against a data source. At least one rule is required. |
sampling_percent |
Optional. The percentage of the records to be selected from the dataset for DataScan.
|
row_filter |
Optional. A filter applied to all rows in a single DataScan job. The filter needs to be a valid SQL expression for a WHERE clause in BigQuery standard SQL syntax. Example: col1 >= 0 AND col2 < 10 |
post_scan_actions |
Optional. Actions to take upon job completion. |
PostScanActions
The configuration of post scan actions of DataQualityScan.
| Fields | |
|---|---|
bigquery_export |
Optional. If set, results will be exported to the provided BigQuery table. |
notification_report |
Optional. If set, results will be sent to the provided notification receipts upon triggers. |
BigQueryExport
The configuration of BigQuery export post scan action.
| Fields | |
|---|---|
results_table |
Optional. The BigQuery table to export DataQualityScan results to. Format: //bigquery.googleapis.com/projects/PROJECT_ID/datasets/DATASET_ID/tables/TABLE_ID |
JobEndTrigger
This type has no fields.
This trigger is triggered whenever a scan job run ends, regardless of the result.
JobFailureTrigger
This type has no fields.
This trigger is triggered when the scan job itself fails, regardless of the result.
NotificationReport
The configuration of notification report post scan action.
| Fields | |
|---|---|
recipients |
Required. The recipients who will receive the notification report. |
score_threshold_trigger |
Optional. If set, report will be sent when score threshold is met. |
job_failure_trigger |
Optional. If set, report will be sent when a scan job fails. |
job_end_trigger |
Optional. If set, report will be sent when a scan job ends. |
Recipients
The individuals or groups who are designated to receive notifications upon triggers.
| Fields | |
|---|---|
emails[] |
Optional. The email recipients who will receive the DataQualityScan results report. |
ScoreThresholdTrigger
This trigger is triggered when the DQ score in the job result is less than a specified input score.
| Fields | |
|---|---|
score_threshold |
Optional. The score range is in [0,100]. |
DataScan
Represents a user-visible job which provides the insights for the related data source.
For example:
- Data Quality: generates queries based on the rules and runs against the data to get data quality check results.
- Data Profile: analyzes the data in table(s) and generates insights about the structure, content and relationships (such as null percent, cardinality, min/max/mean, etc).
| Fields | |
|---|---|
name |
Output only. The relative resource name of the scan, of the form: |
uid |
Output only. System generated globally unique ID for the scan. This ID will be different if the scan is deleted and re-created with the same name. |
description |
Optional. Description of the scan.
|
display_name |
Optional. User friendly display name.
|
labels |
Optional. User-defined labels for the scan. |
state |
Output only. Current state of the DataScan. |
create_time |
Output only. The time when the scan was created. |
update_time |
Output only. The time when the scan was last updated. |
data |
Required. The data source for DataScan. |
execution_spec |
Optional. DataScan execution settings. If not specified, the fields in it will use their default values. |
execution_status |
Output only. Status of the data scan execution. |
type |
Output only. The type of DataScan. |
Union field spec. Data scan related setting. The settings are required and immutable. After you configure the settings for one type of data scan, you can't change the data scan to a different type of data scan. spec can be only one of the following: |
|
data_quality_spec |
Settings for a data quality scan. |
data_profile_spec |
Settings for a data profile scan. |
Union field result. The result of the data scan. result can be only one of the following: |
|
data_quality_result |
Output only. The result of a data quality scan. |
data_profile_result |
Output only. The result of a data profile scan. |
ExecutionSpec
DataScan execution settings.
| Fields | |
|---|---|
trigger |
Optional. Spec related to how often and when a scan should be triggered. If not specified, the default is |
Union field When an option is selected for incremental scan, it cannot be unset or changed. If not specified, a data scan will run for all data in the table. |
|
field |
Immutable. The unnested field (of type Date or Timestamp) that contains values which monotonically increase over time. If not specified, a data scan will run for all data in the table. |
ExecutionStatus
Status of the data scan execution.
| Fields | |
|---|---|
latest_job_start_time |
The time when the latest DataScanJob started. |
latest_job_end_time |
The time when the latest DataScanJob ended. |
latest_job_create_time |
Optional. The time when the DataScanJob execution was created. |
DataScanEvent
These messages contain information about the execution of a datascan. The monitored resource is 'DataScan' Next ID: 13
| Fields | |
|---|---|
data_source |
The data source of the data scan |
job_id |
The identifier of the specific data scan job this log entry is for. |
create_time |
The time when the data scan job was created. |
start_time |
The time when the data scan job started to run. |
end_time |
The time when the data scan job finished. |
type |
The type of the data scan. |
state |
The status of the data scan job. |
message |
The message describing the data scan job event. |
spec_version |
A version identifier of the spec which was used to execute this job. |
trigger |
The trigger type of the data scan job. |
scope |
The scope of the data scan (e.g. full, incremental). |
post_scan_actions_result |
The result of post scan actions. |
Union field result. The result of the data scan job. result can be only one of the following: |
|
data_profile |
Data profile result for data profile type data scan. |
data_quality |
Data quality result for data quality type data scan. |
Union field appliedConfigs. The applied configs in the data scan job. appliedConfigs can be only one of the following: |
|
data_profile_configs |
Applied configs for data profile type data scan. |
data_quality_configs |
Applied configs for data quality type data scan. |
DataProfileAppliedConfigs
Applied configs for data profile type data scan job.
| Fields | |
|---|---|
sampling_percent |
The percentage of the records selected from the dataset for DataScan.
|
row_filter_applied |
Boolean indicating whether a row filter was applied in the DataScan job. |
column_filter_applied |
Boolean indicating whether a column filter was applied in the DataScan job. |
DataProfileResult
Data profile result for data scan job.
| Fields | |
|---|---|
row_count |
The count of rows processed in the data scan job. |
DataQualityAppliedConfigs
Applied configs for data quality type data scan job.
| Fields | |
|---|---|
sampling_percent |
The percentage of the records selected from the dataset for DataScan.
|
row_filter_applied |
Boolean indicating whether a row filter was applied in the DataScan job. |
DataQualityResult
Data quality result for data scan job.
| Fields | |
|---|---|
row_count |
The count of rows processed in the data scan job. |
passed |
Whether the data quality result was |
dimension_passed |
The result of each dimension for data quality result. The key of the map is the name of the dimension. The value is the bool value depicting whether the dimension result was |
score |
The table-level data quality score for the data scan job. The data quality score ranges between [0, 100] (up to two decimal points). |
dimension_score |
The score of each dimension for data quality result. The key of the map is the name of the dimension. The value is the data quality score for the dimension. The score ranges between [0, 100] (up to two decimal points). |
column_score |
The score of each column scanned in the data scan job. The key of the map is the name of the column. The value is the data quality score for the column. The score ranges between [0, 100] (up to two decimal points). |
PostScanActionsResult
Post scan actions result for data scan job.
| Fields | |
|---|---|
bigquery_export_result |
The result of BigQuery export post scan action. |
BigQueryExportResult
The result of BigQuery export post scan action.
| Fields | |
|---|---|
state |
Execution state for the BigQuery exporting. |
message |
Additional information about the BigQuery exporting. |
State
Execution state for the exporting.
| Enums | |
|---|---|
STATE_UNSPECIFIED |
The exporting state is unspecified. |
SUCCEEDED |
The exporting completed successfully. |
FAILED |
The exporting is no longer running due to an error. |
SKIPPED |
The exporting is skipped due to no valid scan result to export (usually caused by scan failed). |
ScanType
The type of the data scan.
| Enums | |
|---|---|
SCAN_TYPE_UNSPECIFIED |
An unspecified data scan type. |
DATA_PROFILE |
Data scan for data profile. |
DATA_QUALITY |
Data scan for data quality. |
Scope
The scope of job for the data scan.
| Enums | |
|---|---|
SCOPE_UNSPECIFIED |
An unspecified scope type. |
FULL |
Data scan runs on all of the data. |
INCREMENTAL |
Data scan runs on incremental data. |
State
The job state of the data scan.
| Enums | |
|---|---|
STATE_UNSPECIFIED |
Unspecified job state. |
STARTED |
Data scan job started. |
SUCCEEDED |
Data scan job successfully completed. |
FAILED |
Data scan job was unsuccessful. |
CANCELLED |
Data scan job was cancelled. |
CREATED |
Data scan job was createed. |
Trigger
The trigger type for the data scan.
| Enums | |
|---|---|
TRIGGER_UNSPECIFIED |
An unspecified trigger type. |
ON_DEMAND |
Data scan triggers on demand. |
SCHEDULE |
Data scan triggers as per schedule. |
DataScanJob
A DataScanJob represents an instance of DataScan execution.
| Fields | |
|---|---|
name |
Output only. The relative resource name of the DataScanJob, of the form: |
uid |
Output only. System generated globally unique ID for the DataScanJob. |
create_time |
Output only. The time when the DataScanJob was created. |
start_time |
Output only. The time when the DataScanJob was started. |
end_time |
Output only. The time when the DataScanJob ended. |
state |
Output only. Execution state for the DataScanJob. |
message |
Output only. Additional information about the current state. |
type |
Output only. The type of the parent DataScan. |
Union field spec. Data scan related setting. spec can be only one of the following: |
|
data_quality_spec |
Output only. Settings for a data quality scan. |
data_profile_spec |
Output only. Settings for a data profile scan. |
Union field result. The result of the data scan. result can be only one of the following: |
|
data_quality_result |
Output only. The result of a data quality scan. |
data_profile_result |
Output only. The result of a data profile scan. |
State
Execution state for the DataScanJob.
| Enums | |
|---|---|
STATE_UNSPECIFIED |
The DataScanJob state is unspecified. |
RUNNING |
The DataScanJob is running. |
CANCELING |
The DataScanJob is canceling. |
CANCELLED |
The DataScanJob cancellation was successful. |
SUCCEEDED |
The DataScanJob completed successfully. |
FAILED |
The DataScanJob is no longer running due to an error. |
PENDING |
The DataScanJob has been created but not started to run yet. |
DataScanType
The type of data scan.
| Enums | |
|---|---|
DATA_SCAN_TYPE_UNSPECIFIED |
The data scan type is unspecified. |
DATA_QUALITY |
Data quality scan. |
DATA_PROFILE |
Data profile scan. |
DataSource
The data source for DataScan.
| Fields | |
|---|---|
Union field source. The source is required and immutable. Once it is set, it cannot be change to others. source can be only one of the following: |
|
entity |
Immutable. The Dataplex entity that represents the data source (e.g. BigQuery table) for DataScan, of the form: |
resource |
Immutable. The service-qualified full resource name of the cloud resource for a DataScan job to scan against. The field could be: BigQuery table of type "TABLE" for DataProfileScan/DataQualityScan Format: //bigquery.googleapis.com/projects/PROJECT_ID/datasets/DATASET_ID/tables/TABLE_ID |
DataTaxonomy
DataTaxonomy represents a set of hierarchical DataAttributes resources, grouped with a common theme Eg: 'SensitiveDataTaxonomy' can have attributes to manage PII data. It is defined at project level.
| Fields | |
|---|---|
name |
Output only. The relative resource name of the DataTaxonomy, of the form: projects/{project_number}/locations/{location_id}/dataTaxonomies/{data_taxonomy_id}. |
uid |
Output only. System generated globally unique ID for the dataTaxonomy. This ID will be different if the DataTaxonomy is deleted and re-created with the same name. |
create_time |
Output only. The time when the DataTaxonomy was created. |
update_time |
Output only. The time when the DataTaxonomy was last updated. |
description |
Optional. Description of the DataTaxonomy. |
display_name |
Optional. User friendly display name. |
labels |
Optional. User-defined labels for the DataTaxonomy. |
attribute_count |
Output only. The number of attributes in the DataTaxonomy. |
etag |
This checksum is computed by the server based on the value of other fields, and may be sent on update and delete requests to ensure the client has an up-to-date value before proceeding. |
class_count |
Output only. The number of classes in the DataTaxonomy. |
DeleteAspectTypeRequest
Delele AspectType Request.
| Fields | |
|---|---|
name |
Required. The resource name of the AspectType: Authorization requires the following IAM permission on the specified resource
|
etag |
Optional. If the client provided etag value does not match the current etag value, the DeleteAspectTypeRequest method returns an ABORTED error response. |
DeleteAssetRequest
Delete asset request.
| Fields | |
|---|---|
name |
Required. The resource name of the asset: Authorization requires the following IAM permission on the specified resource
|
DeleteContentRequest
Delete content request.
| Fields | |
|---|---|
name |
Required. The resource name of the content: projects/{project_id}/locations/{location_id}/lakes/{lake_id}/content/{content_id} Authorization requires the following IAM permission on the specified resource
|
DeleteDataAttributeBindingRequest
Delete DataAttributeBinding request.
| Fields | |
|---|---|
name |
Required. The resource name of the DataAttributeBinding: projects/{project_number}/locations/{location_id}/dataAttributeBindings/{data_attribute_binding_id} Authorization requires the following IAM permission on the specified resource
|
etag |
Required. If the client provided etag value does not match the current etag value, the DeleteDataAttributeBindingRequest method returns an ABORTED error response. Etags must be used when calling the DeleteDataAttributeBinding. |
DeleteDataAttributeRequest
Delete DataAttribute request.
| Fields | |
|---|---|
name |
Required. The resource name of the DataAttribute: projects/{project_number}/locations/{location_id}/dataTaxonomies/{dataTaxonomy}/attributes/{data_attribute_id} Authorization requires the following IAM permission on the specified resource
|
etag |
Optional. If the client provided etag value does not match the current etag value, the DeleteDataAttribute method returns an ABORTED error response. |
DeleteDataScanRequest
Delete dataScan request.
| Fields | |
|---|---|
name |
Required. The resource name of the dataScan: Authorization requires the following IAM permission on the specified resource
|
DeleteDataTaxonomyRequest
Delete DataTaxonomy request.
| Fields | |
|---|---|
name |
Required. The resource name of the DataTaxonomy: projects/{project_number}/locations/{location_id}/dataTaxonomies/{data_taxonomy_id} Authorization requires the following IAM permission on the specified resource
|
etag |
Optional. If the client provided etag value does not match the current etag value,the DeleteDataTaxonomy method returns an ABORTED error. |
DeleteEntityRequest
Delete a metadata entity request.
| Fields | |
|---|---|
name |
Required. The resource name of the entity: Authorization requires the following IAM permission on the specified resource
|
etag |
Required. The etag associated with the entity, which can be retrieved with a [GetEntity][] request. |
DeleteEntryGroupRequest
Delete EntryGroup Request.
| Fields | |
|---|---|
name |
Required. The resource name of the EntryGroup: Authorization requires the following IAM permission on the specified resource
|
etag |
Optional. If the client provided etag value does not match the current etag value, the DeleteEntryGroupRequest method returns an ABORTED error response. |
DeleteEntryRequest
Delete Entry request.
| Fields | |
|---|---|
name |
Required. The resource name of the Entry: |
DeleteEntryTypeRequest
Delele EntryType Request.
| Fields | |
|---|---|
name |
Required. The resource name of the EntryType: Authorization requires the following IAM permission on the specified resource
|
etag |
Optional. If the client provided etag value does not match the current etag value, the DeleteEntryTypeRequest method returns an ABORTED error response. |
DeleteEnvironmentRequest
Delete environment request.
| Fields | |
|---|---|
name |
Required. The resource name of the environment: Authorization requires the following IAM permission on the specified resource
|
DeleteLakeRequest
Delete lake request.
| Fields | |
|---|---|
name |
Required. The resource name of the lake: Authorization requires the following IAM permission on the specified resource
|
DeletePartitionRequest
Delete metadata partition request.
| Fields | |
|---|---|
name |
Required. The resource name of the partition. format: Authorization requires the following IAM permission on the specified resource
|
etag |
Optional. The etag associated with the partition. |
DeleteTaskRequest
Delete task request.
| Fields | |
|---|---|
name |
Required. The resource name of the task: Authorization requires the following IAM permission on the specified resource
|
DeleteZoneRequest
Delete zone request.
| Fields | |
|---|---|
name |
Required. The resource name of the zone: Authorization requires the following IAM permission on the specified resource
|
DiscoveryEvent
The payload associated with Discovery data processing.
| Fields | |
|---|---|
message |
The log message. |
lake_id |
The id of the associated lake. |
zone_id |
The id of the associated zone. |
asset_id |
The id of the associated asset. |
data_location |
The data location associated with the event. |
type |
The type of the event being logged. |
Union field details. Additional details about the event. details can be only one of the following: |
|
config |
Details about discovery configuration in effect. |
entity |
Details about the entity associated with the event. |
partition |
Details about the partition associated with the event. |
action |
Details about the action associated with the event. |
ActionDetails
Details about the action.
| Fields | |
|---|---|
type |
The type of action. Eg. IncompatibleDataSchema, InvalidDataFormat |
ConfigDetails
Details about configuration events.
| Fields | |
|---|---|
parameters |
A list of discovery configuration parameters in effect. The keys are the field paths within DiscoverySpec. Eg. includePatterns, excludePatterns, csvOptions.disableTypeInference, etc. |
EntityDetails
Details about the entity.
| Fields | |
|---|---|
entity |
The name of the entity resource. The name is the fully-qualified resource name. |
type |
The type of the entity resource. |
EntityType
The type of the entity.
| Enums | |
|---|---|
ENTITY_TYPE_UNSPECIFIED |
An unspecified event type. |
TABLE |
Entities representing structured data. |
FILESET |
Entities representing unstructured data. |
EventType
The type of the event.
| Enums | |
|---|---|
EVENT_TYPE_UNSPECIFIED |
An unspecified event type. |
CONFIG |
An event representing discovery configuration in effect. |
ENTITY_CREATED |
An event representing a metadata entity being created. |
ENTITY_UPDATED |
An event representing a metadata entity being updated. |
ENTITY_DELETED |
An event representing a metadata entity being deleted. |
PARTITION_CREATED |
An event representing a partition being created. |
PARTITION_UPDATED |
An event representing a partition being updated. |
PARTITION_DELETED |
An event representing a partition being deleted. |
PartitionDetails
Details about the partition.
| Fields | |
|---|---|
partition |
The name to the partition resource. The name is the fully-qualified resource name. |
entity |
The name to the containing entity resource. The name is the fully-qualified resource name. |
type |
The type of the containing entity resource. |
sampled_data_locations[] |
The locations of the data items (e.g., a Cloud Storage objects) sampled for metadata inference. |
Entity
Represents tables and fileset metadata contained within a zone.
| Fields | |
|---|---|
name |
Output only. The resource name of the entity, of the form: |
display_name |
Optional. Display name must be shorter than or equal to 256 characters. |
description |
Optional. User friendly longer description text. Must be shorter than or equal to 1024 characters. |
create_time |
Output only. The time when the entity was created. |
update_time |
Output only. The time when the entity was last updated. |
id |
Required. A user-provided entity ID. It is mutable, and will be used as the published table name. Specifying a new ID in an update entity request will override the existing value. The ID must contain only letters (a-z, A-Z), numbers (0-9), and underscores, and consist of 256 or fewer characters. |
etag |
Optional. The etag associated with the entity, which can be retrieved with a [GetEntity][] request. Required for update and delete requests. |
type |
Required. Immutable. The type of entity. |
asset |
Required. Immutable. The ID of the asset associated with the storage location containing the entity data. The entity must be with in the same zone with the asset. |
data_path |
Required. Immutable. The storage path of the entity data. For Cloud Storage data, this is the fully-qualified path to the entity, such as |
data_path_pattern |
Optional. The set of items within the data path constituting the data in the entity, represented as a glob path. Example: |
catalog_entry |
Output only. The name of the associated Data Catalog entry. |
system |
Required. Immutable. Identifies the storage system of the entity data. |
format |
Required. Identifies the storage format of the entity data. It does not apply to entities with data stored in BigQuery. |
compatibility |
Output only. Metadata stores that the entity is compatible with. |
access |
Output only. Identifies the access mechanism to the entity. Not user settable. |
uid |
Output only. System generated unique ID for the Entity. This ID will be different if the Entity is deleted and re-created with the same name. |
schema |
Required. The description of the data structure and layout. The schema is not included in list responses. It is only included in |
CompatibilityStatus
Provides compatibility information for various metadata stores.
| Fields | |
|---|---|
hive_metastore |
Output only. Whether this entity is compatible with Hive Metastore. |
bigquery |
Output only. Whether this entity is compatible with BigQuery. |
Compatibility
Provides compatibility information for a specific metadata store.
| Fields | |
|---|---|
compatible |
Output only. Whether the entity is compatible and can be represented in the metadata store. |
reason |
Output only. Provides additional detail if the entity is incompatible with the metadata store. |
Type
The type of entity.
| Enums | |
|---|---|
TYPE_UNSPECIFIED |
Type unspecified. |
TABLE |
Structured and semi-structured data. |
FILESET |
Unstructured data. |
Entry
An entry is a representation of a data resource that can be described by various metadata.
| Fields | |
|---|---|
name |
Identifier. The relative resource name of the entry, in the format |
entry_type |
Required. Immutable. The relative resource name of the entry type that was used to create this entry, in the format |
create_time |
Output only. The time when the entry was created in Dataplex. |
update_time |
Output only. The time when the entry was last updated in Dataplex. |
aspects |
Optional. The aspects that are attached to the entry. Depending on how the aspect is attached to the entry, the format of the aspect key can be one of the following:
|
parent_entry |
Optional. Immutable. The resource name of the parent entry. |
fully_qualified_name |
Optional. A name for the entry that can be referenced by an external system. For more information, see Fully qualified names. The maximum size of the field is 4000 characters. |
entry_source |
Optional. Information related to the source system of the data resource that is represented by the entry. |
EntryGroup
An Entry Group represents a logical grouping of one or more Entries.
| Fields | |
|---|---|
name |
Output only. The relative resource name of the EntryGroup, in the format projects/{project_id_or_number}/locations/{location_id}/entryGroups/{entry_group_id}. |
uid |
Output only. System generated globally unique ID for the EntryGroup. If you delete and recreate the EntryGroup with the same name, this ID will be different. |
create_time |
Output only. The time when the EntryGroup was created. |
update_time |
Output only. The time when the EntryGroup was last updated. |
description |
Optional. Description of the EntryGroup. |
display_name |
Optional. User friendly display name. |
labels |
Optional. User-defined labels for the EntryGroup. |
etag |
This checksum is computed by the service, and might be sent on update and delete requests to ensure the client has an up-to-date value before proceeding. |
transfer_status |
Output only. Denotes the transfer status of the Entry Group. It is unspecified for Entry Group created from Dataplex API. |
EntrySource
Information related to the source system of the data resource that is represented by the entry.
| Fields | |
|---|---|
resource |
The name of the resource in the source system. Maximum length is 4,000 characters. |
system |
The name of the source system. Maximum length is 64 characters. |
platform |
The platform containing the source system. Maximum length is 64 characters. |
display_name |
A user-friendly display name. Maximum length is 500 characters. |
description |
A description of the data resource. Maximum length is 2,000 characters. |
labels |
User-defined labels. The maximum size of keys and values is 128 characters each. |
ancestors[] |
Immutable. The entries representing the ancestors of the data resource in the source system. |
create_time |
The time when the resource was created in the source system. |
update_time |
The time when the resource was last updated in the source system. If the entry exists in the system and its |
location |
Output only. Location of the resource in the source system. You can search the entry by this location. By default, this should match the location of the entry group containing this entry. A different value allows capturing the source location for data external to Google Cloud. |
Ancestor
Information about individual items in the hierarchy that is associated with the data resource.
| Fields | |
|---|---|
name |
Optional. The name of the ancestor resource. |
type |
Optional. The type of the ancestor resource. |
EntryType
Entry Type is a template for creating Entries.
| Fields | |
|---|---|
name |
Output only. The relative resource name of the EntryType, of the form: projects/{project_number}/locations/{location_id}/entryTypes/{entry_type_id}. |
uid |
Output only. System generated globally unique ID for the EntryType. This ID will be different if the EntryType is deleted and re-created with the same name. |
create_time |
Output only. The time when the EntryType was created. |
update_time |
Output only. The time when the EntryType was last updated. |
description |
Optional. Description of the EntryType. |
display_name |
Optional. User friendly display name. |
labels |
Optional. User-defined labels for the EntryType. |
etag |
Optional. This checksum is computed by the service, and might be sent on update and delete requests to ensure the client has an up-to-date value before proceeding. |
type_aliases[] |
Optional. Indicates the classes this Entry Type belongs to, for example, TABLE, DATABASE, MODEL. |
platform |
Optional. The platform that Entries of this type belongs to. |
system |
Optional. The system that Entries of this type belongs to. Examples include CloudSQL, MariaDB etc |
required_aspects[] |
AspectInfo for the entry type. |
authorization |
Immutable. Authorization defined for this type. |
AspectInfo
| Fields | |
|---|---|
type |
Required aspect type for the entry type. |
Authorization
Authorization for an Entry Type.
| Fields | |
|---|---|
alternate_use_permission |
Immutable. The IAM permission grantable on the Entry Group to allow access to instantiate Entries of Dataplex owned Entry Types, only settable for Dataplex owned Types. |
EntryView
View for controlling which parts of an entry are to be returned.
| Enums | |
|---|---|
ENTRY_VIEW_UNSPECIFIED |
Unspecified EntryView. Defaults to FULL. |
BASIC |
Returns entry only, without aspects. |
FULL |
Returns all required aspects as well as the keys of all non-required aspects. |
CUSTOM |
Returns aspects matching custom fields in GetEntryRequest. If the number of aspects exceeds 100, the first 100 will be returned. |
ALL |
Returns all aspects. If the number of aspects exceeds 100, the first 100 will be returned. |
Environment
Environment represents a user-visible compute infrastructure for analytics within a lake.
| Fields | |
|---|---|
name |
Output only. The relative resource name of the environment, of the form: projects/{project_id}/locations/{location_id}/lakes/{lake_id}/environment/{environment_id} |
display_name |
Optional. User friendly display name. |
uid |
Output only. System generated globally unique ID for the environment. This ID will be different if the environment is deleted and re-created with the same name. |
create_time |
Output only. Environment creation time. |
update_time |
Output only. The time when the environment was last updated. |
labels |
Optional. User defined labels for the environment. |
description |
Optional. Description of the environment. |
state |
Output only. Current state of the environment. |
infrastructure_spec |
Required. Infrastructure specification for the Environment. |
session_spec |
Optional. Configuration for sessions created for this environment. |
session_status |
Output only. Status of sessions created for this environment. |
endpoints |
Output only. URI Endpoints to access sessions associated with the Environment. |
Endpoints
URI Endpoints to access sessions associated with the Environment.
| Fields | |
|---|---|
notebooks |
Output only. URI to serve notebook APIs |
sql |
Output only. URI to serve SQL APIs |
InfrastructureSpec
Configuration for the underlying infrastructure used to run workloads.
| Fields | |
|---|---|
Union field resources. Hardware config resources can be only one of the following: |
|
compute |
Optional. Compute resources needed for analyze interactive workloads. |
Union field runtime. Software config runtime can be only one of the following: |
|
os_image |
Required. Software Runtime Configuration for analyze interactive workloads. |
ComputeResources
Compute resources associated with the analyze interactive workloads.
| Fields | |
|---|---|
disk_size_gb |
Optional. Size in GB of the disk. Default is 100 GB. |
node_count |
Optional. Total number of nodes in the sessions created for this environment. |
max_node_count |
Optional. Max configurable nodes. If max_node_count > node_count, then auto-scaling is enabled. |
OsImageRuntime
Software Runtime Configuration to run Analyze.
| Fields | |
|---|---|
image_version |
Required. Dataplex Image version. |
java_libraries[] |
Optional. List of Java jars to be included in the runtime environment. Valid input includes Cloud Storage URIs to Jar binaries. For example, gs://bucket-name/my/path/to/file.jar |
python_packages[] |
Optional. A list of python packages to be installed. Valid formats include Cloud Storage URI to a PIP installable library. For example, gs://bucket-name/my/path/to/lib.tar.gz |
properties |
Optional. Spark properties to provide configuration for use in sessions created for this environment. The properties to set on daemon config files. Property keys are specified in |
SessionSpec
Configuration for sessions created for this environment.
| Fields | |
|---|---|
max_idle_duration |
Optional. The idle time configuration of the session. The session will be auto-terminated at the end of this period. |
enable_fast_startup |
Optional. If True, this causes sessions to be pre-created and available for faster startup to enable interactive exploration use-cases. This defaults to False to avoid additional billed charges. These can only be set to True for the environment with name set to "default", and with default configuration. |
SessionStatus
Status of sessions created for this environment.
| Fields | |
|---|---|
active |
Output only. Queries over sessions to mark whether the environment is currently active or not |
GenerateDataQualityRulesRequest
Request details for generating data quality rule recommendations.
| Fields | |
|---|---|
name |
Required. The name must be one of the following:
|
GenerateDataQualityRulesResponse
Response details for data quality rule recommendations.
| Fields | |
|---|---|
rule[] |
The data quality rules that Dataplex generates based on the results of a data profiling scan. |
GetAspectTypeRequest
Get AspectType request.
| Fields | |
|---|---|
name |
Required. The resource name of the AspectType: Authorization requires the following IAM permission on the specified resource
|
GetAssetRequest
Get asset request.
| Fields | |
|---|---|
name |
Required. The resource name of the asset: Authorization requires the following IAM permission on the specified resource
|
GetContentRequest
Get content request.
| Fields | |
|---|---|
name |
Required. The resource name of the content: projects/{project_id}/locations/{location_id}/lakes/{lake_id}/content/{content_id} Authorization requires the following IAM permission on the specified resource
|
view |
Optional. Specify content view to make a partial request. |
ContentView
Specifies whether the request should return the full or the partial representation.
| Enums | |
|---|---|
CONTENT_VIEW_UNSPECIFIED |
Content view not specified. Defaults to BASIC. The API will default to the BASIC view. |
BASIC |
Will not return the data_text field. |
FULL |
Returns the complete proto. |
GetDataAttributeBindingRequest
Get DataAttributeBinding request.
| Fields | |
|---|---|
name |
Required. The resource name of the DataAttributeBinding: projects/{project_number}/locations/{location_id}/dataAttributeBindings/{data_attribute_binding_id} Authorization requires the following IAM permission on the specified resource
|
GetDataAttributeRequest
Get DataAttribute request.
| Fields | |
|---|---|
name |
Required. The resource name of the dataAttribute: projects/{project_number}/locations/{location_id}/dataTaxonomies/{dataTaxonomy}/attributes/{data_attribute_id} Authorization requires the following IAM permission on the specified resource
|
GetDataScanJobRequest
Get DataScanJob request.
| Fields | |
|---|---|
name |
Required. The resource name of the DataScanJob: Authorization requires the following IAM permission on the specified resource
|
view |
Optional. Select the DataScanJob view to return. Defaults to |
DataScanJobView
DataScanJob view options.
| Enums | |
|---|---|
DATA_SCAN_JOB_VIEW_UNSPECIFIED |
The API will default to the BASIC view. |
BASIC |
Basic view that does not include spec and result. |
FULL |
Include everything. |
GetDataScanRequest
Get dataScan request.
| Fields | |
|---|---|
name |
Required. The resource name of the dataScan: Authorization requires the following IAM permission on the specified resource
|
view |
Optional. Select the DataScan view to return. Defaults to |
DataScanView
DataScan view options.
| Enums | |
|---|---|
DATA_SCAN_VIEW_UNSPECIFIED |
The API will default to the BASIC view. |
BASIC |
Basic view that does not include spec and result. |
FULL |
Include everything. |
GetDataTaxonomyRequest
Get DataTaxonomy request.
| Fields | |
|---|---|
name |
Required. The resource name of the DataTaxonomy: projects/{project_number}/locations/{location_id}/dataTaxonomies/{data_taxonomy_id} Authorization requires the following IAM permission on the specified resource
|
GetEntityRequest
Get metadata entity request.
| Fields | |
|---|---|
name |
Required. The resource name of the entity: Authorization requires the following IAM permission on the specified resource
|
view |
Optional. Used to select the subset of entity information to return. Defaults to |
EntityView
Entity views for get entity partial result.
| Enums | |
|---|---|
ENTITY_VIEW_UNSPECIFIED |
The API will default to the BASIC view. |
BASIC |
Minimal view that does not include the schema. |
SCHEMA |
Include basic information and schema. |
FULL |
Include everything. Currently, this is the same as the SCHEMA view. |
GetEntryGroupRequest
Get EntryGroup request.
| Fields | |
|---|---|
name |
Required. The resource name of the EntryGroup: Authorization requires the following IAM permission on the specified resource
|
GetEntryRequest
Get Entry request.
| Fields | |
|---|---|
name |
Required. The resource name of the Entry: |
view |
Optional. View to control which parts of an entry the service should return. |
aspect_types[] |
Optional. Limits the aspects returned to the provided aspect types. It only works for CUSTOM view. |
paths[] |
Optional. Limits the aspects returned to those associated with the provided paths within the Entry. It only works for CUSTOM view. |
GetEntryTypeRequest
Get EntryType request.
| Fields | |
|---|---|
name |
Required. The resource name of the EntryType: Authorization requires the following IAM permission on the specified resource
|
GetEnvironmentRequest
Get environment request.
| Fields | |
|---|---|
name |
Required. The resource name of the environment: Authorization requires the following IAM permission on the specified resource
|
GetJobRequest
Get job request.
| Fields | |
|---|---|
name |
Required. The resource name of the job: Authorization requires the following IAM permission on the specified resource
|
GetLakeRequest
Get lake request.
| Fields | |
|---|---|
name |
Required. The resource name of the lake: Authorization requires the following IAM permission on the specified resource
|
GetMetadataJobRequest
Get metadata job request.
| Fields | |
|---|---|
name |
Required. The resource name of the metadata job, in the format Authorization requires the following IAM permission on the specified resource
|
GetPartitionRequest
Get metadata partition request.
| Fields | |
|---|---|
name |
Required. The resource name of the partition: Authorization requires the following IAM permission on the specified resource
|
GetTaskRequest
Get task request.
| Fields | |
|---|---|
name |
Required. The resource name of the task: Authorization requires the following IAM permission on the specified resource
|
GetZoneRequest
Get zone request.
| Fields | |
|---|---|
name |
Required. The resource name of the zone: Authorization requires the following IAM permission on the specified resource
|
GovernanceEvent
Payload associated with Governance related log events.
| Fields | |
|---|---|
message |
The log message. |
event_type |
The type of the event. |
entity |
Entity resource information if the log event is associated with a specific entity. |
Entity
Information about Entity resource that the log event is associated with.
| Fields | |
|---|---|
entity |
The Entity resource the log event is associated with. Format: |
entity_type |
Type of entity. |
EntityType
Type of entity.
| Enums | |
|---|---|
ENTITY_TYPE_UNSPECIFIED |
An unspecified Entity type. |
TABLE |
Table entity type. |
FILESET |
Fileset entity type. |
EventType
Type of governance log event.
| Enums | |
|---|---|
EVENT_TYPE_UNSPECIFIED |
An unspecified event type. |
RESOURCE_IAM_POLICY_UPDATE |
Resource IAM policy update event. |
BIGQUERY_TABLE_CREATE |
BigQuery table create event. |
BIGQUERY_TABLE_UPDATE |
BigQuery table update event. |
BIGQUERY_TABLE_DELETE |
BigQuery table delete event. |
BIGQUERY_CONNECTION_CREATE |
BigQuery connection create event. |
BIGQUERY_CONNECTION_UPDATE |
BigQuery connection update event. |
BIGQUERY_CONNECTION_DELETE |
BigQuery connection delete event. |
BIGQUERY_TAXONOMY_CREATE |
BigQuery taxonomy created. |
BIGQUERY_POLICY_TAG_CREATE |
BigQuery policy tag created. |
BIGQUERY_POLICY_TAG_DELETE |
BigQuery policy tag deleted. |
BIGQUERY_POLICY_TAG_SET_IAM_POLICY |
BigQuery set iam policy for policy tag. |
ACCESS_POLICY_UPDATE |
Access policy update event. |
GOVERNANCE_RULE_MATCHED_RESOURCES |
Number of resources matched with particular Query. |
GOVERNANCE_RULE_SEARCH_LIMIT_EXCEEDS |
Rule processing exceeds the allowed limit. |
GOVERNANCE_RULE_ERRORS |
Rule processing errors. |
GOVERNANCE_RULE_PROCESSING |
Governance rule processing Event. |
ImportItem
An object that describes the values that you want to set for an entry and its attached aspects when you import metadata. Used when you run a metadata import job. See CreateMetadataJob.
You provide a collection of import items in a metadata import file. For more information about how to create a metadata import file, see Metadata import file.
| Fields | |
|---|---|
entry |
Information about an entry and its attached aspects. |
update_mask |
The fields to update, in paths that are relative to the In The Dataplex also determines which entries and aspects to modify by comparing the values and timestamps that you provide in the metadata import file with the values and timestamps that exist in your project. For more information, see Comparison logic. |
aspect_keys[] |
The aspects to modify. Supports the following syntaxes:
Replace If you leave this field empty, it is treated as specifying exactly those aspects that are present within the specified entry. In |
Job
A job represents an instance of a task.
| Fields | |
|---|---|
name |
Output only. The relative resource name of the job, of the form: |
uid |
Output only. System generated globally unique ID for the job. |
start_time |
Output only. The time when the job was started. |
end_time |
Output only. The time when the job ended. |
state |
Output only. Execution state for the job. |
retry_count |
Output only. The number of times the job has been retried (excluding the initial attempt). |
service |
Output only. The underlying service running a job. |