This page shows how to get started with the Cloud Client Libraries for the
BigQuery Migration API. Client libraries make it easier to access
Google Cloud APIs from a supported language. Although you can use
Google Cloud APIs directly by making raw requests to the server, client
libraries provide simplifications that significantly reduce the amount of code
you need to write.
Read more about the Cloud Client Libraries
and the older Google API Client Libraries in
Client libraries explained.
Install the client library
Set up authentication
To authenticate calls to Google Cloud APIs, client libraries support
Application Default Credentials (ADC);
the libraries look for credentials in a set of defined locations and use those credentials
to authenticate requests to the API. With ADC, you can make
credentials available to your application in a variety of environments, such as local
development or production, without needing to modify your application code.
For production environments, the way you set up ADC depends on the service
and context. For more information, see Set up Application Default Credentials.
For a local development environment, you can set up ADC with the credentials
that are associated with your Google Account:
-
Install the Google Cloud CLI, then
initialize it by running the following command:
gcloud init
-
If you're using a local shell, then create local authentication credentials for your user
account:
gcloud auth application-default login
You don't need to do this if you're using Cloud Shell.
A sign-in screen appears. After you sign in, your credentials are stored in the
local credential file used by ADC.
Use the client library
The following example demonstrates some basic interactions with the BigQuery Migration API.
Go
// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The bigquery_migration_quickstart application demonstrates basic usage of the
// BigQuery migration API by executing a workflow that performs a batch SQL
// translation task.
package main
import (
"context"
"flag"
"fmt"
"log"
"time"
migration "cloud.google.com/go/bigquery/migration/apiv2"
"cloud.google.com/go/bigquery/migration/apiv2/migrationpb"
)
func main() {
// Define command line flags for controlling the behavior of this quickstart.
projectID := flag.String("project_id", "", "Cloud Project ID.")
location := flag.String("location", "us", "BigQuery Migration location used for interactions.")
outputPath := flag.String("output", "", "Cloud Storage path for translated resources.")
// Parse flags and do some minimal validation.
flag.Parse()
if *projectID == "" {
log.Fatal("empty --project_id specified, please provide a valid project ID")
}
if *location == "" {
log.Fatal("empty --location specified, please provide a valid location")
}
if *outputPath == "" {
log.Fatalf("empty --output specified, please provide a valid cloud storage path")
}
ctx := context.Background()
migClient, err := migration.NewClient(ctx)
if err != nil {
log.Fatalf("migration.NewClient: %v", err)
}
defer migClient.Close()
workflow, err := executeTranslationWorkflow(ctx, migClient, *projectID, *location, *outputPath)
if err != nil {
log.Fatalf("workflow execution failed: %v\n", err)
}
reportWorkflowStatus(workflow)
}
// executeTranslationWorkflow constructs a migration workflow that performs batch SQL translation.
func executeTranslationWorkflow(ctx context.Context, client *migration.Client, projectID, location, outPath string) (*migrationpb.MigrationWorkflow, error) {
// Construct the workflow creation request. In this workflow, we have only a single translation task present.
req := &migrationpb.CreateMigrationWorkflowRequest{
Parent: fmt.Sprintf("projects/%s/locations/%s", projectID, location),
MigrationWorkflow: &migrationpb.MigrationWorkflow{
DisplayName: "example SQL conversion",
Tasks: map[string]*migrationpb.MigrationTask{
"example_conversion": {
Type: "Translation_Teradata2BQ",
TaskDetails: &migrationpb.MigrationTask_TranslationConfigDetails{
TranslationConfigDetails: &migrationpb.TranslationConfigDetails{
SourceLocation: &migrationpb.TranslationConfigDetails_GcsSourcePath{
GcsSourcePath: "gs://cloud-samples-data/bigquery/migration/translation/input/",
},
TargetLocation: &migrationpb.TranslationConfigDetails_GcsTargetPath{
GcsTargetPath: outPath,
},
SourceDialect: &migrationpb.Dialect{
DialectValue: &migrationpb.Dialect_TeradataDialect{
TeradataDialect: &migrationpb.TeradataDialect{
Mode: migrationpb.TeradataDialect_SQL,
},
},
},
TargetDialect: &migrationpb.Dialect{
DialectValue: &migrationpb.Dialect_BigqueryDialect{},
},
},
},
},
},
},
}
// Create the workflow using the request.
workflow, err := client.CreateMigrationWorkflow(ctx, req)
if err != nil {
return nil, fmt.Errorf("CreateMigrationWorkflow: %w", err)
}
fmt.Printf("workflow created: %s", workflow.GetName())
// This is an asyncronous process, so we now poll the workflow
// until completion or a suitable timeout has elapsed.
timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
defer cancel()
for {
select {
case <-timeoutCtx.Done():
return nil, fmt.Errorf("task %s didn't complete due to context expiring", workflow.GetName())
default:
polledWorkflow, err := client.GetMigrationWorkflow(timeoutCtx, &migrationpb.GetMigrationWorkflowRequest{
Name: workflow.GetName(),
})
if err != nil {
return nil, fmt.Errorf("polling ended in error: %w", err)
}
if polledWorkflow.GetState() == migrationpb.MigrationWorkflow_COMPLETED {
// polledWorkflow contains the most recent metadata about the workflow, so we return that.
return polledWorkflow, nil
}
// workflow still isn't complete, so sleep briefly before polling again.
time.Sleep(5 * time.Second)
}
}
}
// reportWorkflowStatus prints information about the workflow execution in a more human readable format.
func reportWorkflowStatus(workflow *migrationpb.MigrationWorkflow) {
fmt.Printf("Migration workflow %s ended in state %s.\n", workflow.GetName(), workflow.GetState().String())
for k, task := range workflow.GetTasks() {
fmt.Printf(" - Task %s had id %s", k, task.GetId())
if task.GetProcessingError() != nil {
fmt.Printf(" with processing error: %s", task.GetProcessingError().GetReason())
}
fmt.Println()
}
}
Additional resources
Go
The following list contains links to more resources related to the
client library for Go:
Java
The following list contains links to more resources related to the
client library for Java:
Python
The following list contains links to more resources related to the
client library for Python:
What's next?
For more background, see the introduction to BigQuery Migration Service page.
