Creating and Managing Transfers Programmatically

This page shows you how to use Storage Transfer Service directly through the REST API, and programmatically with Java and Python in two common scenarios. To create a transfer using the Google Cloud Platform Console, see Creating and Managing Transfers with the Console.

Transfer from Amazon S3 to Google Cloud Storage

In this example, you'll learn how to move files from Amazon S3 to a Google Cloud Storage bucket. Be sure to review Configuring Access and Pricing to understand the implications of moving data from Amazon S3 to Cloud Storage.

To create the transfer job

REST

Request using transferJobs create:
POST https://storagetransfer.googleapis.com/v1/transferJobs
{
    "description": "YOUR DESCRIPTION",
    "status": "ENABLED",
    "projectId": "PROJECT_ID",
    "schedule": {
        "scheduleStartDate": {
            "day": 1,
            "month": 1,
            "year": 2015
        },
        "scheduleEndDate": {
            "day": 1,
            "month": 1,
            "year": 2015
        },
        "startTimeOfDay": {
            "hours": 1,
            "minutes": 1
        }
    },
    "transferSpec": {
        "awsS3DataSource": {
            "bucketName": "AWS_SOURCE_NAME",
            "awsAccessKey": {
                "accessKeyId": "AWS_ACCESS_KEY_ID",
                "secretAccessKey": "AWS_SECRET_ACCESS_KEY"
            }
        },
        "gcsDataSink": {
            "bucketName": "GCS_SINK_NAME"
        },
    }
}
Response:
200 OK
{
    "transferJob": [
        {
            "creationTime": "2015-01-01T01:01:00.000000000Z",
            "description": "YOUR DESCRIPTION",
            "name": "transferJobs/00000000000000000000",
            "status": "ENABLED",
            "lastModificationTime": "2015-01-01T01:01:00.000000000Z",
            "projectId": "PROJECT_ID",
            "schedule": {
                "scheduleStartDate": {
                    "day": 1,
                    "month": 1,
                    "year": 2015
                },
                "scheduleEndDate": {
                    "day": 1,
                    "month": 1,
                    "year": 2015
                },
                "startTimeOfDay": {
                    "hours": 1,
                    "minutes": 1
                }
            },
            "transferSpec": {
                "awsS3DataSource": {
                    "bucketName": "AWS_SOURCE_NAME"
                },
                "gcsDataSink": {
                    "bucketName": "GCS_SINK_NAME"
                },
                "objectConditions": {},
                "transferOptions": {}
            }
        }
    ]
}

Java

For how to create a Storage Transfer Service client, refer to Creating a Client for a Google APIs Library.

package com.google.cloud.storage.storagetransfer.samples;

import com.google.api.services.storagetransfer.v1.Storagetransfer;
import com.google.api.services.storagetransfer.v1.model.AwsAccessKey;
import com.google.api.services.storagetransfer.v1.model.AwsS3Data;
import com.google.api.services.storagetransfer.v1.model.Date;
import com.google.api.services.storagetransfer.v1.model.GcsData;
import com.google.api.services.storagetransfer.v1.model.Schedule;
import com.google.api.services.storagetransfer.v1.model.TimeOfDay;
import com.google.api.services.storagetransfer.v1.model.TransferJob;
import com.google.api.services.storagetransfer.v1.model.TransferSpec;

import java.io.IOException;
import java.io.PrintStream;

/**
 * Creates a one-off transfer job from Amazon S3 to Google Cloud Storage.
 */
public final class AwsRequester {
  /**
   * Creates and executes a request for a TransferJob from Amazon S3 to Cloud Storage.
   *
   * <p>The {@code startDate} and {@code startTime} parameters should be set according to the UTC
   * Time Zone. See:
   * https://developers.google.com/resources/api-libraries/documentation/storagetransfer/v1/java/latest/com/google/api/services/storagetransfer/v1/model/Schedule.html#getStartTimeOfDay()
   *
   * @return the response TransferJob if the request is successful
   * @throws InstantiationException
   *           if instantiation fails when building the TransferJob
   * @throws IllegalAccessException
   *           if an illegal access occurs when building the TransferJob
   * @throws IOException
   *           if the client failed to complete the request
   */
  public static TransferJob createAwsTransferJob(
      String projectId,
      String jobDescription,
      String awsSourceBucket,
      String gcsSinkBucket,
      String startDate,
      String startTime,
      String awsAccessKeyId,
      String awsSecretAccessKey)
      throws InstantiationException, IllegalAccessException, IOException {
    Date date = TransferJobUtils.createDate(startDate);
    TimeOfDay time = TransferJobUtils.createTimeOfDay(startTime);
    TransferJob transferJob =
        new TransferJob()
            .setDescription(jobDescription)
            .setProjectId(projectId)
            .setTransferSpec(
                new TransferSpec()
                    .setAwsS3DataSource(
                        new AwsS3Data()
                            .setBucketName(awsSourceBucket)
                            .setAwsAccessKey(
                                new AwsAccessKey()
                                    .setAccessKeyId(awsAccessKeyId)
                                    .setSecretAccessKey(awsSecretAccessKey)))
                    .setGcsDataSink(new GcsData().setBucketName(gcsSinkBucket)))
            .setSchedule(
                new Schedule()
                    .setScheduleStartDate(date)
                    .setScheduleEndDate(date)
                    .setStartTimeOfDay(time))
            .setStatus("ENABLED");

    Storagetransfer client = TransferClientCreator.createStorageTransferClient();
    return client.transferJobs().create(transferJob).execute();
  }

  public static void run(PrintStream out)
      throws InstantiationException, IllegalAccessException, IOException {
    String projectId = TransferJobUtils.getPropertyOrFail("projectId");
    String jobDescription = TransferJobUtils.getPropertyOrFail("jobDescription");
    String awsSourceBucket = TransferJobUtils.getPropertyOrFail("awsSourceBucket");
    String gcsSinkBucket = TransferJobUtils.getPropertyOrFail("gcsSinkBucket");
    String startDate = TransferJobUtils.getPropertyOrFail("startDate");
    String startTime = TransferJobUtils.getPropertyOrFail("startTime");
    String awsAccessKeyId = TransferJobUtils.getEnvOrFail("AWS_ACCESS_KEY_ID");
    String awsSecretAccessKey = TransferJobUtils.getEnvOrFail("AWS_SECRET_ACCESS_KEY");

    TransferJob responseT =
        createAwsTransferJob(
            projectId,
            jobDescription,
            awsSourceBucket,
            gcsSinkBucket,
            startDate,
            startTime,
            awsAccessKeyId,
            awsSecretAccessKey);
    out.println("Return transferJob: " + responseT.toPrettyString());
  }

  /**
   * Output the contents of a successfully created TransferJob.
   */
  public static void main(String[] args) {
    try {
      run(System.out);
    } catch (Exception e) {
      e.printStackTrace();
    }
  }
}

Python

For how to create a Storage Transfer Service client, refer to Creating a Client for a Google APIs Library.

"""Command-line sample that creates a one-time transfer from Amazon S3 to
Google Cloud Storage.

This sample is used on this page:

    https://cloud.google.com/storage/transfer/create-transfer

For more information, see README.md.
"""

import argparse
import datetime
import json

import googleapiclient.discovery


def main(description, project_id, year, month, day, hours, minutes,
         source_bucket, access_key, secret_access_key, sink_bucket):
    """Create a one-off transfer from Amazon S3 to Google Cloud Storage."""
    storagetransfer = googleapiclient.discovery.build('storagetransfer', 'v1')

    # Edit this template with desired parameters.
    # Specify times below using US Pacific Time Zone.
    transfer_job = {
        'description': description,
        'status': 'ENABLED',
        'projectId': project_id,
        'schedule': {
            'scheduleStartDate': {
                'day': day,
                'month': month,
                'year': year
            },
            'scheduleEndDate': {
                'day': day,
                'month': month,
                'year': year
            },
            'startTimeOfDay': {
                'hours': hours,
                'minutes': minutes
            }
        },
        'transferSpec': {
            'awsS3DataSource': {
                'bucketName': source_bucket,
                'awsAccessKey': {
                    'accessKeyId': access_key,
                    'secretAccessKey': secret_access_key
                }
            },
            'gcsDataSink': {
                'bucketName': sink_bucket
            }
        }
    }

    result = storagetransfer.transferJobs().create(body=transfer_job).execute()
    print('Returned transferJob: {}'.format(
        json.dumps(result, indent=4)))


if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('description', help='Transfer description.')
    parser.add_argument('project_id', help='Your Google Cloud project ID.')
    parser.add_argument('date', help='Date YYYY/MM/DD.')
    parser.add_argument('time', help='Time (24hr) HH:MM.')
    parser.add_argument('source_bucket', help='Source bucket name.')
    parser.add_argument('access_key', help='Your AWS access key id.')
    parser.add_argument('secret_access_key', help='Your AWS secret access '
                        'key.')
    parser.add_argument('sink_bucket', help='Sink bucket name.')

    args = parser.parse_args()
    date = datetime.datetime.strptime(args.date, '%Y/%m/%d')
    time = datetime.datetime.strptime(args.time, '%H:%M')

    main(
        args.description,
        args.project_id,
        date.year,
        date.month,
        date.day,
        time.hour,
        time.minute,
        args.source_bucket,
        args.access_key,
        args.secret_access_key,
        args.sink_bucket)

Transfer between Cloud Storage buckets

In this example, you'll learn how to move files from one Cloud Storage bucket to another. For example, you can move data from a Multi-Regional Storage bucket to a Nearline Storage bucket to take advantage of cheaper storage costs for Cloud Storage Nearline for data you want to keep but are not actively using.

To create the transfer job

REST

Request using transferJobs create:
POST https://storagetransfer.googleapis.com/v1/transferJobs
{
    "description": "YOUR DESCRIPTION",
    "status": "ENABLED",
    "projectId": "PROJECT_ID",
    "schedule": {
        "scheduleStartDate": {
            "day": 1,
            "month": 1,
            "year": 2015
        },
        "startTimeOfDay": {
            "hours": 1,
            "minutes": 1
        }
    },
    "transferSpec": {
        "gcsDataSource": {
            "bucketName": "GCS_SOURCE_NAME",
        },
        "gcsDataSink": {
            "bucketName": "GCS_NEARLINE_SINK_NAME"
        },
        "objectConditions": {
            "minTimeElapsedSinceLastModification": "2592000s"
        },
        "transferOptions": {
            "deleteObjectsFromSourceAfterTransfer": true
        }
    }
}
Response:
200 OK
{
    "transferJob": [
        {
            "creationTime": "2015-01-01T01:01:00.000000000Z",
            "description": "YOUR DESCRIPTION",
            "name": "transferJobs/00000000000000000000",
            "status": "ENABLED",
            "lastModificationTime": "2015-01-01T01:01:00.000000000Z",
            "projectId": "PROJECT_ID",
            "schedule": {
                "scheduleStartDate": {
                    "day": 1,
                    "month": 1,
                    "year": 2015
                },
                "startTimeOfDay": {
                    "hours": 1,
                    "minutes": 1
                }
            },
            "transferSpec": {
                "gcsDataSource": {
                    "bucketName": "GCS_SOURCE_NAME",
                },
                "gcsDataSink": {
                    "bucketName": "GCS_NEARLINE_SINK_NAME"
                },
                "objectConditions": {
                    "minTimeElapsedSinceLastModification": "2592000.000s"
                },
                "transferOptions": {
                    "deleteObjectsFromSourceAfterTransfer": true
                }
            }
        }
    ]
}

Java

For how to create a Storage Transfer Service client, refer to Creating a Client for a Google APIs Library.

package com.google.cloud.storage.storagetransfer.samples;

import com.google.api.services.storagetransfer.v1.Storagetransfer;
import com.google.api.services.storagetransfer.v1.model.Date;
import com.google.api.services.storagetransfer.v1.model.GcsData;
import com.google.api.services.storagetransfer.v1.model.ObjectConditions;
import com.google.api.services.storagetransfer.v1.model.Schedule;
import com.google.api.services.storagetransfer.v1.model.TimeOfDay;
import com.google.api.services.storagetransfer.v1.model.TransferJob;
import com.google.api.services.storagetransfer.v1.model.TransferOptions;
import com.google.api.services.storagetransfer.v1.model.TransferSpec;

import java.io.IOException;
import java.io.PrintStream;

/**
 * Creates a daily transfer from a standard Cloud Storage bucket to a Cloud Storage Nearline
 * bucket for files untouched for 30 days.
 */
public final class NearlineRequester {

  /**
   * Creates and executes a request for a TransferJob to Cloud Storage Nearline.
   *
   * <p>The {@code startDate} and {@code startTime} parameters should be set according to the UTC
   * Time Zone. See:
   * https://developers.google.com/resources/api-libraries/documentation/storagetransfer/v1/java/latest/com/google/api/services/storagetransfer/v1/model/Schedule.html#getStartTimeOfDay()
   *
   * @return the response TransferJob if the request is successful
   * @throws InstantiationException
   *           if instantiation fails when building the TransferJob
   * @throws IllegalAccessException
   *           if an illegal access occurs when building the TransferJob
   * @throws IOException
   *           if the client failed to complete the request
   */
  public static TransferJob createNearlineTransferJob(
      String projectId,
      String jobDescription,
      String gcsSourceBucket,
      String gcsNearlineSinkBucket,
      String startDate,
      String startTime)
      throws InstantiationException, IllegalAccessException, IOException {
    Date date = TransferJobUtils.createDate(startDate);
    TimeOfDay time = TransferJobUtils.createTimeOfDay(startTime);
    TransferJob transferJob =
        new TransferJob()
            .setDescription(jobDescription)
            .setProjectId(projectId)
            .setTransferSpec(
                new TransferSpec()
                    .setGcsDataSource(new GcsData().setBucketName(gcsSourceBucket))
                    .setGcsDataSink(new GcsData().setBucketName(gcsNearlineSinkBucket))
                    .setObjectConditions(
                        new ObjectConditions()
                            .setMinTimeElapsedSinceLastModification("2592000s" /* 30 days */))
                    .setTransferOptions(
                        new TransferOptions()
                            .setDeleteObjectsFromSourceAfterTransfer(true)))
            .setSchedule(
                new Schedule().setScheduleStartDate(date).setStartTimeOfDay(time))
            .setStatus("ENABLED");

    Storagetransfer client = TransferClientCreator.createStorageTransferClient();
    return client.transferJobs().create(transferJob).execute();
  }

  public static void run(PrintStream out)
      throws InstantiationException, IllegalAccessException, IOException {
    String projectId = TransferJobUtils.getPropertyOrFail("projectId");
    String jobDescription = TransferJobUtils.getPropertyOrFail("jobDescription");
    String gcsSourceBucket = TransferJobUtils.getPropertyOrFail("gcsSourceBucket");
    String gcsNearlineSinkBucket = TransferJobUtils.getPropertyOrFail("gcsNearlineSinkBucket");
    String startDate = TransferJobUtils.getPropertyOrFail("startDate");
    String startTime = TransferJobUtils.getPropertyOrFail("startTime");

    TransferJob responseT =
        createNearlineTransferJob(
            projectId,
            jobDescription,
            gcsSourceBucket,
            gcsNearlineSinkBucket,
            startDate,
            startTime);
    out.println("Return transferJob: " + responseT.toPrettyString());
  }

  /**
   * Output the contents of a successfully created TransferJob.
   *
   * @param args
   *          arguments from the command line
   */
  public static void main(String[] args) {
    try {
      run(System.out);
    } catch (Exception e) {
      e.printStackTrace();
    }
  }
}

Python

For how to create a Storage Transfer Service client, refer to Creating a Client for a Google APIs Library.

"""Command-line sample that creates a one-time transfer from Google Cloud
Storage standard class to the Nearline storage class."

This sample is used on this page:

    https://cloud.google.com/storage/transfer/create-transfer

For more information, see README.md.
"""

import argparse
import datetime
import json

import googleapiclient.discovery


def main(description, project_id, year, month, day, hours, minutes,
         source_bucket, sink_bucket):
    """Create a transfer from the Google Cloud Storage Standard class to the
    Nearline Storage class."""
    storagetransfer = googleapiclient.discovery.build('storagetransfer', 'v1')

    # Edit this template with desired parameters.
    # Specify times below using US Pacific Time Zone.
    transfer_job = {
        'description': description,
        'status': 'ENABLED',
        'projectId': project_id,
        'schedule': {
            'scheduleStartDate': {
                'day': day,
                'month': month,
                'year': year
            },
            'startTimeOfDay': {
                'hours': hours,
                'minutes': minutes
            }
        },
        'transferSpec': {
            'gcsDataSource': {
                'bucketName': source_bucket
            },
            'gcsDataSink': {
                'bucketName': sink_bucket
            },
            'objectConditions': {
                'minTimeElapsedSinceLastModification': '2592000s'
            },
            'transferOptions': {
                'deleteObjectsFromSourceAfterTransfer': 'true'
            }
        }
    }

    result = storagetransfer.transferJobs().create(body=transfer_job).execute()
    print('Returned transferJob: {}'.format(
        json.dumps(result, indent=4)))


if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('description', help='Transfer description.')
    parser.add_argument('project_id', help='Your Google Cloud project ID.')
    parser.add_argument('date', help='Date YYYY/MM/DD.')
    parser.add_argument('time', help='Time (24hr) HH:MM.')
    parser.add_argument('source_bucket', help='Source bucket name.')
    parser.add_argument('sink_bucket', help='Sink bucket name.')

    args = parser.parse_args()
    date = datetime.datetime.strptime(args.date, '%Y/%m/%d')
    time = datetime.datetime.strptime(args.time, '%H:%M')

    main(
        args.description,
        args.project_id,
        date.year,
        date.month,
        date.day,
        time.hour,
        time.minute,
        args.source_bucket,
        args.sink_bucket)

Check transfer job status

For either of the examples above, you might want to check the status of your transfer job. The following example code returns the status of a transfer job based on a job ID and your project ID. You can get the job ID from the response of the job request.

REST

Request using transferJobs get:
GET https://storagetransfer.googleapis.com/v1/transferOperations?filter=%7B"project_id":"PROJECT_ID","job_id":%5B"transferJobs/00000000000000000000"%5D%7D
Response:

Cloud Storage

If your source is a Google Cloud Storage bucket, the response looks like:

200 OK
{
    "operations": [
        {
            "done": true,
            "metadata": {
                "@type": "type.googleapis.com/google.storagetransfer.v1.TransferOperation",
                "counters": {},
                "endTime": "2015-01-01T01:01:00.000Z",
                "name": "transferOperations/000000000000000000",
                "projectId": "PROJECT_ID",
                "startTime": "2015-01-01T01:01:00.000",
                "transferSpec": {
                    "gcsDataSink": {
                        "bucketName": "GCS_NEARLINE_SINK_NAME"
                    },
                    "gcsDataSource": {
                        "bucketName": "GCS_SOURCE_NAME"
                    },
                    "objectConditions": {
                        "minTimeElapsedSinceLastModification": "2592000.000s"
                    },
                    "transferOptions": {
                        "deleteObjectsFromSourceAfterTransfer": true
                    }
                },
                "transferStatus": "SUCCESS"
            },
            "name": "transferOperations/000000000000000000",
            "response": {
                "@type": "type.googleapis.com/google.protobuf.Empty"
            }
        }
    ]
}

Amazon S3

If your source is an Amazon S3 bucket, the response looks like:

200 OK
{
    "operations": [
        {
            "done": true,
            "metadata": {
                "@type": "type.googleapis.com/google.storagetransfer.v1.TransferOperation",
                "counters": {},
                "endTime": "2015-01-01T01:01:00.000Z",
                "name": "transferOperations/000000000000000000",
                "projectId": "PROJECT_ID",
                "startTime": "2015-01-01T01:01:00.000",
                "transferSpec": {
                    "awsS3DataSource": {
                        "bucketName": "AWS_SOURCE_NAME"
                    },
                    "gcsDataSink": {
                        "bucketName": "GCS_SINK_NAME"
                    },
                    "objectConditions": {},
                    "transferOptions": {}
                },
                "transferStatus": "SUCCESS"
            },
            "name": "transferOperations/000000000000000000",
            "response": {
                "@type": "type.googleapis.com/google.protobuf.Empty"
            }
        }
    ]
}

Java

For how to create a Storage Transfer Service client, refer to Setting Up Your Application.

package com.google.cloud.storage.storagetransfer.samples;

import com.google.api.services.storagetransfer.v1.Storagetransfer;
import com.google.api.services.storagetransfer.v1.model.ListOperationsResponse;

import java.io.IOException;
import java.util.logging.Logger;

/**
 * Queries for TransferOperations associated with a specific TransferJob. A TransferJob is done when
 * all of its associated TransferOperations have completed.
 *
 */
public final class RequestChecker {

  private static final String PROJECT_ID = "YOUR_PROJECT_ID";
  private static final String JOB_NAME = "YOUR_JOB_NAME";

  private static final Logger LOG = Logger.getLogger(RequestChecker.class.getName());

  /**
   * Creates and executes a query for all associated TransferOperations.
   *
   * @param client
   *          a Storagetransfer client, for interacting with the Storage Transfer API
   * @param projectId
   *          the project to query within
   * @param jobName
   *          the job Name of the relevant TransferJob
   * @return an object containing information on associated TransferOperations
   * @throws IOException
   *           if the client failed to complete the request
   */
  public static ListOperationsResponse checkTransfer(
      Storagetransfer client, String projectId, String jobName) throws IOException {
    return client
        .transferOperations()
        .list("transferOperations")
        .setFilter("{\"project_id\": \"" + projectId + "\", \"job_names\": [\"" + jobName + "\"] }")
        .execute();
  }

  /**
   * Output the returned list of TransferOperations.
   *
   * @param args
   *          arguments from the command line
   */
  public static void main(String[] args) {
    try {
      ListOperationsResponse resp =
          checkTransfer(TransferClientCreator.createStorageTransferClient(), PROJECT_ID, JOB_NAME);
      LOG.info("Result of transferOperations/list: " + resp.toPrettyString());
    } catch (Exception e) {
      e.printStackTrace();
    }
  }
}

Python

For how to create a Storage Transfer Service client, refer to Setting Up Your Application.

"""Command-line sample that checks the status of an in-process transfer.

This sample is used on this page:

    https://cloud.google.com/storage/transfer/create-transfer

For more information, see README.md.
"""

import argparse
import json

import googleapiclient.discovery


def main(project_id, job_name):
    """Review the transfer operations associated with a transfer job."""
    storagetransfer = googleapiclient.discovery.build('storagetransfer', 'v1')

    filterString = (
        '{{"project_id": "{project_id}", '
        '"job_names": ["{job_name}"]}}'
    ).format(project_id=project_id, job_name=job_name)

    result = storagetransfer.transferOperations().list(
        name="transferOperations",
        filter=filterString).execute()
    print('Result of transferOperations/list: {}'.format(
        json.dumps(result, indent=4, sort_keys=True)))


if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('project_id', help='Your Google Cloud project ID.')
    parser.add_argument('job_name', help='Your job name.')

    args = parser.parse_args()

    main(args.project_id, args.job_name)

Cancel transfers

To cancel a single transfer operation, use the transferOperations cancel method. To delete an entire transfer job, including future transfer operations that are scheduled for it, set the transfer job's status to DELETED using the transferJobs patch method.

Monitor your resources on the go

Get the Google Cloud Console app to help you manage your projects.

Send feedback about...

Cloud Storage Documentation