Read Avro format

Use the BigQueryIO connector to read in Avro format.

Explore further

For detailed documentation that includes this code sample, see the following:

Code sample


To authenticate to Dataflow, set up Application Default Credentials. For more information, see Set up authentication for a local development environment.

import org.apache.avro.generic.GenericRecord;
import org.apache.avro.util.Utf8;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.coders.DefaultCoder;
import org.apache.beam.sdk.extensions.avro.coders.AvroCoder;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.transforms.MapElements;
import org.apache.beam.sdk.transforms.SerializableFunction;
import org.apache.beam.sdk.values.TypeDescriptor;

public class BigQueryReadAvro {

  // A custom datatype to hold a record from the source table.
  public static class MyData {
    public String name;
    public Long age;

    // Function to convert Avro records to MyData instances.
    public static class FromSchemaAndRecord
            implements SerializableFunction<SchemaAndRecord, MyData> {
      @Override public MyData apply(SchemaAndRecord elem) {
        MyData data = new MyData();
        GenericRecord record = elem.getRecord(); = ((Utf8) record.get("user_name")).toString();
        data.age = (Long) record.get("age");
        return data;

  public static void main(String[] args) {
    // Parse the pipeline options passed into the application. Example:
    //   --projectId=$PROJECT_ID --datasetName=$DATASET_NAME --tableName=$TABLE_NAME
    // For more information, see
    ExamplePipelineOptions options = PipelineOptionsFactory.fromArgs(args)

    // Create a pipeline and apply transforms.
    Pipeline pipeline = Pipeline.create(options);
        // Read table data into Avro records, using an application-defined parsing function.
        .apply( MyData.FromSchemaAndRecord())
        // The output from the previous step is a PCollection<MyData>.
            .via((MyData x) -> {
              System.out.printf("Name: %s, Age: %d%n",, x.age);
              return x;

What's next

To search and filter code samples for other Google Cloud products, see the Google Cloud sample browser.