Class PslSparkUtils (0.3.4)

public class PslSparkUtils

Inheritance

java.lang.Object > PslSparkUtils

Inherited Members

Object.clone()

Object.equals(Object)

Object.finalize()

Object.getClass()

Object.hashCode()

Object.notify()

Object.notifyAll()

Object.toString()

Object.wait()

Object.wait(long)

Object.wait(long,int)

Constructors

PslSparkUtils()

public PslSparkUtils()

Methods

convertAttributesToSparkMap(ListMultimap<String,ByteString> attributeMap)

public static ArrayBasedMapData convertAttributesToSparkMap(ListMultimap<String,ByteString> attributeMap)

Parameter

Name	Description
attributeMap	`com.google.common.collect.ListMultimap<String,ByteString>`

Returns

Type	Description
org.apache.spark.sql.catalyst.util.ArrayBasedMapData

getSparkEndOffset(SparkSourceOffset headOffset, SparkSourceOffset startOffset, long maxMessagesPerBatch, long topicPartitionCount)

public static SparkSourceOffset getSparkEndOffset(SparkSourceOffset headOffset, SparkSourceOffset startOffset, long maxMessagesPerBatch, long topicPartitionCount)

Parameters

Name	Description
headOffset	`SparkSourceOffset`
startOffset	`SparkSourceOffset`
maxMessagesPerBatch	`long`
topicPartitionCount	`long`

Returns

Type	Description
SparkSourceOffset

getSparkStartOffset(CursorClient cursorClient, SubscriptionPath subscriptionPath, long topicPartitionCount)

public static SparkSourceOffset getSparkStartOffset(CursorClient cursorClient, SubscriptionPath subscriptionPath, long topicPartitionCount)

Parameters

Name	Description
cursorClient	`com.google.cloud.pubsublite.internal.CursorClient`
subscriptionPath	`com.google.cloud.pubsublite.SubscriptionPath`
topicPartitionCount	`long`

Returns

Type	Description
SparkSourceOffset

toInternalRow(SequencedMessage msg, SubscriptionPath subscription, Partition partition)

public static InternalRow toInternalRow(SequencedMessage msg, SubscriptionPath subscription, Partition partition)

Parameters

Name	Description
msg	`com.google.cloud.pubsublite.SequencedMessage`
subscription	`com.google.cloud.pubsublite.SubscriptionPath`
partition	`com.google.cloud.pubsublite.Partition`

Returns

Type	Description
org.apache.spark.sql.catalyst.InternalRow

toPslPartitionOffset(SparkPartitionOffset sparkPartitionOffset)

public static PslPartitionOffset toPslPartitionOffset(SparkPartitionOffset sparkPartitionOffset)

Parameter

Name	Description
sparkPartitionOffset	`com.google.cloud.pubsublite.spark.SparkPartitionOffset`

Returns

Type	Description
PslPartitionOffset

toPslSourceOffset(SparkSourceOffset sparkSourceOffset)

public static PslSourceOffset toPslSourceOffset(SparkSourceOffset sparkSourceOffset)

Parameter

Name	Description
sparkSourceOffset	`SparkSourceOffset`

Returns

Type	Description
PslSourceOffset

toPubSubMessage(StructType inputSchema, InternalRow row)

public static Message toPubSubMessage(StructType inputSchema, InternalRow row)

Parameters

Name	Description
inputSchema	`org.apache.spark.sql.types.StructType`
row	`org.apache.spark.sql.catalyst.InternalRow`

Returns

Type	Description
com.google.cloud.pubsublite.Message

toSparkSourceOffset(PslSourceOffset pslSourceOffset)

public static SparkSourceOffset toSparkSourceOffset(PslSourceOffset pslSourceOffset)

Parameter

Name	Description
pslSourceOffset	`PslSourceOffset`

Returns

Type	Description
SparkSourceOffset

verifyWriteInputSchema(StructType inputSchema)

public static void verifyWriteInputSchema(StructType inputSchema)

Make sure data fields for publish have expected Spark DataType if they exist.

Parameter

Name	Description
inputSchema	`org.apache.spark.sql.types.StructType` input table schema to write to Pub/Sub Lite.