Support frequent and distributed counters
Many realtime apps have documents that act as counters. For example, you might count 'likes' on a post, or 'favorites' of a specific item.
In Firestore, you can't update a single document at an unlimited rate. If you have a counter based on single document and frequent enough increments to it you will eventually see contention on the updates to the document. See Updates to a single document.
Solution: Distributed counters
To support more frequent counter updates, create a distributed counter. Each counter is a document with a subcollection of "shards," and the value of the counter is the sum of the value of the shards.
Write throughput increases linearly with the number of shards, so a distributed counter with 10 shards can handle 10x as many writes as a traditional counter.
// counters/${ID}
{
"num_shards": NUM_SHARDS,
"shards": [subcollection]
}
// counters/${ID}/shards/${NUM}
{
"count": 123
}
// counters/${ID} struct Counter { let numShards: Int init(numShards: Int) { self.numShards = numShards } } // counters/${ID}/shards/${NUM} struct Shard { let count: Int init(count: Int) { self.count = count } }
// counters/${ID} @interface FIRCounter : NSObject @property (nonatomic, readonly) NSInteger shardCount; @end @implementation FIRCounter - (instancetype)initWithShardCount:(NSInteger)shardCount { self = [super init]; if (self != nil) { _shardCount = shardCount; } return self; } @end // counters/${ID}/shards/${NUM} @interface FIRShard : NSObject @property (nonatomic, readonly) NSInteger count; @end @implementation FIRShard - (instancetype)initWithCount:(NSInteger)count { self = [super init]; if (self != nil) { _count = count; } return self; } @end
// counters/${ID} data class Counter(var numShards: Int) // counters/${ID}/shards/${NUM} data class Shard(var count: Int)
// counters/${ID} public class Counter { int numShards; public Counter(int numShards) { this.numShards = numShards; } } // counters/${ID}/shards/${NUM} public class Shard { int count; public Shard(int count) { this.count = count; } }
Not applicable, see the counter increment snippet below.
Not applicable, see the counter initialization snippet below.
The following code initializes a distributed counter:
function createCounter(ref, num_shards) { var batch = db.batch(); // Initialize the counter document batch.set(ref, { num_shards: num_shards }); // Initialize each shard with count=0 for (let i = 0; i < num_shards; i++) { const shardRef = ref.collection('shards').doc(i.toString()); batch.set(shardRef, { count: 0 }); } // Commit the write batch return batch.commit(); }
func createCounter(ref: DocumentReference, numShards: Int) async { do { try await ref.setData(["numShards": numShards]) for i in 0...numShards { try await ref.collection("shards").document(String(i)).setData(["count": 0]) } } catch { // ... } }
- (void)createCounterAtReference:(FIRDocumentReference *)reference shardCount:(NSInteger)shardCount { [reference setData:@{ @"numShards": @(shardCount) } completion:^(NSError * _Nullable error) { for (NSInteger i = 0; i < shardCount; i++) { NSString *shardName = [NSString stringWithFormat:@"%ld", (long)shardCount]; [[[reference collectionWithPath:@"shards"] documentWithPath:shardName] setData:@{ @"count": @(0) }]; } }]; }
fun createCounter(ref: DocumentReference, numShards: Int): Task<Void> { // Initialize the counter document, then initialize each shard. return ref.set(Counter(numShards)) .continueWithTask { task -> if (!task.isSuccessful) { throw task.exception!! } val tasks = arrayListOf<Task<Void>>() // Initialize each shard with count=0 for (i in 0 until numShards) { val makeShard = ref.collection("shards") .document(i.toString()) .set(Shard(0)) tasks.add(makeShard) } Tasks.whenAll(tasks) } }
public Task<Void> createCounter(final DocumentReference ref, final int numShards) { // Initialize the counter document, then initialize each shard. return ref.set(new Counter(numShards)) .continueWithTask(new Continuation<Void, Task<Void>>() { @Override public Task<Void> then(@NonNull Task<Void> task) throws Exception { if (!task.isSuccessful()) { throw task.getException(); } List<Task<Void>> tasks = new ArrayList<>(); // Initialize each shard with count=0 for (int i = 0; i < numShards; i++) { Task<Void> makeShard = ref.collection("shards") .document(String.valueOf(i)) .set(new Shard(0)); tasks.add(makeShard); } return Tasks.whenAll(tasks); } }); }
Not applicable, see the counter increment snippet below.
To increment the counter, choose a random shard and increment the count:
function incrementCounter(ref, num_shards) { // Select a shard of the counter at random const shard_id = Math.floor(Math.random() * num_shards).toString(); const shard_ref = ref.collection('shards').doc(shard_id); // Update count return shard_ref.update("count", firebase.firestore.FieldValue.increment(1)); }
func incrementCounter(ref: DocumentReference, numShards: Int) { // Select a shard of the counter at random let shardId = Int(arc4random_uniform(UInt32(numShards))) let shardRef = ref.collection("shards").document(String(shardId)) shardRef.updateData([ "count": FieldValue.increment(Int64(1)) ]) }
- (void)incrementCounterAtReference:(FIRDocumentReference *)reference shardCount:(NSInteger)shardCount { // Select a shard of the counter at random NSInteger shardID = (NSInteger)arc4random_uniform((uint32_t)shardCount); NSString *shardName = [NSString stringWithFormat:@"%ld", (long)shardID]; FIRDocumentReference *shardReference = [[reference collectionWithPath:@"shards"] documentWithPath:shardName]; [shardReference updateData:@{ @"count": [FIRFieldValue fieldValueForIntegerIncrement:1] }]; }
fun incrementCounter(ref: DocumentReference, numShards: Int): Task<Void> { val shardId = Math.floor(Math.random() * numShards).toInt() val shardRef = ref.collection("shards").document(shardId.toString()) return shardRef.update("count", FieldValue.increment(1)) }
public Task<Void> incrementCounter(final DocumentReference ref, final int numShards) { int shardId = (int) Math.floor(Math.random() * numShards); DocumentReference shardRef = ref.collection("shards").document(String.valueOf(shardId)); return shardRef.update("count", FieldValue.increment(1)); }
To get the total count, query for all shards and sum their count
fields:
function getCount(ref) { // Sum the count of each shard in the subcollection return ref.collection('shards').get().then((snapshot) => { let total_count = 0; snapshot.forEach((doc) => { total_count += doc.data().count; }); return total_count; }); }
func getCount(ref: DocumentReference) async { do { let querySnapshot = try await ref.collection("shards").getDocuments() var totalCount = 0 for document in querySnapshot.documents { let count = document.data()["count"] as! Int totalCount += count } print("Total count is \(totalCount)") } catch { // handle error } }
- (void)getCountWithReference:(FIRDocumentReference *)reference { [[reference collectionWithPath:@"shards"] getDocumentsWithCompletion:^(FIRQuerySnapshot *snapshot, NSError *error) { NSInteger totalCount = 0; if (error != nil) { // Error getting shards // ... } else { for (FIRDocumentSnapshot *document in snapshot.documents) { NSInteger count = [document[@"count"] integerValue]; totalCount += count; } NSLog(@"Total count is %ld", (long)totalCount); } }]; }
fun getCount(ref: DocumentReference): Task<Int> { // Sum the count of each shard in the subcollection return ref.collection("shards").get() .continueWith { task -> var count = 0 for (snap in task.result!!) { val shard = snap.toObject<Shard>() count += shard.count } count } }
public Task<Integer> getCount(final DocumentReference ref) { // Sum the count of each shard in the subcollection return ref.collection("shards").get() .continueWith(new Continuation<QuerySnapshot, Integer>() { @Override public Integer then(@NonNull Task<QuerySnapshot> task) throws Exception { int count = 0; for (DocumentSnapshot snap : task.getResult()) { Shard shard = snap.toObject(Shard.class); count += shard.count; } return count; } }); }
Limitations
The solution shown above is a scalable way to create shared counters in Firestore, but you should be aware of the following limitations:
- Shard count - The number of shards controls the performance of the distributed counter. With too few shards, some transactions may have to retry before succeeding, which will slow writes. With too many shards, reads become slower and more expensive. You can offset the read-expense by keeping the counter total in a separate roll-up document which is updated at a slower cadence, and having clients read from that document to get the total. The tradeoff is that clients will have to wait for the roll-up document to be updated, instead of computing the total by reading all of the shards immediately after any update.
- Cost - The cost of reading a counter value increases linearly with the number of shards, because the entire shards subcollection must be loaded.