datastore-contention.py

from multiprocessing import Pool
from gcloud.exceptions import Conflict
from gcloud import datastore

import random
import traceback
import os
import uuid
import time
import sys


if __name__ == "__main__":
  project = sys.argv[1]

  id = str(uuid.uuid4())

  c = datastore.Client(project)

  def f(n):
    attempts = 0
    start = time.time()

    time.sleep(n * 0.1)

    while True:
      try:
        with c.transaction():
          entity = c.get(c.key('Test', id))

          if entity is None:
            entity = datastore.Entity(key=c.key('Test', id))

          try:
            visitors = entity['visitors']
          except KeyError:
            entity['visitors'] = visitors = []

          visitors.append(int(n))
          c.put(entity)
      except Conflict as e:
        attempts += 1
        time.sleep(random.random() * 0.1)
        continue

      break

    return (n, time.time() - start, attempts, visitors)


  pool = Pool(processes=16)
  results = pool.map(f, range(16))

  print(id)

  for (n, diff, attempts, visitors) in sorted(results, key=lambda e: len(e[3])):
    print("{:02}: (took {:.2f}, attempts: {}): {}".format(n, diff, attempts, visitors))

#> python3 datastore-contention.py <project>

e896afe4-f485-49a2-9ce2-fbd76527a8cc
04: (took 0.98, attempts: 0): [4]
06: (took 1.40, attempts: 0): [4, 6]
15: (took 2.13, attempts: 0): [4, 6, 15]
14: (took 2.77, attempts: 1): [4, 6, 15, 14]
08: (took 4.06, attempts: 1): [4, 6, 15, 14, 8]
10: (took 4.69, attempts: 1): [4, 6, 15, 14, 8, 10]
00: (took 5.83, attempts: 3): [4, 6, 15, 14, 8, 10, 0]
01: (took 6.47, attempts: 2): [4, 6, 15, 14, 8, 10, 0, 1]
05: (took 7.39, attempts: 4): [4, 6, 15, 14, 8, 10, 0, 1, 5]
02: (took 7.77, attempts: 4): [4, 6, 15, 14, 8, 10, 0, 1, 5, 2]
09: (took 8.65, attempts: 5): [4, 6, 15, 14, 8, 10, 0, 1, 5, 2, 9]
12: (took 9.39, attempts: 5): [4, 6, 15, 14, 8, 10, 0, 1, 5, 2, 9, 12]
11: (took 9.98, attempts: 6): [4, 6, 15, 14, 8, 10, 0, 1, 5, 2, 9, 12, 11]
07: (took 10.79, attempts: 6): [4, 6, 15, 14, 8, 10, 0, 1, 5, 2, 9, 12, 11, 7]
03: (took 11.43, attempts: 7): [4, 6, 15, 14, 8, 10, 0, 1, 5, 2, 9, 12, 11, 7, 3]
13: (took 13.28, attempts: 8): [4, 6, 15, 14, 8, 10, 0, 1, 5, 2, 9, 12, 11, 7, 3, 13]