API'de Bulk Operasyonlar: Batch Boyutu Optimizasyonu

1# Yanlış: Birer birer gönder
2for product in products:
3api.update_product(product) # 10 milyon istek!
4
5# Doğru: Batch'le gönder
6batch_size = 1000
7for i in range(0, len(products), batch_size):
8batch = products[i:i + batch_size]
9api.bulk_update(batch)

1import time
2import requests
3
4batch_sizes = [100, 500, 1000, 5000]
5results = {}
6
7for batch_size in batch_sizes:
8start = time.time()
9for i in range(0, len(products), batch_size):
10batch = products[i:i + batch_size]
11response = requests.post(
12'https://api.example.com/bulk-update',
13json={'items': batch}
14)
15if response.status_code!= 200:
16print(f"Hata (batch {batch_size}): {response.status_code}")
17break
18elapsed = time.time() - start
19results[batch_size] = elapsed
20print(f"Batch {batch_size}: {elapsed:.2f}s")
21
22# Çıktı örneği:
23# Batch 100: 125.34s
24# Batch 500: 28.45s
25# Batch 1000: 15.22s
26# Batch 5000: 18.91s (timeout riskine giriyor)

1def bulk_update_with_retry(items, batch_size=1000, max_retries=3):
2failed_items = []
3current_batch = items
4retry_count = 0
5
6while current_batch and retry_count < max_retries:
7for i in range(0, len(current_batch), batch_size):
8batch = current_batch[i:i + batch_size]
9try:
10response = requests.post(
11'https://api.example.com/bulk-update',
12json={'items': batch},
13timeout=30
14)
15if response.status_code == 200:
16print(f" {len(batch)} kayıt başarılı")
17elif response.status_code == 413: # Payload çok büyük
18# Batch'i yarıya indir
19smaller_batch = batch[:len(batch)//2]
20current_batch.extend(smaller_batch)
21print(f" Batch çok büyük, küçültüldü")
22else:
23failed_items.extend(batch)
24except requests.Timeout:
25# Zaman aşımında retry listesine ekle
26failed_items.extend(batch)
27print(f" Timeout, {len(batch)} kayıt retry listesine eklendi")
28
29current_batch = failed_items
30failed_items = []
31retry_count += 1
32batch_size = max(batch_size // 2, 100) # Batch'i küçült
33
34return len(current_batch) == 0, current_batch
35
36success, remaining = bulk_update_with_retry(products)
37if success:
38print("Tüm ürünler güncellendi")
39else:
40print(f"{len(remaining)} ürün başarısız")

1# Rate limit: 100 bin istek/dakika = 1666 istek/saniye
2# Her batch 1000 kayıt = 1 istek
3# Güvenli batch: 1000 kayıt, her istekten sonra 0.001s bekle
4
5time.sleep(0.001) # 1 milisaniye
6response = requests.post(...)