fix(zopk): Add Brave API rate limit handling and heartbeat limit fix
Some checks are pending
NordaBiz Tests / Unit & Integration Tests (push) Waiting to run
NordaBiz Tests / E2E Tests (Playwright) (push) Blocked by required conditions
NordaBiz Tests / Smoke Tests (Production) (push) Blocked by required conditions
NordaBiz Tests / Send Failure Notification (push) Blocked by required conditions
Some checks are pending
NordaBiz Tests / Unit & Integration Tests (push) Waiting to run
NordaBiz Tests / E2E Tests (Playwright) (push) Blocked by required conditions
NordaBiz Tests / Smoke Tests (Production) (push) Blocked by required conditions
NordaBiz Tests / Send Failure Notification (push) Blocked by required conditions
Brave free tier was returning 429 for ~50% of queries due to back-to-back requests. Added 1.1s delay between queries and retry with exponential backoff (1.5s, 3s). Heartbeat endpoint exempted from Flask-Limiter and interval increased from 30s to 60s to reduce log noise. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
c7a42c3766
commit
c4c113aa6f
@ -13,6 +13,7 @@ from flask import jsonify, request, session, current_app
|
||||
from flask_login import current_user
|
||||
|
||||
from database import SessionLocal, UserSession, UserClick, PageView, JSError
|
||||
from extensions import limiter
|
||||
from . import bp
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@ -88,6 +89,7 @@ def api_analytics_track():
|
||||
|
||||
|
||||
@bp.route('/analytics/heartbeat', methods=['POST'])
|
||||
@limiter.exempt
|
||||
def api_analytics_heartbeat():
|
||||
"""Keep session alive and update duration"""
|
||||
analytics_session_id = session.get('analytics_session_id')
|
||||
|
||||
@ -7,7 +7,7 @@
|
||||
(function() {
|
||||
'use strict';
|
||||
|
||||
const HEARTBEAT_INTERVAL = 30000; // 30 seconds
|
||||
const HEARTBEAT_INTERVAL = 60000; // 60 seconds
|
||||
const SCROLL_DEBOUNCE_MS = 500;
|
||||
const TRACK_ENDPOINT = '/api/analytics/track';
|
||||
const HEARTBEAT_ENDPOINT = '/api/analytics/heartbeat';
|
||||
|
||||
@ -21,6 +21,7 @@ Created: 2026-01-11
|
||||
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import hashlib
|
||||
import logging
|
||||
import unicodedata
|
||||
@ -678,6 +679,10 @@ class ZOPKNewsService:
|
||||
|
||||
if progress_callback:
|
||||
progress_callback('search', f"Brave: {query_config['description']} ({len(brave_items)})", i + 1, len(BRAVE_QUERIES))
|
||||
|
||||
# Rate limit: Brave free tier ~1 req/s
|
||||
if i < len(BRAVE_QUERIES) - 1:
|
||||
time.sleep(1.1)
|
||||
else:
|
||||
process_log.append({
|
||||
'phase': 'search',
|
||||
@ -947,50 +952,63 @@ class ZOPKNewsService:
|
||||
}
|
||||
|
||||
def _search_brave_single(self, query: str) -> List[NewsItem]:
|
||||
"""Search Brave API with a single query"""
|
||||
"""Search Brave API with a single query, with retry on 429"""
|
||||
if not self.brave_api_key:
|
||||
return []
|
||||
|
||||
items = []
|
||||
try:
|
||||
headers = {
|
||||
'Accept': 'application/json',
|
||||
'X-Subscription-Token': self.brave_api_key
|
||||
}
|
||||
params = {
|
||||
'q': query,
|
||||
'count': 10, # Fewer results per query (we have 8 queries)
|
||||
'freshness': 'pw', # past week (more relevant than past month)
|
||||
'country': 'pl',
|
||||
'search_lang': 'pl'
|
||||
}
|
||||
max_retries = 2
|
||||
|
||||
response = requests.get(
|
||||
'https://api.search.brave.com/res/v1/news/search',
|
||||
headers=headers,
|
||||
params=params,
|
||||
timeout=30
|
||||
)
|
||||
for attempt in range(max_retries + 1):
|
||||
try:
|
||||
headers = {
|
||||
'Accept': 'application/json',
|
||||
'X-Subscription-Token': self.brave_api_key
|
||||
}
|
||||
params = {
|
||||
'q': query,
|
||||
'count': 10,
|
||||
'freshness': 'pw',
|
||||
'country': 'pl',
|
||||
'search_lang': 'pl'
|
||||
}
|
||||
|
||||
if response.status_code == 200:
|
||||
results = response.json().get('results', [])
|
||||
for item in results:
|
||||
if item.get('url'):
|
||||
items.append(NewsItem(
|
||||
title=item.get('title', 'Bez tytułu'),
|
||||
url=item['url'],
|
||||
description=item.get('description', ''),
|
||||
source_name=item.get('source', ''),
|
||||
source_type='brave',
|
||||
source_id=f'brave_{query[:20]}',
|
||||
published_at=datetime.now(),
|
||||
image_url=item.get('thumbnail', {}).get('src')
|
||||
))
|
||||
else:
|
||||
logger.error(f"Brave API error for '{query[:30]}': {response.status_code}")
|
||||
response = requests.get(
|
||||
'https://api.search.brave.com/res/v1/news/search',
|
||||
headers=headers,
|
||||
params=params,
|
||||
timeout=30
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Brave search error: {e}")
|
||||
if response.status_code == 200:
|
||||
results = response.json().get('results', [])
|
||||
for item in results:
|
||||
if item.get('url'):
|
||||
items.append(NewsItem(
|
||||
title=item.get('title', 'Bez tytułu'),
|
||||
url=item['url'],
|
||||
description=item.get('description', ''),
|
||||
source_name=item.get('source', ''),
|
||||
source_type='brave',
|
||||
source_id=f'brave_{query[:20]}',
|
||||
published_at=datetime.now(),
|
||||
image_url=item.get('thumbnail', {}).get('src')
|
||||
))
|
||||
break # success
|
||||
elif response.status_code == 429:
|
||||
if attempt < max_retries:
|
||||
wait = 1.5 * (2 ** attempt) # 1.5s, 3s
|
||||
logger.warning(f"Brave API 429 for '{query[:30]}', retry {attempt+1} after {wait}s")
|
||||
time.sleep(wait)
|
||||
else:
|
||||
logger.error(f"Brave API 429 for '{query[:30]}' after {max_retries} retries, skipping")
|
||||
else:
|
||||
logger.error(f"Brave API error for '{query[:30]}': {response.status_code}")
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Brave search error: {e}")
|
||||
break
|
||||
|
||||
return items
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user