fix(zopk): Add Brave API rate limit handling and heartbeat limit fix
Some checks are pending
NordaBiz Tests / Unit & Integration Tests (push) Waiting to run
NordaBiz Tests / E2E Tests (Playwright) (push) Blocked by required conditions
NordaBiz Tests / Smoke Tests (Production) (push) Blocked by required conditions
NordaBiz Tests / Send Failure Notification (push) Blocked by required conditions
Some checks are pending
NordaBiz Tests / Unit & Integration Tests (push) Waiting to run
NordaBiz Tests / E2E Tests (Playwright) (push) Blocked by required conditions
NordaBiz Tests / Smoke Tests (Production) (push) Blocked by required conditions
NordaBiz Tests / Send Failure Notification (push) Blocked by required conditions
Brave free tier was returning 429 for ~50% of queries due to back-to-back requests. Added 1.1s delay between queries and retry with exponential backoff (1.5s, 3s). Heartbeat endpoint exempted from Flask-Limiter and interval increased from 30s to 60s to reduce log noise. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
c7a42c3766
commit
c4c113aa6f
@ -13,6 +13,7 @@ from flask import jsonify, request, session, current_app
|
|||||||
from flask_login import current_user
|
from flask_login import current_user
|
||||||
|
|
||||||
from database import SessionLocal, UserSession, UserClick, PageView, JSError
|
from database import SessionLocal, UserSession, UserClick, PageView, JSError
|
||||||
|
from extensions import limiter
|
||||||
from . import bp
|
from . import bp
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@ -88,6 +89,7 @@ def api_analytics_track():
|
|||||||
|
|
||||||
|
|
||||||
@bp.route('/analytics/heartbeat', methods=['POST'])
|
@bp.route('/analytics/heartbeat', methods=['POST'])
|
||||||
|
@limiter.exempt
|
||||||
def api_analytics_heartbeat():
|
def api_analytics_heartbeat():
|
||||||
"""Keep session alive and update duration"""
|
"""Keep session alive and update duration"""
|
||||||
analytics_session_id = session.get('analytics_session_id')
|
analytics_session_id = session.get('analytics_session_id')
|
||||||
|
|||||||
@ -7,7 +7,7 @@
|
|||||||
(function() {
|
(function() {
|
||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
const HEARTBEAT_INTERVAL = 30000; // 30 seconds
|
const HEARTBEAT_INTERVAL = 60000; // 60 seconds
|
||||||
const SCROLL_DEBOUNCE_MS = 500;
|
const SCROLL_DEBOUNCE_MS = 500;
|
||||||
const TRACK_ENDPOINT = '/api/analytics/track';
|
const TRACK_ENDPOINT = '/api/analytics/track';
|
||||||
const HEARTBEAT_ENDPOINT = '/api/analytics/heartbeat';
|
const HEARTBEAT_ENDPOINT = '/api/analytics/heartbeat';
|
||||||
|
|||||||
@ -21,6 +21,7 @@ Created: 2026-01-11
|
|||||||
|
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
import time
|
||||||
import hashlib
|
import hashlib
|
||||||
import logging
|
import logging
|
||||||
import unicodedata
|
import unicodedata
|
||||||
@ -678,6 +679,10 @@ class ZOPKNewsService:
|
|||||||
|
|
||||||
if progress_callback:
|
if progress_callback:
|
||||||
progress_callback('search', f"Brave: {query_config['description']} ({len(brave_items)})", i + 1, len(BRAVE_QUERIES))
|
progress_callback('search', f"Brave: {query_config['description']} ({len(brave_items)})", i + 1, len(BRAVE_QUERIES))
|
||||||
|
|
||||||
|
# Rate limit: Brave free tier ~1 req/s
|
||||||
|
if i < len(BRAVE_QUERIES) - 1:
|
||||||
|
time.sleep(1.1)
|
||||||
else:
|
else:
|
||||||
process_log.append({
|
process_log.append({
|
||||||
'phase': 'search',
|
'phase': 'search',
|
||||||
@ -947,50 +952,63 @@ class ZOPKNewsService:
|
|||||||
}
|
}
|
||||||
|
|
||||||
def _search_brave_single(self, query: str) -> List[NewsItem]:
|
def _search_brave_single(self, query: str) -> List[NewsItem]:
|
||||||
"""Search Brave API with a single query"""
|
"""Search Brave API with a single query, with retry on 429"""
|
||||||
if not self.brave_api_key:
|
if not self.brave_api_key:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
items = []
|
items = []
|
||||||
try:
|
max_retries = 2
|
||||||
headers = {
|
|
||||||
'Accept': 'application/json',
|
|
||||||
'X-Subscription-Token': self.brave_api_key
|
|
||||||
}
|
|
||||||
params = {
|
|
||||||
'q': query,
|
|
||||||
'count': 10, # Fewer results per query (we have 8 queries)
|
|
||||||
'freshness': 'pw', # past week (more relevant than past month)
|
|
||||||
'country': 'pl',
|
|
||||||
'search_lang': 'pl'
|
|
||||||
}
|
|
||||||
|
|
||||||
response = requests.get(
|
for attempt in range(max_retries + 1):
|
||||||
'https://api.search.brave.com/res/v1/news/search',
|
try:
|
||||||
headers=headers,
|
headers = {
|
||||||
params=params,
|
'Accept': 'application/json',
|
||||||
timeout=30
|
'X-Subscription-Token': self.brave_api_key
|
||||||
)
|
}
|
||||||
|
params = {
|
||||||
|
'q': query,
|
||||||
|
'count': 10,
|
||||||
|
'freshness': 'pw',
|
||||||
|
'country': 'pl',
|
||||||
|
'search_lang': 'pl'
|
||||||
|
}
|
||||||
|
|
||||||
if response.status_code == 200:
|
response = requests.get(
|
||||||
results = response.json().get('results', [])
|
'https://api.search.brave.com/res/v1/news/search',
|
||||||
for item in results:
|
headers=headers,
|
||||||
if item.get('url'):
|
params=params,
|
||||||
items.append(NewsItem(
|
timeout=30
|
||||||
title=item.get('title', 'Bez tytułu'),
|
)
|
||||||
url=item['url'],
|
|
||||||
description=item.get('description', ''),
|
|
||||||
source_name=item.get('source', ''),
|
|
||||||
source_type='brave',
|
|
||||||
source_id=f'brave_{query[:20]}',
|
|
||||||
published_at=datetime.now(),
|
|
||||||
image_url=item.get('thumbnail', {}).get('src')
|
|
||||||
))
|
|
||||||
else:
|
|
||||||
logger.error(f"Brave API error for '{query[:30]}': {response.status_code}")
|
|
||||||
|
|
||||||
except Exception as e:
|
if response.status_code == 200:
|
||||||
logger.error(f"Brave search error: {e}")
|
results = response.json().get('results', [])
|
||||||
|
for item in results:
|
||||||
|
if item.get('url'):
|
||||||
|
items.append(NewsItem(
|
||||||
|
title=item.get('title', 'Bez tytułu'),
|
||||||
|
url=item['url'],
|
||||||
|
description=item.get('description', ''),
|
||||||
|
source_name=item.get('source', ''),
|
||||||
|
source_type='brave',
|
||||||
|
source_id=f'brave_{query[:20]}',
|
||||||
|
published_at=datetime.now(),
|
||||||
|
image_url=item.get('thumbnail', {}).get('src')
|
||||||
|
))
|
||||||
|
break # success
|
||||||
|
elif response.status_code == 429:
|
||||||
|
if attempt < max_retries:
|
||||||
|
wait = 1.5 * (2 ** attempt) # 1.5s, 3s
|
||||||
|
logger.warning(f"Brave API 429 for '{query[:30]}', retry {attempt+1} after {wait}s")
|
||||||
|
time.sleep(wait)
|
||||||
|
else:
|
||||||
|
logger.error(f"Brave API 429 for '{query[:30]}' after {max_retries} retries, skipping")
|
||||||
|
else:
|
||||||
|
logger.error(f"Brave API error for '{query[:30]}': {response.status_code}")
|
||||||
|
break
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Brave search error: {e}")
|
||||||
|
break
|
||||||
|
|
||||||
return items
|
return items
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user