From c4c113aa6fd1dcf1a5f8d240445b90303ca8dcbc Mon Sep 17 00:00:00 2001 From: Maciej Pienczyn Date: Mon, 9 Feb 2026 14:43:34 +0100 Subject: [PATCH] fix(zopk): Add Brave API rate limit handling and heartbeat limit fix Brave free tier was returning 429 for ~50% of queries due to back-to-back requests. Added 1.1s delay between queries and retry with exponential backoff (1.5s, 3s). Heartbeat endpoint exempted from Flask-Limiter and interval increased from 30s to 60s to reduce log noise. Co-Authored-By: Claude Opus 4.6 --- blueprints/api/routes_analytics.py | 2 + static/js/analytics-tracker.js | 2 +- zopk_news_service.py | 92 ++++++++++++++++++------------ 3 files changed, 58 insertions(+), 38 deletions(-) diff --git a/blueprints/api/routes_analytics.py b/blueprints/api/routes_analytics.py index 2931bd4..c4ee9ec 100644 --- a/blueprints/api/routes_analytics.py +++ b/blueprints/api/routes_analytics.py @@ -13,6 +13,7 @@ from flask import jsonify, request, session, current_app from flask_login import current_user from database import SessionLocal, UserSession, UserClick, PageView, JSError +from extensions import limiter from . import bp logger = logging.getLogger(__name__) @@ -88,6 +89,7 @@ def api_analytics_track(): @bp.route('/analytics/heartbeat', methods=['POST']) +@limiter.exempt def api_analytics_heartbeat(): """Keep session alive and update duration""" analytics_session_id = session.get('analytics_session_id') diff --git a/static/js/analytics-tracker.js b/static/js/analytics-tracker.js index 215a55c..ecf4229 100644 --- a/static/js/analytics-tracker.js +++ b/static/js/analytics-tracker.js @@ -7,7 +7,7 @@ (function() { 'use strict'; - const HEARTBEAT_INTERVAL = 30000; // 30 seconds + const HEARTBEAT_INTERVAL = 60000; // 60 seconds const SCROLL_DEBOUNCE_MS = 500; const TRACK_ENDPOINT = '/api/analytics/track'; const HEARTBEAT_ENDPOINT = '/api/analytics/heartbeat'; diff --git a/zopk_news_service.py b/zopk_news_service.py index fb8f50e..210a3ae 100644 --- a/zopk_news_service.py +++ b/zopk_news_service.py @@ -21,6 +21,7 @@ Created: 2026-01-11 import os import re +import time import hashlib import logging import unicodedata @@ -678,6 +679,10 @@ class ZOPKNewsService: if progress_callback: progress_callback('search', f"Brave: {query_config['description']} ({len(brave_items)})", i + 1, len(BRAVE_QUERIES)) + + # Rate limit: Brave free tier ~1 req/s + if i < len(BRAVE_QUERIES) - 1: + time.sleep(1.1) else: process_log.append({ 'phase': 'search', @@ -947,50 +952,63 @@ class ZOPKNewsService: } def _search_brave_single(self, query: str) -> List[NewsItem]: - """Search Brave API with a single query""" + """Search Brave API with a single query, with retry on 429""" if not self.brave_api_key: return [] items = [] - try: - headers = { - 'Accept': 'application/json', - 'X-Subscription-Token': self.brave_api_key - } - params = { - 'q': query, - 'count': 10, # Fewer results per query (we have 8 queries) - 'freshness': 'pw', # past week (more relevant than past month) - 'country': 'pl', - 'search_lang': 'pl' - } + max_retries = 2 - response = requests.get( - 'https://api.search.brave.com/res/v1/news/search', - headers=headers, - params=params, - timeout=30 - ) + for attempt in range(max_retries + 1): + try: + headers = { + 'Accept': 'application/json', + 'X-Subscription-Token': self.brave_api_key + } + params = { + 'q': query, + 'count': 10, + 'freshness': 'pw', + 'country': 'pl', + 'search_lang': 'pl' + } - if response.status_code == 200: - results = response.json().get('results', []) - for item in results: - if item.get('url'): - items.append(NewsItem( - title=item.get('title', 'Bez tytułu'), - url=item['url'], - description=item.get('description', ''), - source_name=item.get('source', ''), - source_type='brave', - source_id=f'brave_{query[:20]}', - published_at=datetime.now(), - image_url=item.get('thumbnail', {}).get('src') - )) - else: - logger.error(f"Brave API error for '{query[:30]}': {response.status_code}") + response = requests.get( + 'https://api.search.brave.com/res/v1/news/search', + headers=headers, + params=params, + timeout=30 + ) - except Exception as e: - logger.error(f"Brave search error: {e}") + if response.status_code == 200: + results = response.json().get('results', []) + for item in results: + if item.get('url'): + items.append(NewsItem( + title=item.get('title', 'Bez tytułu'), + url=item['url'], + description=item.get('description', ''), + source_name=item.get('source', ''), + source_type='brave', + source_id=f'brave_{query[:20]}', + published_at=datetime.now(), + image_url=item.get('thumbnail', {}).get('src') + )) + break # success + elif response.status_code == 429: + if attempt < max_retries: + wait = 1.5 * (2 ** attempt) # 1.5s, 3s + logger.warning(f"Brave API 429 for '{query[:30]}', retry {attempt+1} after {wait}s") + time.sleep(wait) + else: + logger.error(f"Brave API 429 for '{query[:30]}' after {max_retries} retries, skipping") + else: + logger.error(f"Brave API error for '{query[:30]}': {response.status_code}") + break + + except Exception as e: + logger.error(f"Brave search error: {e}") + break return items