maciejpi/nordabiz
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix(zopk): Align scrape SSE stream format with frontend expectations
Some checks are pending
NordaBiz Tests / Unit & Integration Tests (push) Waiting to run
Details
NordaBiz Tests / E2E Tests (Playwright) (push) Blocked by required conditions
Details
NordaBiz Tests / Smoke Tests (Production) (push) Blocked by required conditions
Details
NordaBiz Tests / Send Failure Notification (push) Blocked by required conditions
Details

Browse Source 
The scrape stream used 'type' field and lacked 'percent', 'message',
'details' - format incompatible with the shared SSE modal handler.
Aligned to match knowledge stream format: status/percent/message/details.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Maciej Pienczyn 2026-02-09 15:29:33 +01:00
parent 4f5575dbcc
commit a174ca3103
1 changed files with 15 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
blueprints/admin/routes_zopk_news.py
View File

@ -817,6 +817,7 @@ def admin_zopk_news_scrape_stream():
def generate():
import json
import time as _time
db = SessionLocal()
try:
scraper = ZOPKContentScraper(db, user_id=user_id)
@ -835,8 +836,13 @@ def admin_zopk_news_scrape_stream():
articles = query.order_by(ZOPKNews.published_at.desc()).limit(limit).all()
total = len(articles)
yield f"data: {json.dumps({'type': 'start', 'total': total})}\n\n"
if total == 0:
yield f"data: {json.dumps({'status': 'complete', 'message': 'Brak artykułów do scrapowania', 'total': 0, 'details': {'success': 0, 'failed': 0, 'skipped': 0}}, ensure_ascii=False)}\n\n"
return
yield f"data: {json.dumps({'current': 0, 'total': total, 'percent': 0, 'status': 'processing', 'message': f'Rozpoczynam scraping {total} artykułów...'}, ensure_ascii=False)}\n\n"
start_time = _time.time()
scraped = 0
failed = 0
skipped = 0
@ -847,20 +853,25 @@ def admin_zopk_news_scrape_stream():
if result.success:
scraped += 1
status = 'success'
msg = f'✓ {result.word_count} słów: {article.title[:50]}'
elif result.status == 'skipped':
skipped += 1
status = 'skipped'
msg = f'⊘ Pominięto: {article.title[:50]}'
else:
failed += 1
status = 'failed'
msg = f'✗ {(result.error or "Błąd")[:40]}: {article.title[:40]}'
yield f"data: {json.dumps({'type': 'progress', 'current': i + 1, 'total': total, 'article': article.title[:50], 'status': status, 'scraped': scraped, 'failed': failed, 'skipped': skipped})}\n\n"
pct = round((i + 1) / total * 100, 1)
yield f"data: {json.dumps({'current': i + 1, 'total': total, 'percent': pct, 'status': status, 'message': msg, 'details': {'success': scraped, 'scraped': scraped, 'failed': failed, 'skipped': skipped}}, ensure_ascii=False)}\n\n"
yield f"data: {json.dumps({'type': 'complete', 'scraped': scraped, 'failed': failed, 'skipped': skipped})}\n\n"
processing_time = round(_time.time() - start_time, 2)
yield f"data: {json.dumps({'current': total, 'total': total, 'percent': 100, 'status': 'complete', 'message': f'Zakończono: {scraped} pobrano, {failed} błędów, {skipped} pominięto', 'details': {'success': scraped, 'scraped': scraped, 'failed': failed, 'skipped': skipped, 'processing_time': processing_time}}, ensure_ascii=False)}\n\n"
except Exception as e:
logger.error(f"Error in scrape stream: {e}")
yield f"data: {json.dumps({'type': 'error', 'error': str(e)})}\n\n"
yield f"data: {json.dumps({'status': 'error', 'message': str(e)}, ensure_ascii=False)}\n\n"
finally:
db.close()