Feature: Detaillierte Worker-Pool Performance-Metriken mit psutil
Neue Metrik-Erfassung für Saxon- und FOP-Worker-Pools: - Kompilierungszeit der Java-Worker-Klassen - Worker-Startzeiten (Summe + Durchschnitt pro Worker) - RAM-Verbrauch vor/nach Transformation (Summe + Durchschnitt) - Automatische Berechnung der RAM-Zunahme in MB und Prozent Technische Details: - Neue WorkerPoolMetrics-Datenklasse in worker_metrics.py - RAM-Messung via psutil (v7.2.1, neu hinzugefügt) - Metriken für beide Saxon-Varianten (JAXP + s9api) - WorkerPoolMetricsDialog mit Tab-basierter UI - Menüeintrag "Projekt → Worker-Pool-Metriken" Metriken werden automatisch erfasst: - Bei Worker-Pool-Initialisierung (Kompilierung + Start) - Vor erster Transformation (RAM-Baseline) - Nach allen Transformationen (RAM-Endwert) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
+79
-4
@@ -9,11 +9,15 @@ Jeder Worker läuft als Daemon und verarbeitet mehrere Transformationen nacheina
|
||||
import logging
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import psutil
|
||||
from pathlib import Path
|
||||
from queue import Queue
|
||||
from typing import Optional
|
||||
import tempfile
|
||||
|
||||
from worker_metrics import WorkerPoolMetrics
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Java-Worker-Code für s9api (wird zur Laufzeit kompiliert)
|
||||
@@ -179,6 +183,9 @@ class SaxonWorkerPoolS9Api:
|
||||
self.worker_class_path: Optional[Path] = None
|
||||
self.worker_log_dir: Optional[Path] = None
|
||||
|
||||
# Performance-Metriken
|
||||
self.metrics = WorkerPoolMetrics()
|
||||
|
||||
# Initialisierung
|
||||
self._compile_worker_class()
|
||||
self._start_workers()
|
||||
@@ -187,6 +194,7 @@ class SaxonWorkerPoolS9Api:
|
||||
|
||||
def _compile_worker_class(self):
|
||||
"""Kompiliert die SaxonS9ApiWorker-Java-Klasse."""
|
||||
start_time = time.time()
|
||||
try:
|
||||
# Erstelle temporäres Verzeichnis
|
||||
self.temp_dir = Path(tempfile.mkdtemp(prefix="saxon_s9api_worker_"))
|
||||
@@ -224,7 +232,13 @@ class SaxonWorkerPoolS9Api:
|
||||
|
||||
self.worker_class_path = self.temp_dir
|
||||
|
||||
logger.info(f"SaxonS9ApiWorker erfolgreich kompiliert: {self.temp_dir}")
|
||||
# Speichere Kompilierungszeit
|
||||
self.metrics.compilation_time_seconds = time.time() - start_time
|
||||
|
||||
logger.info(
|
||||
f"SaxonS9ApiWorker erfolgreich kompiliert: {self.temp_dir} "
|
||||
f"({self.metrics.compilation_time_seconds:.3f}s)"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Fehler beim Kompilieren von SaxonS9ApiWorker: {e}")
|
||||
@@ -267,6 +281,7 @@ class SaxonWorkerPoolS9Api:
|
||||
self.worker_log_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for i in range(self.num_workers):
|
||||
worker_start_time = time.time()
|
||||
try:
|
||||
# Starte JVM-Prozess mit SaxonS9ApiWorker
|
||||
cmd = [str(self.java_vm_path), "-cp", full_classpath, "SaxonS9ApiWorker"]
|
||||
@@ -290,8 +305,6 @@ class SaxonWorkerPoolS9Api:
|
||||
logger.debug(f"S9Api Worker {i} gestartet (PID: {process.pid}, stderr: {stderr_log})")
|
||||
|
||||
# Warte kurz damit Worker initialisieren kann
|
||||
import time
|
||||
|
||||
time.sleep(0.1)
|
||||
|
||||
# Prüfe ob Worker noch läuft
|
||||
@@ -304,11 +317,22 @@ class SaxonWorkerPoolS9Api:
|
||||
f"S9Api Worker {i} ist sofort beendet (Exit Code: {process.returncode})\nstderr:\n{stderr_content}"
|
||||
)
|
||||
|
||||
# Speichere Worker-Startzeit
|
||||
worker_elapsed = time.time() - worker_start_time
|
||||
self.metrics.worker_start_times.append(worker_elapsed)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Fehler beim Starten von S9Api Worker {i}: {e}")
|
||||
raise
|
||||
|
||||
logger.info(f"{len(self.workers)} Saxon-S9Api-Worker erfolgreich gestartet")
|
||||
# Berechne Aggregat-Werte für Worker-Startzeiten
|
||||
self.metrics.calculate_aggregates()
|
||||
|
||||
logger.info(
|
||||
f"{len(self.workers)} Saxon-S9Api-Worker erfolgreich gestartet "
|
||||
f"(Summe: {self.metrics.total_worker_start_time_seconds:.3f}s, "
|
||||
f"Durchschnitt: {self.metrics.average_worker_start_time_seconds:.3f}s)"
|
||||
)
|
||||
|
||||
def transform(
|
||||
self, source_xml: Path, xsl_stylesheet: Path, output_fo: Path, xslt_params: dict[str, str]
|
||||
@@ -399,6 +423,57 @@ class SaxonWorkerPoolS9Api:
|
||||
# Gebe Worker-Lock frei
|
||||
self.worker_locks[worker_idx].release()
|
||||
|
||||
def measure_ram_usage(self) -> tuple[float, float, list[float]]:
|
||||
"""
|
||||
Misst den aktuellen RAM-Verbrauch aller Worker-Prozesse.
|
||||
|
||||
Returns:
|
||||
tuple: (total_mb, average_mb, per_worker_mb_list)
|
||||
"""
|
||||
ram_per_worker = []
|
||||
|
||||
for i, worker in enumerate(self.workers):
|
||||
try:
|
||||
if worker.poll() is None: # Worker läuft noch
|
||||
process = psutil.Process(worker.pid)
|
||||
# Hole Speicherinfo (RSS = Resident Set Size in Bytes)
|
||||
mem_info = process.memory_info()
|
||||
ram_mb = mem_info.rss / (1024 * 1024) # Konvertiere zu MB
|
||||
ram_per_worker.append(ram_mb)
|
||||
else:
|
||||
logger.warning(f"Worker {i} ist nicht mehr aktiv (kann RAM nicht messen)")
|
||||
except (psutil.NoSuchProcess, psutil.AccessDenied) as e:
|
||||
logger.warning(f"Konnte RAM für Worker {i} nicht messen: {e}")
|
||||
|
||||
total_ram = sum(ram_per_worker)
|
||||
average_ram = total_ram / len(ram_per_worker) if ram_per_worker else 0.0
|
||||
|
||||
return total_ram, average_ram, ram_per_worker
|
||||
|
||||
def capture_ram_before_transform(self):
|
||||
"""Erfasst RAM-Verbrauch vor der ersten Transformation."""
|
||||
total, average, per_worker = self.measure_ram_usage()
|
||||
self.metrics.ram_before_transform_mb_per_worker = per_worker
|
||||
self.metrics.total_ram_before_mb = total
|
||||
self.metrics.average_ram_before_mb = average
|
||||
|
||||
logger.info(
|
||||
f"RAM vor Transformation: {self.metrics.total_ram_before_mb:.1f} MB "
|
||||
f"(Durchschnitt: {self.metrics.average_ram_before_mb:.1f} MB/Worker)"
|
||||
)
|
||||
|
||||
def capture_ram_after_transform(self):
|
||||
"""Erfasst RAM-Verbrauch nach allen Transformationen."""
|
||||
total, average, per_worker = self.measure_ram_usage()
|
||||
self.metrics.ram_after_transform_mb_per_worker = per_worker
|
||||
self.metrics.total_ram_after_mb = total
|
||||
self.metrics.average_ram_after_mb = average
|
||||
|
||||
logger.info(
|
||||
f"RAM nach Transformation: {self.metrics.total_ram_after_mb:.1f} MB "
|
||||
f"(Durchschnitt: {self.metrics.average_ram_after_mb:.1f} MB/Worker)"
|
||||
)
|
||||
|
||||
def shutdown(self):
|
||||
"""Beendet alle Worker-Prozesse sauber."""
|
||||
logger.info("Beende Saxon-S9Api-Worker-Pool...")
|
||||
|
||||
Reference in New Issue
Block a user