mytec/rfcp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Phase 2.2: performance optimizations, debug tools, app close fix

Browse Source
This commit is contained in:
mytec
2026-01-31 20:31:53 +02:00
parent fb2b55caff
commit 26f8067c94
18 changed files with 1006 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files

413
backend/app/services/coverage_service.py
View File

@@ -1,6 +1,44 @@
import math
import os
import sys
import time
import threading
import numpy as np
import asyncio
from concurrent.futures import ThreadPoolExecutor
from typing import List, Optional, Tuple
_coverage_log_file = None
def _clog(msg: str):
"""Coverage debug log — always flushed, with timestamp and thread name.
Writes to stdout, stderr, AND a file so output is always available."""
global _coverage_log_file
ts = time.strftime('%H:%M:%S')
thr = threading.current_thread().name
line = f"[COVERAGE {ts}] [{thr}] {msg}"
print(line, flush=True)
# Backup: also write to stderr in case stdout is broken
try:
sys.stderr.write(line + '\n')
sys.stderr.flush()
except Exception:
pass
# Backup: also write to a file
try:
if _coverage_log_file is None:
log_dir = os.environ.get('RFCP_DATA_PATH', './data')
os.makedirs(log_dir, exist_ok=True)
log_path = os.path.join(log_dir, 'coverage-debug.log')
_coverage_log_file = open(log_path, 'a')
_coverage_log_file.write(f"\n{'='*60}\n")
_coverage_log_file.write(f"[COVERAGE {ts}] Log started\n")
_coverage_log_file.flush()
_coverage_log_file.write(line + '\n')
_coverage_log_file.flush()
except Exception:
pass
from pydantic import BaseModel
from app.services.terrain_service import terrain_service, TerrainService
from app.services.los_service import los_service
@@ -142,6 +180,100 @@ class CoverageService:
self.buildings = buildings_service
self.los = los_service
async def _fetch_osm_grid_aligned(
self,
min_lat: float, min_lon: float,
max_lat: float, max_lon: float,
settings: CoverageSettings
) -> dict:
"""
Fetch OSM data using 1-degree grid-aligned cells.
This ensures cache keys match the region download grid,
so pre-cached data is actually used.
"""
t0 = time.time()
lat_start = int(math.floor(min_lat))
lat_end = int(math.floor(max_lat))
lon_start = int(math.floor(min_lon))
lon_end = int(math.floor(max_lon))
cells = []
for lat_int in range(lat_start, lat_end + 1):
for lon_int in range(lon_start, lon_end + 1):
cells.append((float(lat_int), float(lon_int),
float(lat_int + 1), float(lon_int + 1)))
buildings: List[Building] = []
streets: List[Street] = []
water_bodies: List[WaterBody] = []
vegetation_areas: List[VegetationArea] = []
cache_stats = {"buildings": "skip", "streets": "skip",
"water": "skip", "vegetation": "skip"}
for cell_min_lat, cell_min_lon, cell_max_lat, cell_max_lon in cells:
cell_label = f"[{cell_min_lat:.0f},{cell_min_lon:.0f}]"
if settings.use_buildings:
t1 = time.time()
chunk = await self.buildings.fetch_buildings(
cell_min_lat, cell_min_lon, cell_max_lat, cell_max_lon
)
dt = time.time() - t1
src = "CACHE" if dt < 0.5 else "API"
buildings.extend(chunk)
cache_stats["buildings"] = src
_clog(f"Buildings {cell_label}: {len(chunk)} items ({src}, {dt:.1f}s)")
if settings.use_street_canyon:
t1 = time.time()
chunk = await street_canyon_service.fetch_streets(
cell_min_lat, cell_min_lon, cell_max_lat, cell_max_lon
)
dt = time.time() - t1
src = "CACHE" if dt < 0.5 else "API"
streets.extend(chunk)
cache_stats["streets"] = src
_clog(f"Streets {cell_label}: {len(chunk)} items ({src}, {dt:.1f}s)")
if settings.use_water_reflection:
t1 = time.time()
chunk = await water_service.fetch_water_bodies(
cell_min_lat, cell_min_lon, cell_max_lat, cell_max_lon
)
dt = time.time() - t1
src = "CACHE" if dt < 0.5 else "API"
water_bodies.extend(chunk)
cache_stats["water"] = src
_clog(f"Water {cell_label}: {len(chunk)} items ({src}, {dt:.1f}s)")
if settings.use_vegetation:
t1 = time.time()
chunk = await vegetation_service.fetch_vegetation(
cell_min_lat, cell_min_lon, cell_max_lat, cell_max_lon
)
dt = time.time() - t1
src = "CACHE" if dt < 0.5 else "API"
vegetation_areas.extend(chunk)
cache_stats["vegetation"] = src
_clog(f"Vegetation {cell_label}: {len(chunk)} items ({src}, {dt:.1f}s)")
total_fetch = time.time() - t0
_clog(f"OSM fetch total: {total_fetch:.1f}s "
f"({len(cells)} cells, "
f"{len(buildings)} bldgs, {len(streets)} streets, "
f"{len(water_bodies)} water, {len(vegetation_areas)} veg)")
_clog(f"Cache status: {cache_stats}")
return {
"buildings": buildings,
"streets": streets,
"water_bodies": water_bodies,
"vegetation_areas": vegetation_areas,
}
async def calculate_coverage(
self,
site: SiteParams,
@@ -152,6 +284,8 @@ class CoverageService:
Returns list of CoveragePoint with RSRP values
"""
calc_start = time.time()
# Apply preset if specified
settings = apply_preset(settings)
@@ -163,6 +297,7 @@ class CoverageService:
settings.radius,
settings.resolution
)
_clog(f"Grid: {len(grid)} points, radius={settings.radius}m, res={settings.resolution}m")
# Calculate bbox for data fetching
lat_delta = settings.radius / 111000
@@ -173,48 +308,80 @@ class CoverageService:
min_lon = site.lon - lon_delta
max_lon = site.lon + lon_delta
# Fetch buildings (if enabled) and build spatial index
buildings: List[Building] = []
_clog(f"Bbox: [{min_lat:.4f}, {min_lon:.4f}, {max_lat:.4f}, {max_lon:.4f}]")
# ━━━ PHASE 1: Fetch OSM data ━━━
_clog("━━━ PHASE 1: Fetching OSM data ━━━")
t_osm = time.time()
osm_data = await self._fetch_osm_grid_aligned(
min_lat, min_lon, max_lat, max_lon, settings
)
osm_time = time.time() - t_osm
buildings = osm_data["buildings"]
streets = osm_data["streets"]
water_bodies = osm_data["water_bodies"]
vegetation_areas = osm_data["vegetation_areas"]
_clog(f"━━━ PHASE 1 done: {osm_time:.1f}s ━━━")
# Build spatial index for buildings
spatial_idx: Optional[SpatialIndex] = None
if settings.use_buildings:
buildings = await self.buildings.fetch_buildings(
min_lat, min_lon, max_lat, max_lon
)
if buildings:
cache_key = f"{min_lat:.3f},{min_lon:.3f},{max_lat:.3f},{max_lon:.3f}"
spatial_idx = get_spatial_index(cache_key, buildings)
if buildings:
cache_key = f"{min_lat:.3f},{min_lon:.3f},{max_lat:.3f},{max_lon:.3f}"
spatial_idx = get_spatial_index(cache_key, buildings)
# Fetch streets (if street canyon enabled)
streets: List[Street] = []
if settings.use_street_canyon:
streets = await street_canyon_service.fetch_streets(
min_lat, min_lon, max_lat, max_lon
)
# ━━━ PHASE 2: Pre-load terrain ━━━
_clog("━━━ PHASE 2: Pre-loading terrain ━━━")
t_terrain = time.time()
tile_names = await self.terrain.ensure_tiles_for_bbox(
min_lat, min_lon, max_lat, max_lon
)
for tn in tile_names:
self.terrain._load_tile(tn)
# Fetch water bodies (if water reflection enabled)
water_bodies: List[WaterBody] = []
if settings.use_water_reflection:
water_bodies = await water_service.fetch_water_bodies(
min_lat, min_lon, max_lat, max_lon
)
site_elevation = self.terrain.get_elevation_sync(site.lat, site.lon)
# Fetch vegetation (if enabled)
vegetation_areas: List[VegetationArea] = []
if settings.use_vegetation:
vegetation_areas = await vegetation_service.fetch_vegetation(
min_lat, min_lon, max_lat, max_lon
)
# Calculate coverage for each point
point_elevations = {}
for lat, lon in grid:
point = await self._calculate_point(
site, lat, lon,
settings, buildings, streets,
spatial_idx, water_bodies, vegetation_areas
)
point_elevations[(lat, lon)] = self.terrain.get_elevation_sync(lat, lon)
terrain_time = time.time() - t_terrain
_clog(f"Tiles: {len(tile_names)}, site elev: {site_elevation:.0f}m, "
f"pre-computed {len(grid)} elevations")
_clog(f"━━━ PHASE 2 done: {terrain_time:.1f}s ━━━")
if point.rsrp >= settings.min_signal:
points.append(point)
# ━━━ PHASE 3: Point calculation (sync, in thread pool) ━━━
_clog(f"━━━ PHASE 3: Calculating {len(grid)} points (threaded) ━━━")
dominant_path_service._log_count = 0 # Reset diagnostic counter
t_points = time.time()
loop = asyncio.get_event_loop()
points, timing = await loop.run_in_executor(
None,
self._run_point_loop,
grid, site, settings, buildings, streets,
spatial_idx, water_bodies, vegetation_areas,
site_elevation, point_elevations
)
points_time = time.time() - t_points
total_time = time.time() - calc_start
_clog(f"━━━ PHASE 3 done: {points_time:.1f}s ━━━")
_clog("=== RESULTS ===")
_clog(f" Grid points: {len(grid)}")
_clog(f" Result points: {len(points)}")
_clog(f" OSM fetch: {osm_time:.1f}s")
_clog(f" Terrain pre-load:{terrain_time:.1f}s")
_clog(f" Point calc: {points_time:.1f}s "
f"({points_time/max(1,len(grid))*1000:.1f}ms/point)")
_clog(f" TOTAL: {total_time:.1f}s")
_clog(f" Tiles in memory: {len(self.terrain._tile_cache)}")
if any(v > 0.001 for v in timing.values()):
_clog("=== PER-STEP BREAKDOWN ===")
for step, dt in timing.items():
if dt > 0.001:
_clog(f" {step:20s} {dt:.3f}s "
f"({dt/max(1,len(grid))*1000:.2f}ms/point)")
return points
@@ -280,7 +447,36 @@ class CoverageService:
return points
async def _calculate_point(
def _run_point_loop(
self, grid, site, settings, buildings, streets,
spatial_idx, water_bodies, vegetation_areas,
site_elevation, point_elevations
):
"""Sync point loop - runs in ThreadPoolExecutor, bypasses event loop."""
points = []
timing = {"los": 0.0, "buildings": 0.0, "antenna": 0.0,
"dominant_path": 0.0, "street_canyon": 0.0,
"reflection": 0.0, "vegetation": 0.0}
total = len(grid)
log_interval = max(1, total // 20)
for i, (lat, lon) in enumerate(grid):
if i % log_interval == 0:
_clog(f"Progress: {i}/{total} ({i*100//total}%)")
point = self._calculate_point_sync(
site, lat, lon, settings, buildings, streets,
spatial_idx, water_bodies, vegetation_areas,
site_elevation, point_elevations.get((lat, lon), 0.0),
timing
)
if point.rsrp >= settings.min_signal:
points.append(point)
_clog(f"Progress: {total}/{total} (100%)")
return points, timing
def _calculate_point_sync(
self,
site: SiteParams,
lat: float, lon: float,
@@ -289,58 +485,62 @@ class CoverageService:
streets: List[Street],
spatial_idx: Optional[SpatialIndex],
water_bodies: List[WaterBody],
vegetation_areas: List[VegetationArea]
vegetation_areas: List[VegetationArea],
site_elevation: float,
point_elevation: float,
timing: dict
) -> CoveragePoint:
"""Calculate RSRP at a single point with all propagation models"""
"""Fully synchronous point calculation. All terrain tiles must be pre-loaded."""
# Distance
distance = TerrainService.haversine_distance(site.lat, site.lon, lat, lon)
if distance < 1:
distance = 1 # Avoid division by zero
distance = 1
# Base path loss (Okumura-Hata for urban)
path_loss = self._okumura_hata(
distance, site.frequency, site.height, 1.5
)
# Base path loss
path_loss = self._okumura_hata(distance, site.frequency, site.height, 1.5)
# Antenna pattern loss (if directional)
# Antenna pattern
antenna_loss = 0.0
if site.azimuth is not None and site.beamwidth:
t0 = time.time()
antenna_loss = self._antenna_pattern_loss(
site.lat, site.lon, lat, lon,
site.azimuth, site.beamwidth
site.lat, site.lon, lat, lon, site.azimuth, site.beamwidth
)
timing["antenna"] += time.time() - t0
# Terrain loss (LoS check)
# Terrain LOS (sync)
terrain_loss = 0.0
has_los = True
if settings.use_terrain:
los_result = await self.los.check_line_of_sight(
site.lat, site.lon, site.height,
lat, lon, 1.5
t0 = time.time()
los_result = self.los.check_line_of_sight_sync(
site.lat, site.lon, site.height, lat, lon, 1.5
)
has_los = los_result["has_los"]
if not has_los:
clearance = los_result["clearance"]
terrain_loss = self._diffraction_loss(clearance, site.frequency)
terrain_loss = self._diffraction_loss(
los_result["clearance"], site.frequency
)
timing["los"] += time.time() - t0
# Building loss — use spatial index for fast lookup
# Building loss (spatial index)
building_loss = 0.0
t0 = time.time()
nearby_buildings = (
spatial_idx.query_line(site.lat, site.lon, lat, lon)
if spatial_idx else buildings
)
if settings.use_buildings and nearby_buildings:
site_total_h = site.height + site_elevation
point_total_h = 1.5 + point_elevation
if settings.use_materials:
for building in nearby_buildings:
intersection = self.buildings.line_intersects_building(
site.lat, site.lon, site.height + await self.terrain.get_elevation(site.lat, site.lon),
lat, lon, 1.5 + await self.terrain.get_elevation(lat, lon),
building
site.lat, site.lon, site_total_h,
lat, lon, point_total_h, building
)
if intersection is not None:
material = materials_service.detect_material(building.tags)
@@ -352,21 +552,23 @@ class CoverageService:
else:
for building in nearby_buildings:
intersection = self.buildings.line_intersects_building(
site.lat, site.lon, site.height + await self.terrain.get_elevation(site.lat, site.lon),
lat, lon, 1.5 + await self.terrain.get_elevation(lat, lon),
building
site.lat, site.lon, site_total_h,
lat, lon, point_total_h, building
)
if intersection is not None:
building_loss += 20.0
has_los = False
break
timing["buildings"] += time.time() - t0
# Dominant path analysis
if settings.use_dominant_path and nearby_buildings:
paths = await dominant_path_service.find_dominant_paths(
# Dominant path (sync) — uses spatial index for O(1) building lookups
if settings.use_dominant_path and (spatial_idx or nearby_buildings):
t0 = time.time()
paths = dominant_path_service.find_dominant_paths_sync(
site.lat, site.lon, site.height,
lat, lon, 1.5,
site.frequency, nearby_buildings
site.frequency, nearby_buildings,
spatial_idx=spatial_idx
)
if paths:
best_path = paths[0]
@@ -375,10 +577,12 @@ class CoverageService:
terrain_loss = 0
building_loss = 0
has_los = best_path.path_type == "direct" and not best_path.materials_crossed
timing["dominant_path"] += time.time() - t0
# Street canyon model
# Street canyon (sync)
if settings.use_street_canyon and streets:
canyon_loss, street_path = await street_canyon_service.calculate_street_canyon_loss(
t0 = time.time()
canyon_loss, _street_path = street_canyon_service.calculate_street_canyon_loss_sync(
site.lat, site.lon, site.height,
lat, lon, 1.5,
site.frequency, streets
@@ -387,80 +591,76 @@ class CoverageService:
path_loss = canyon_loss
terrain_loss = 0
building_loss = 0
timing["street_canyon"] += time.time() - t0
# Vegetation loss
# Vegetation (already sync)
veg_loss = 0.0
if settings.use_vegetation and vegetation_areas:
t0 = time.time()
veg_loss = vegetation_service.calculate_vegetation_loss(
site.lat, site.lon, lat, lon,
vegetation_areas, settings.season
site.lat, site.lon, lat, lon, vegetation_areas, settings.season
)
timing["vegetation"] += time.time() - t0
# Reflections (building + ground/water)
# Reflections (sync)
reflection_gain = 0.0
if settings.use_reflections and nearby_buildings:
t0 = time.time()
is_over_water = False
if settings.use_water_reflection and water_bodies:
is_over_water = water_service.point_over_water(lat, lon, water_bodies) is not None
reflection_paths = await reflection_service.find_reflection_paths(
refl_paths = reflection_service.find_reflection_paths_sync(
site.lat, site.lon, site.height,
lat, lon, 1.5,
site.frequency, nearby_buildings,
include_ground=True
)
# If over water, replace ground reflection with stronger water reflection
if is_over_water and reflection_paths:
if is_over_water and refl_paths:
water_path = reflection_service._calculate_ground_reflection(
site.lat, site.lon, site.height,
lat, lon, 1.5,
site.frequency, is_water=True
)
if water_path:
reflection_paths = [
p for p in reflection_paths if "ground" not in p.materials
]
reflection_paths.append(water_path)
reflection_paths.sort(key=lambda p: p.total_loss)
refl_paths = [p for p in refl_paths if "ground" not in p.materials]
refl_paths.append(water_path)
refl_paths.sort(key=lambda p: p.total_loss)
if reflection_paths:
direct_rsrp = site.power + site.gain - path_loss - antenna_loss - terrain_loss - building_loss - veg_loss
if refl_paths:
direct_rsrp = (site.power + site.gain - path_loss - antenna_loss
- terrain_loss - building_loss - veg_loss)
combined_rsrp = reflection_service.combine_paths(
direct_rsrp, reflection_paths, site.power + site.gain
direct_rsrp, refl_paths, site.power + site.gain
)
reflection_gain = max(0, combined_rsrp - direct_rsrp)
timing["reflection"] += time.time() - t0
elif settings.use_water_reflection and water_bodies and not settings.use_reflections:
# Water reflection without full reflection model
is_over_water = water_service.point_over_water(lat, lon, water_bodies) is not None
if is_over_water:
reflection_gain = 3.0 # ~3dB boost over water
reflection_gain = 3.0
# Rain attenuation
# Rain
rain_loss = 0.0
if settings.rain_rate > 0:
rain_loss = weather_service.calculate_rain_attenuation(
site.frequency,
distance / 1000, # km
settings.rain_rate
site.frequency, distance / 1000, settings.rain_rate
)
# Indoor penetration loss
# Indoor
indoor_loss = 0.0
if settings.indoor_loss_type != "none":
indoor_loss = indoor_service.calculate_indoor_loss(
site.frequency,
settings.indoor_loss_type
site.frequency, settings.indoor_loss_type
)
# Atmospheric absorption
# Atmospheric
atmo_loss = 0.0
if settings.use_atmospheric:
atmo_loss = atmospheric_service.calculate_atmospheric_loss(
site.frequency,
distance / 1000,
settings.temperature_c,
settings.humidity_percent
site.frequency, distance / 1000,
settings.temperature_c, settings.humidity_percent
)
# Final RSRP
@@ -470,18 +670,11 @@ class CoverageService:
+ reflection_gain)
return CoveragePoint(
lat=lat,
lon=lon,
rsrp=rsrp,
distance=distance,
has_los=has_los,
terrain_loss=terrain_loss,
building_loss=building_loss,
reflection_gain=reflection_gain,
vegetation_loss=veg_loss,
rain_loss=rain_loss,
indoor_loss=indoor_loss,
atmospheric_loss=atmo_loss,
lat=lat, lon=lon, rsrp=rsrp, distance=distance,
has_los=has_los, terrain_loss=terrain_loss,
building_loss=building_loss, reflection_gain=reflection_gain,
vegetation_loss=veg_loss, rain_loss=rain_loss,
indoor_loss=indoor_loss, atmospheric_loss=atmo_loss,
)
def _okumura_hata(