mytec/rfcp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

@mytec: feat: Phase 3.0 Architecture Refactor ✅

Browse Source 
Major refactoring of RFCP backend:
- Modular propagation models (8 models)
- SharedMemoryManager for terrain data
- ProcessPoolExecutor parallel processing
- WebSocket progress streaming
- Building filtering pipeline (351k → 15k)
- 82 unit tests

Performance: Standard preset 38s → 5s (7.6x speedup)

Known issue: Detailed preset timeout (fix in 3.1.0)
This commit is contained in:
mytec
2026-02-01 23:12:26 +02:00
parent 1dde56705a
commit defa3ad440
71 changed files with 7134 additions and 256 deletions
Download Patch File Download Diff File Expand all files Collapse all files

367
backend/app/services/coverage_service.py
View File

@@ -6,7 +6,7 @@ import threading
import numpy as np
import asyncio
from concurrent.futures import ThreadPoolExecutor
from typing import List, Optional, Tuple
from typing import List, Optional, Tuple, Callable
_coverage_log_file = None
@@ -58,6 +58,141 @@ from app.services.parallel_coverage_service import (
CancellationToken,
)
# ── New propagation models (Phase 3.0) ──
from app.propagation.base import PropagationModel, PropagationInput, PropagationOutput
from app.propagation.free_space import FreeSpaceModel
from app.propagation.okumura_hata import OkumuraHataModel
from app.propagation.cost231_hata import Cost231HataModel
from app.propagation.cost231_wi import Cost231WIModel
from app.propagation.itu_r_p1546 import ITUR_P1546Model
from app.propagation.longley_rice import LongleyRiceModel
from app.propagation.itu_r_p526 import KnifeEdgeDiffractionModel
# Pre-instantiate models (stateless, thread-safe)
_PROPAGATION_MODELS = {
'free_space': FreeSpaceModel(),
'okumura_hata': OkumuraHataModel(),
'cost231_hata': Cost231HataModel(),
'cost231_wi': Cost231WIModel(),
'itu_r_p1546': ITUR_P1546Model(),
'longley_rice': LongleyRiceModel(),
}
_DIFFRACTION_MODEL = KnifeEdgeDiffractionModel()
def select_propagation_model(frequency_mhz: float, environment: str = "urban") -> PropagationModel:
"""Select the best propagation model for a given frequency and environment.
Model selection logic:
- < 150 MHz: Longley-Rice (ITM, designed for VHF)
- 150-520 MHz: ITU-R P.1546 (urban) / Longley-Rice (rural)
- 520-1500 MHz: Okumura-Hata
- 1500-2000 MHz: COST-231 Hata
- > 2000 MHz: Free-Space Path Loss
"""
if frequency_mhz < 150:
return _PROPAGATION_MODELS['longley_rice']
elif frequency_mhz <= 520:
if environment in ('rural', 'open'):
return _PROPAGATION_MODELS['longley_rice']
return _PROPAGATION_MODELS['itu_r_p1546']
elif frequency_mhz <= 1500:
return _PROPAGATION_MODELS['okumura_hata']
elif frequency_mhz <= 2000:
return _PROPAGATION_MODELS['cost231_hata']
else:
return _PROPAGATION_MODELS['free_space']
# ── OSM data filtering ──
# OSM fetches use 1-degree grid cells — much larger than the coverage radius.
# Passing all buildings to ProcessPool workers causes MemoryError (pickle copy
# per worker). Filter to coverage bbox and cap count for safety.
MAX_BUILDINGS_FOR_WORKERS = 15000
def _filter_buildings_to_bbox(
buildings: list,
min_lat: float, min_lon: float,
max_lat: float, max_lon: float,
site_lat: float, site_lon: float,
log_fn=None,
) -> list:
"""Filter buildings to coverage bbox and cap at MAX_BUILDINGS_FOR_WORKERS.
Returns buildings sorted by distance to site (nearest first) so the
cap preserves buildings most likely to affect coverage.
"""
if not buildings or len(buildings) <= MAX_BUILDINGS_FOR_WORKERS:
return buildings
original = len(buildings)
# Fast bbox filter: keep buildings with any vertex inside the bbox
# Use a small buffer (~500m ≈ 0.005°) for LOS checks near edges
buf = 0.005
filtered = []
for b in buildings:
for lon_pt, lat_pt in b.geometry:
if (min_lat - buf) <= lat_pt <= (max_lat + buf) and \
(min_lon - buf) <= lon_pt <= (max_lon + buf):
filtered.append(b)
break
if log_fn:
log_fn(f"Building bbox filter: {original} -> {len(filtered)}")
# If still too many, sort by centroid distance and cap
if len(filtered) > MAX_BUILDINGS_FOR_WORKERS:
def _centroid_dist(b):
lats = [p[1] for p in b.geometry]
lons = [p[0] for p in b.geometry]
clat = sum(lats) / len(lats)
clon = sum(lons) / len(lons)
return (clat - site_lat) ** 2 + (clon - site_lon) ** 2
filtered.sort(key=_centroid_dist)
filtered = filtered[:MAX_BUILDINGS_FOR_WORKERS]
if log_fn:
log_fn(f"Building distance cap: -> {len(filtered)} (nearest to site)")
return filtered
def _filter_osm_list_to_bbox(items: list, min_lat: float, min_lon: float,
max_lat: float, max_lon: float,
max_count: int = 20000) -> list:
"""Filter OSM items (streets/water/vegetation) to coverage bbox.
Items must have a .geometry attribute (list of [lon, lat] pairs) or
lat/lon attributes. Returns at most max_count items.
"""
if not items or len(items) <= max_count:
return items
buf = 0.005
filtered = []
for item in items:
geom = getattr(item, 'geometry', None) or getattr(item, 'points', None)
if geom:
for pt in geom:
if isinstance(pt, (list, tuple)) and len(pt) >= 2:
lon_pt, lat_pt = pt[0], pt[1]
elif hasattr(pt, 'lat'):
lat_pt, lon_pt = pt.lat, pt.lon
else:
continue
if (min_lat - buf) <= lat_pt <= (max_lat + buf) and \
(min_lon - buf) <= lon_pt <= (max_lon + buf):
filtered.append(item)
break
else:
# No geometry — keep it
filtered.append(item)
return filtered[:max_count]
class CoveragePoint(BaseModel):
lat: float
@@ -79,6 +214,9 @@ class CoverageSettings(BaseModel):
resolution: float = 200 # meters
min_signal: float = -120 # dBm threshold
# Environment type for propagation model selection
environment: str = "urban" # urban, suburban, rural, open
# Layer toggles
use_terrain: bool = True
use_buildings: bool = True
@@ -283,11 +421,13 @@ class CoverageService:
site: SiteParams,
settings: CoverageSettings,
cancel_token: Optional[CancellationToken] = None,
progress_fn: Optional[Callable[[str, float], None]] = None,
) -> List[CoveragePoint]:
"""
Calculate coverage grid for a single site
Returns list of CoveragePoint with RSRP values
Returns list of CoveragePoint with RSRP values.
progress_fn(phase, pct): optional callback for progress updates (0.0-1.0).
"""
calc_start = time.time()
@@ -317,6 +457,9 @@ class CoverageService:
# ━━━ PHASE 1: Fetch OSM data ━━━
_clog("━━━ PHASE 1: Fetching OSM data ━━━")
if progress_fn:
progress_fn("Fetching map data", 0.10)
await asyncio.sleep(0) # Yield so progress_sender can flush WS message
t_osm = time.time()
osm_data = await self._fetch_osm_grid_aligned(
min_lat, min_lon, max_lat, max_lon, settings
@@ -329,6 +472,17 @@ class CoverageService:
vegetation_areas = osm_data["vegetation_areas"]
_clog(f"━━━ PHASE 1 done: {osm_time:.1f}s ━━━")
# ── Filter OSM data to coverage area ──
# OSM cells are 1-degree wide, often far larger than the coverage radius.
# Passing 350k buildings to ProcessPool workers causes MemoryError (pickle).
buildings = _filter_buildings_to_bbox(
buildings, min_lat, min_lon, max_lat, max_lon,
site.lat, site.lon, _clog,
)
streets = _filter_osm_list_to_bbox(streets, min_lat, min_lon, max_lat, max_lon)
water_bodies = _filter_osm_list_to_bbox(water_bodies, min_lat, min_lon, max_lat, max_lon)
vegetation_areas = _filter_osm_list_to_bbox(vegetation_areas, min_lat, min_lon, max_lat, max_lon)
# Build spatial index for buildings
spatial_idx: Optional[SpatialIndex] = None
if buildings:
@@ -337,6 +491,9 @@ class CoverageService:
# ━━━ PHASE 2: Pre-load terrain ━━━
_clog("━━━ PHASE 2: Pre-loading terrain ━━━")
if progress_fn:
progress_fn("Loading terrain", 0.25)
await asyncio.sleep(0)
t_terrain = time.time()
tile_names = await self.terrain.ensure_tiles_for_bbox(
min_lat, min_lon, max_lat, max_lon
@@ -355,6 +512,9 @@ class CoverageService:
_clog(f"━━━ PHASE 2 done: {terrain_time:.1f}s ━━━")
# ━━━ PHASE 2.5: Vectorized pre-computation (GPU/NumPy) ━━━
if progress_fn:
progress_fn("Pre-computing propagation", 0.35)
await asyncio.sleep(0)
from app.services.gpu_service import gpu_service
t_gpu = time.time()
@@ -365,7 +525,8 @@ class CoverageService:
grid_lats, grid_lons, site.lat, site.lon
)
pre_path_loss = gpu_service.precompute_path_loss(
pre_distances, site.frequency, site.height
pre_distances, site.frequency, site.height,
environment=getattr(settings, 'environment', 'urban'),
)
# Build lookup dict for point loop
@@ -377,8 +538,11 @@ class CoverageService:
}
gpu_time = time.time() - t_gpu
env = getattr(settings, 'environment', 'urban')
selected_model = select_propagation_model(site.frequency, env)
_clog(f"━━━ PHASE 2.5: Vectorized pre-computation done: {gpu_time:.3f}s "
f"({len(grid)} points, backend={'GPU' if gpu_service.available else 'CPU/NumPy'}) ━━━")
f"({len(grid)} points, model={selected_model.name}, freq={site.frequency}MHz, "
f"env={env}, backend={'GPU' if gpu_service.available else 'CPU/NumPy'}) ━━━")
# ━━━ PHASE 3: Point calculation ━━━
dominant_path_service._log_count = 0 # Reset diagnostic counter
@@ -387,6 +551,10 @@ class CoverageService:
use_parallel = len(grid) > 100 and get_cpu_count() > 1
num_workers = get_cpu_count()
if progress_fn:
progress_fn("Calculating coverage", 0.40)
await asyncio.sleep(0)
if use_parallel:
backend = get_parallel_backend()
_clog(f"━━━ PHASE 3: Calculating {len(grid)} points "
@@ -404,6 +572,7 @@ class CoverageService:
site_elevation, num_workers, _clog,
cancel_token=cancel_token,
precomputed=precomputed,
progress_fn=progress_fn,
),
)
@@ -426,9 +595,14 @@ class CoverageService:
site_elevation, point_elevations,
cancel_token=cancel_token,
precomputed=precomputed,
progress_fn=progress_fn,
),
)
if progress_fn:
progress_fn("Finalizing", 0.95)
await asyncio.sleep(0)
points_time = time.time() - t_points
total_time = time.time() - calc_start
@@ -522,6 +696,7 @@ class CoverageService:
spatial_idx, water_bodies, vegetation_areas,
site_elevation, point_elevations,
cancel_token=None, precomputed=None,
progress_fn=None,
):
"""Sync point loop - runs in ThreadPoolExecutor, bypasses event loop."""
points = []
@@ -538,6 +713,8 @@ class CoverageService:
if i % log_interval == 0:
_clog(f"Progress: {i}/{total} ({i*100//total}%)")
if progress_fn:
progress_fn("Calculating coverage", 0.40 + 0.55 * (i / total))
pre = precomputed.get((lat, lon)) if precomputed else None
@@ -581,11 +758,20 @@ class CoverageService:
if distance < 1:
distance = 1
# Base path loss (use precomputed if available)
# Base path loss (use precomputed if available, else use new model)
if precomputed_path_loss is not None:
path_loss = precomputed_path_loss
else:
path_loss = self._okumura_hata(distance, site.frequency, site.height, 1.5)
env = getattr(settings, 'environment', 'urban')
model = select_propagation_model(site.frequency, env)
prop_input = PropagationInput(
frequency_mhz=site.frequency,
distance_m=distance,
tx_height_m=site.height,
rx_height_m=1.5,
environment=env,
)
path_loss = model.calculate(prop_input).path_loss_db
# Antenna pattern
antenna_loss = 0.0
@@ -649,90 +835,105 @@ class CoverageService:
timing["buildings"] += time.time() - t0
# Dominant path (vectorized NumPy) — replaces loop-based sync version
if settings.use_dominant_path and (spatial_idx or nearby_buildings):
# Only enter when there are actual buildings (spatial_idx with data OR non-empty list)
has_building_data = nearby_buildings or (spatial_idx is not None and spatial_idx._grid)
if settings.use_dominant_path and has_building_data:
t0 = time.time()
dominant = find_dominant_paths_vectorized(
site.lat, site.lon, site.height,
lat, lon, 1.5,
site.frequency, nearby_buildings,
spatial_idx=spatial_idx,
)
if dominant['path_type'] == 'direct':
# Direct LOS confirmed by vectorized check
has_los = True
building_loss = 0.0
elif dominant['path_type'] == 'reflection':
# Reflection path bypasses buildings — reduce building loss
building_loss = max(0.0, building_loss - (10.0 - dominant['total_loss']))
has_los = False
elif dominant['path_type'] == 'diffraction':
# Diffraction: use estimated loss if worse than current
if dominant['total_loss'] > building_loss:
building_loss = dominant['total_loss']
has_los = False
try:
dominant = find_dominant_paths_vectorized(
site.lat, site.lon, site.height,
lat, lon, 1.5,
site.frequency, nearby_buildings,
spatial_idx=spatial_idx,
)
if dominant['path_type'] == 'direct':
has_los = True
building_loss = 0.0
elif dominant['path_type'] == 'reflection':
building_loss = max(0.0, building_loss - (10.0 - dominant['total_loss']))
has_los = False
elif dominant['path_type'] == 'diffraction':
if dominant['total_loss'] > building_loss:
building_loss = dominant['total_loss']
has_los = False
except Exception:
pass # Skip dominant path on error — use base model
timing["dominant_path"] += time.time() - t0
# Street canyon (sync)
if settings.use_street_canyon and streets:
t0 = time.time()
canyon_loss, _street_path = street_canyon_service.calculate_street_canyon_loss_sync(
site.lat, site.lon, site.height,
lat, lon, 1.5,
site.frequency, streets
)
if canyon_loss < (path_loss + terrain_loss + building_loss):
path_loss = canyon_loss
terrain_loss = 0
building_loss = 0
try:
canyon_loss, _street_path = street_canyon_service.calculate_street_canyon_loss_sync(
site.lat, site.lon, site.height,
lat, lon, 1.5,
site.frequency, streets
)
# Only use street canyon if it's a finite improvement
if math.isfinite(canyon_loss) and canyon_loss < (path_loss + terrain_loss + building_loss):
path_loss = canyon_loss
terrain_loss = 0
building_loss = 0
except Exception:
pass # Skip street canyon on error
timing["street_canyon"] += time.time() - t0
# Vegetation (already sync)
veg_loss = 0.0
if settings.use_vegetation and vegetation_areas:
t0 = time.time()
veg_loss = vegetation_service.calculate_vegetation_loss(
site.lat, site.lon, lat, lon, vegetation_areas, settings.season
)
try:
veg_loss = vegetation_service.calculate_vegetation_loss(
site.lat, site.lon, lat, lon, vegetation_areas, settings.season
)
except Exception:
veg_loss = 0.0
timing["vegetation"] += time.time() - t0
# Reflections (sync)
reflection_gain = 0.0
if settings.use_reflections and nearby_buildings:
t0 = time.time()
is_over_water = False
if settings.use_water_reflection and water_bodies:
is_over_water = water_service.point_over_water(lat, lon, water_bodies) is not None
try:
is_over_water = False
if settings.use_water_reflection and water_bodies:
is_over_water = water_service.point_over_water(lat, lon, water_bodies) is not None
refl_paths = reflection_service.find_reflection_paths_sync(
site.lat, site.lon, site.height,
lat, lon, 1.5,
site.frequency, nearby_buildings,
include_ground=True
)
if is_over_water and refl_paths:
water_path = reflection_service._calculate_ground_reflection(
refl_paths = reflection_service.find_reflection_paths_sync(
site.lat, site.lon, site.height,
lat, lon, 1.5,
site.frequency, is_water=True
site.frequency, nearby_buildings,
include_ground=True
)
if water_path:
refl_paths = [p for p in refl_paths if "ground" not in p.materials]
refl_paths.append(water_path)
refl_paths.sort(key=lambda p: p.total_loss)
if refl_paths:
direct_rsrp = (site.power + site.gain - path_loss - antenna_loss
- terrain_loss - building_loss - veg_loss)
combined_rsrp = reflection_service.combine_paths(
direct_rsrp, refl_paths, site.power + site.gain
)
reflection_gain = max(0, combined_rsrp - direct_rsrp)
if is_over_water and refl_paths:
water_path = reflection_service._calculate_ground_reflection(
site.lat, site.lon, site.height,
lat, lon, 1.5,
site.frequency, is_water=True
)
if water_path:
refl_paths = [p for p in refl_paths if "ground" not in p.materials]
refl_paths.append(water_path)
refl_paths.sort(key=lambda p: p.total_loss)
if refl_paths:
direct_rsrp = (site.power + site.gain - path_loss - antenna_loss
- terrain_loss - building_loss - veg_loss)
combined_rsrp = reflection_service.combine_paths(
direct_rsrp, refl_paths, site.power + site.gain
)
reflection_gain = max(0, combined_rsrp - direct_rsrp)
except Exception:
reflection_gain = 0.0
timing["reflection"] += time.time() - t0
elif settings.use_water_reflection and water_bodies and not settings.use_reflections:
is_over_water = water_service.point_over_water(lat, lon, water_bodies) is not None
if is_over_water:
reflection_gain = 3.0
try:
is_over_water = water_service.point_over_water(lat, lon, water_bodies) is not None
if is_over_water:
reflection_gain = 3.0
except Exception:
pass
# Rain
rain_loss = 0.0
@@ -770,26 +971,6 @@ class CoverageService:
indoor_loss=indoor_loss, atmospheric_loss=atmo_loss,
)
def _okumura_hata(
self,
distance: float,
frequency: float,
tx_height: float,
rx_height: float
) -> float:
"""Okumura-Hata path loss model (urban). Returns path loss in dB."""
d_km = distance / 1000
if d_km < 0.1:
d_km = 0.1
a_hm = (1.1 * np.log10(frequency) - 0.7) * rx_height - (1.56 * np.log10(frequency) - 0.8)
L = (69.55 + 26.16 * np.log10(frequency) - 13.82 * np.log10(tx_height) - a_hm +
(44.9 - 6.55 * np.log10(tx_height)) * np.log10(d_km))
return L
def _antenna_pattern_loss(
self,
site_lat: float, site_lon: float,
@@ -831,20 +1012,8 @@ class CoverageService:
return (bearing + 360) % 360
def _diffraction_loss(self, clearance: float, frequency: float) -> float:
"""Knife-edge diffraction loss. Returns additional loss in dB."""
if clearance >= 0:
return 0.0
v = abs(clearance) / 10
if v <= 0:
loss = 0
elif v < 2.4:
loss = 6.02 + 9.11 * v - 1.27 * v**2
else:
loss = 13.0 + 20 * np.log10(v)
return min(loss, 40)
"""Knife-edge diffraction loss using ITU-R P.526 model."""
return _DIFFRACTION_MODEL.calculate_clearance_loss(clearance, frequency)
# Singleton