diff options
| author | DanConwayDev <DanConwayDev@protonmail.com> | 2025-12-11 11:57:36 +0000 |
|---|---|---|
| committer | DanConwayDev <DanConwayDev@protonmail.com> | 2025-12-11 11:57:36 +0000 |
| commit | 4941490233a728bc7c64fa80a53d15f772a1219f (patch) | |
| tree | 7fc1bbf6114deb29b5a736b467abf785ea915f02 /src/sync/health.rs | |
| parent | 6cd7535f2d5f65477ef11b17a4661745ec3a2881 (diff) | |
sync: add sync_base_backoff_secs config for better testing
Diffstat (limited to 'src/sync/health.rs')
| -rw-r--r-- | src/sync/health.rs | 58 |
1 files changed, 42 insertions, 16 deletions
diff --git a/src/sync/health.rs b/src/sync/health.rs index 51bd5ae..f9a5f3a 100644 --- a/src/sync/health.rs +++ b/src/sync/health.rs | |||
| @@ -27,8 +27,8 @@ const DEAD_RETRY_INTERVAL_HOURS: u64 = 24; | |||
| 27 | /// Default maximum backoff duration in seconds (1 hour) | 27 | /// Default maximum backoff duration in seconds (1 hour) |
| 28 | const DEFAULT_MAX_BACKOFF_SECS: u64 = 3600; | 28 | const DEFAULT_MAX_BACKOFF_SECS: u64 = 3600; |
| 29 | 29 | ||
| 30 | /// Base backoff duration in seconds | 30 | /// Default base backoff duration in seconds |
| 31 | const BASE_BACKOFF_SECS: u64 = 5; | 31 | const DEFAULT_BASE_BACKOFF_SECS: u64 = 5; |
| 32 | 32 | ||
| 33 | /// Health state of a relay connection | 33 | /// Health state of a relay connection |
| 34 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] | 34 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| @@ -93,6 +93,7 @@ impl RelayHealth { | |||
| 93 | pub struct RelayHealthTracker { | 93 | pub struct RelayHealthTracker { |
| 94 | health: DashMap<String, RelayHealth>, | 94 | health: DashMap<String, RelayHealth>, |
| 95 | max_backoff_secs: u64, | 95 | max_backoff_secs: u64, |
| 96 | base_backoff_secs: u64, | ||
| 96 | } | 97 | } |
| 97 | 98 | ||
| 98 | impl RelayHealthTracker { | 99 | impl RelayHealthTracker { |
| @@ -101,6 +102,7 @@ impl RelayHealthTracker { | |||
| 101 | Self { | 102 | Self { |
| 102 | health: DashMap::new(), | 103 | health: DashMap::new(), |
| 103 | max_backoff_secs: config.sync_max_backoff_secs, | 104 | max_backoff_secs: config.sync_max_backoff_secs, |
| 105 | base_backoff_secs: config.sync_base_backoff_secs, | ||
| 104 | } | 106 | } |
| 105 | } | 107 | } |
| 106 | 108 | ||
| @@ -109,6 +111,7 @@ impl RelayHealthTracker { | |||
| 109 | Self { | 111 | Self { |
| 110 | health: DashMap::new(), | 112 | health: DashMap::new(), |
| 111 | max_backoff_secs: DEFAULT_MAX_BACKOFF_SECS, | 113 | max_backoff_secs: DEFAULT_MAX_BACKOFF_SECS, |
| 114 | base_backoff_secs: DEFAULT_BASE_BACKOFF_SECS, | ||
| 112 | } | 115 | } |
| 113 | } | 116 | } |
| 114 | 117 | ||
| @@ -117,9 +120,18 @@ impl RelayHealthTracker { | |||
| 117 | Self { | 120 | Self { |
| 118 | health: DashMap::new(), | 121 | health: DashMap::new(), |
| 119 | max_backoff_secs, | 122 | max_backoff_secs, |
| 123 | base_backoff_secs: DEFAULT_BASE_BACKOFF_SECS, | ||
| 120 | } | 124 | } |
| 121 | } | 125 | } |
| 122 | 126 | ||
| 127 | /// Get the base backoff duration in seconds | ||
| 128 | /// | ||
| 129 | /// This is used by SyncManager to set connection timeout | ||
| 130 | /// (connection timeout should not exceed base backoff) | ||
| 131 | pub fn base_backoff_secs(&self) -> u64 { | ||
| 132 | self.base_backoff_secs | ||
| 133 | } | ||
| 134 | |||
| 123 | /// Record a successful connection to a relay | 135 | /// Record a successful connection to a relay |
| 124 | /// | 136 | /// |
| 125 | /// Resets the relay to Healthy state and clears failure counters. | 137 | /// Resets the relay to Healthy state and clears failure counters. |
| @@ -188,6 +200,7 @@ impl RelayHealthTracker { | |||
| 188 | health.state = HealthState::Degraded; | 200 | health.state = HealthState::Degraded; |
| 189 | let backoff = Self::get_backoff_duration( | 201 | let backoff = Self::get_backoff_duration( |
| 190 | health.consecutive_failures, | 202 | health.consecutive_failures, |
| 203 | self.base_backoff_secs, | ||
| 191 | self.max_backoff_secs, | 204 | self.max_backoff_secs, |
| 192 | ); | 205 | ); |
| 193 | health.next_retry_at = Some(now + backoff); | 206 | health.next_retry_at = Some(now + backoff); |
| @@ -277,9 +290,18 @@ impl RelayHealthTracker { | |||
| 277 | 290 | ||
| 278 | /// Calculate the backoff duration based on failure count | 291 | /// Calculate the backoff duration based on failure count |
| 279 | /// | 292 | /// |
| 280 | /// Uses exponential backoff: base * 2^failures, capped at max_backoff | 293 | /// Uses exponential backoff: base * 2^(failures-1), capped at max_backoff |
| 281 | pub fn get_backoff_duration(consecutive_failures: u32, max_backoff_secs: u64) -> Duration { | 294 | /// |
| 282 | let backoff_secs = BASE_BACKOFF_SECS | 295 | /// # Arguments |
| 296 | /// * `consecutive_failures` - Number of consecutive failures (1 = first failure) | ||
| 297 | /// * `base_backoff_secs` - Base backoff time in seconds | ||
| 298 | /// * `max_backoff_secs` - Maximum backoff cap in seconds | ||
| 299 | pub fn get_backoff_duration( | ||
| 300 | consecutive_failures: u32, | ||
| 301 | base_backoff_secs: u64, | ||
| 302 | max_backoff_secs: u64, | ||
| 303 | ) -> Duration { | ||
| 304 | let backoff_secs = base_backoff_secs | ||
| 283 | .saturating_mul(2u64.saturating_pow(consecutive_failures.saturating_sub(1))); | 305 | .saturating_mul(2u64.saturating_pow(consecutive_failures.saturating_sub(1))); |
| 284 | Duration::from_secs(backoff_secs.min(max_backoff_secs)) | 306 | Duration::from_secs(backoff_secs.min(max_backoff_secs)) |
| 285 | } | 307 | } |
| @@ -345,39 +367,43 @@ mod tests { | |||
| 345 | 367 | ||
| 346 | #[test] | 368 | #[test] |
| 347 | fn test_backoff_increases_exponentially() { | 369 | fn test_backoff_increases_exponentially() { |
| 348 | // failure 1: 5s | 370 | let base = DEFAULT_BASE_BACKOFF_SECS; // 5 seconds |
| 371 | let max = 3600u64; | ||
| 372 | |||
| 373 | // failure 1: 5s (base * 2^0 = 5) | ||
| 349 | assert_eq!( | 374 | assert_eq!( |
| 350 | RelayHealthTracker::get_backoff_duration(1, 3600), | 375 | RelayHealthTracker::get_backoff_duration(1, base, max), |
| 351 | Duration::from_secs(5) | 376 | Duration::from_secs(5) |
| 352 | ); | 377 | ); |
| 353 | // failure 2: 10s | 378 | // failure 2: 10s (base * 2^1 = 10) |
| 354 | assert_eq!( | 379 | assert_eq!( |
| 355 | RelayHealthTracker::get_backoff_duration(2, 3600), | 380 | RelayHealthTracker::get_backoff_duration(2, base, max), |
| 356 | Duration::from_secs(10) | 381 | Duration::from_secs(10) |
| 357 | ); | 382 | ); |
| 358 | // failure 3: 20s | 383 | // failure 3: 20s (base * 2^2 = 20) |
| 359 | assert_eq!( | 384 | assert_eq!( |
| 360 | RelayHealthTracker::get_backoff_duration(3, 3600), | 385 | RelayHealthTracker::get_backoff_duration(3, base, max), |
| 361 | Duration::from_secs(20) | 386 | Duration::from_secs(20) |
| 362 | ); | 387 | ); |
| 363 | // failure 4: 40s | 388 | // failure 4: 40s (base * 2^3 = 40) |
| 364 | assert_eq!( | 389 | assert_eq!( |
| 365 | RelayHealthTracker::get_backoff_duration(4, 3600), | 390 | RelayHealthTracker::get_backoff_duration(4, base, max), |
| 366 | Duration::from_secs(40) | 391 | Duration::from_secs(40) |
| 367 | ); | 392 | ); |
| 368 | // failure 5: 80s | 393 | // failure 5: 80s (base * 2^4 = 80) |
| 369 | assert_eq!( | 394 | assert_eq!( |
| 370 | RelayHealthTracker::get_backoff_duration(5, 3600), | 395 | RelayHealthTracker::get_backoff_duration(5, base, max), |
| 371 | Duration::from_secs(80) | 396 | Duration::from_secs(80) |
| 372 | ); | 397 | ); |
| 373 | } | 398 | } |
| 374 | 399 | ||
| 375 | #[test] | 400 | #[test] |
| 376 | fn test_backoff_capped_at_max() { | 401 | fn test_backoff_capped_at_max() { |
| 402 | let base = DEFAULT_BASE_BACKOFF_SECS; | ||
| 377 | let max_backoff = 3600u64; | 403 | let max_backoff = 3600u64; |
| 378 | // After many failures, should cap at max_backoff (1 hour) | 404 | // After many failures, should cap at max_backoff (1 hour) |
| 379 | assert_eq!( | 405 | assert_eq!( |
| 380 | RelayHealthTracker::get_backoff_duration(20, max_backoff), | 406 | RelayHealthTracker::get_backoff_duration(20, base, max_backoff), |
| 381 | Duration::from_secs(max_backoff) | 407 | Duration::from_secs(max_backoff) |
| 382 | ); | 408 | ); |
| 383 | } | 409 | } |