git.saurik.com Git - apple/xnu.git/blob - osfmk/kern/remote

1 /*

3 *

4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

5 *

6 * This file contains Original Code and/or Modifications of Original Code

7 * as defined in and that are subject to the Apple Public Source License

8 * Version 2.0 (the 'License'). You may not use this file except in

9 * compliance with the License. The rights granted to you under the License

10 * may not be used to create, or enable the creation or redistribution of,

11 * unlawful or unlicensed copies of an Apple operating system, or to

12 * circumvent, violate, or enable the circumvention or violation of, any

13 * terms of an Apple operating system software license agreement.

14 *

15 * Please obtain a copy of the License at

16 * http://www.opensource.apple.com/apsl/ and read it before using this file.

17 *

18 * The Original Code and all software distributed under the License are

19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER

20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,

21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,

22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.

23 * Please see the License for the specific language governing rights and

24 * limitations under the License.

25 *

26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@

27 */

28 #include <mach/mach_time.h>

29 #include <mach/clock_types.h>

30 #include <kern/misc_protos.h>

31 #include <kern/clock.h>

32 #include <kern/remote_time.h>

33 #include <kern/spl.h>

34 #include <kern/locks.h>

35 #include <sys/kdebug.h>

36 #include <machine/machine_routines.h>

37 #include <kern/assert.h>

38 #include <kern/kern_types.h>

39 #include <kern/thread.h>

40 #include <machine/commpage.h>

41 #include <machine/atomic.h>

 LCK_GRP_DECLARE(bt_lck_grp, "bridge timestamp");

44 LCK_SPIN_DECLARE(bt_spin_lock, &bt_lck_grp);

45 LCK_SPIN_DECLARE(bt_ts_conversion_lock, &bt_lck_grp);

46 LCK_SPIN_DECLARE(bt_maintenance_lock, &bt_lck_grp);

48 #if CONFIG_MACH_BRIDGE_SEND_TIME

50 uint32_t bt_enable_flag = 0;

51 _Atomic uint32_t bt_init_flag = 0;

 void mach_bridge_timer_maintenance(void);

 uint32_t mach_bridge_timer_enable(uint32_t new_value, int change);

56 /*

57 * When CONFIG_MACH_BRIDGE_SEND_TIME is defined, it is expected

58 * that a machine-specific timestamp sending routine such as

59 * void mach_bridge_send_timestamp(uint64_t); has also been defined.

60 */

 extern void mach_bridge_send_timestamp(uint64_t);

63 void

64 mach_bridge_timer_maintenance(void)

65 {

         if (!os_atomic_load(&bt_init_flag, acquire)) {

67 return;

68 }

70 lck_spin_lock(&bt_maintenance_lock);

71 if (!bt_enable_flag) {

72 goto done;

73 }

74 mach_bridge_send_timestamp(0);

76 done:

77 lck_spin_unlock(&bt_maintenance_lock);

78 }

80 /*

81 * If change = 0, return the current value of bridge_timer_enable

82 * If change = 1, update bridge_timer_enable and return the updated

83 * value

84 */

85 uint32_t

 mach_bridge_timer_enable(uint32_t new_value, int change)

87 {

88 uint32_t current_value = 0;

         assert(os_atomic_load(&bt_init_flag, relaxed));

90 lck_spin_lock(&bt_maintenance_lock);

91 if (change) {

92 bt_enable_flag = new_value;

93 }

94 current_value = bt_enable_flag;

95 lck_spin_unlock(&bt_maintenance_lock);

96 return current_value;

97 }

99 #endif /* CONFIG_MACH_BRIDGE_SEND_TIME */

100

101 #if CONFIG_MACH_BRIDGE_RECV_TIME

102 #include <machine/machine_remote_time.h>

103

104 /*

105 * functions used by machine-specific code

106 * that implements CONFIG_MACH_BRIDGE_RECV_TIME

107 */

 void mach_bridge_add_timestamp(uint64_t remote_timestamp, uint64_t local_timestamp);

 void bt_calibration_thread_start(void);

 void bt_params_add(struct bt_params *params);

111

112 /* function called by sysctl */

 struct bt_params bt_params_get_latest(void);

114

115 /*

116 * Platform specific bridge time receiving interface.

117 * These variables should be exported by the platform specific time receiving code.

118 */

119 extern _Atomic uint32_t bt_init_flag;

120

121 static uint64_t received_local_timestamp = 0;

122 static uint64_t received_remote_timestamp = 0;

123 /*

124 * Buffer the previous timestamp pairs and rate

125 * It is protected by the bt_ts_conversion_lock

126 */

127 #define BT_PARAMS_COUNT 10

128 static struct bt_params bt_params_hist[BT_PARAMS_COUNT] = {};

129 static int bt_params_idx = -1;

130

131 void

 bt_params_add(struct bt_params *params)

133 {

134 lck_spin_assert(&bt_ts_conversion_lock, LCK_ASSERT_OWNED);

135

         bt_params_idx = (bt_params_idx + 1) % BT_PARAMS_COUNT;

137 bt_params_hist[bt_params_idx] = *params;

138 }

139

140 #if defined(XNU_TARGET_OS_BRIDGE)

141 static inline struct bt_params*

142 bt_params_find(uint64_t local_ts)

143 {

144 lck_spin_assert(&bt_ts_conversion_lock, LCK_ASSERT_OWNED);

145

146 int idx = bt_params_idx;

         if (idx < 0) {

148 return NULL;

149 }

150 do {

                 if (local_ts >= bt_params_hist[idx].base_local_ts) {

152 return &bt_params_hist[idx];

153 }

                 if (--idx < 0) {

155 idx = BT_PARAMS_COUNT - 1;

156 }

         } while (idx != bt_params_idx);

158

159 return NULL;

160 }

161 #endif /* defined(XNU_TARGET_OS_BRIDGE) */

162

163 static inline struct bt_params

164 bt_params_get_latest_locked(void)

165 {

166 lck_spin_assert(&bt_ts_conversion_lock, LCK_ASSERT_OWNED);

167

168 struct bt_params latest_params = {};

         if (bt_params_idx >= 0) {

170 latest_params = bt_params_hist[bt_params_idx];

171 }

172

173 return latest_params;

174 }

175

176 struct bt_params

177 bt_params_get_latest(void)

178 {

179 struct bt_params latest_params = {};

180

181 /* Check if ts_converison_lock has been initialized */

         if (os_atomic_load(&bt_init_flag, acquire)) {

183 lck_spin_lock(&bt_ts_conversion_lock);

184 latest_params = bt_params_get_latest_locked();

185 lck_spin_unlock(&bt_ts_conversion_lock);

186 }

187 return latest_params;

188 }

189

190 /*

191 * Conditions: bt_spin_lock held and called from primary interrupt context

192 */

193 void

 mach_bridge_add_timestamp(uint64_t remote_timestamp, uint64_t local_timestamp)

195 {

196 lck_spin_assert(&bt_spin_lock, LCK_ASSERT_OWNED);

197

198 /* sleep/wake might return the same mach_absolute_time as the previous timestamp pair */

199 if ((received_local_timestamp == local_timestamp) ||

200 (received_remote_timestamp == remote_timestamp)) {

201 return;

202 }

203

204 received_local_timestamp = local_timestamp;

205 received_remote_timestamp = remote_timestamp;

206 thread_wakeup((event_t)bt_params_hist);

207 }

208

209 static double

 mach_bridge_compute_rate(uint64_t new_local_ts, uint64_t new_remote_ts,

211 uint64_t old_local_ts, uint64_t old_remote_ts)

212 {

         int64_t rdiff = (int64_t)new_remote_ts - (int64_t)old_remote_ts;

         int64_t ldiff = (int64_t)new_local_ts - (int64_t)old_local_ts;

         double calc_rate = ((double)rdiff) / (double)ldiff;

216 return calc_rate;

217 }

218

219 #define MAX_RECALCULATE_COUNT 8

220 #define CUMULATIVE_RATE_DECAY_CONSTANT 0.01

221 #define CUMULATIVE_RATE_WEIGHT 0.99

222 #define INITIAL_RATE 1.0

223 #define MIN_INITIAL_SAMPLE_COUNT 10

224 #define MAX_INITIAL_SAMPLE_COUNT 50

225 #define MAX_SKIP_RESET_COUNT 2

226 #define MIN_LOCAL_TS_DISTANCE_NS 100000000 /* 100 ms */

227 #define MAX_LOCAL_TS_DISTANCE_NS 350000000 /* 350 ms */

228 #define TS_PAIR_MISMATCH_THRESHOLD_NS 50000000 /* 50 ms */

229 #define MAX_TS_PAIR_MISMATCHES 5

230 #define MAX_TS_PAIR_MISMATCH_RESET_COUNT 3

231 #define MIN_OBSERVED_RATE 0.8

232 #define MAX_OBSERVED_RATE 1.2

233

234 static void

235 bt_calibration_thread(void)

236 {

         static uint64_t prev_local_ts = 0, prev_remote_ts = 0, curr_local_ts = 0, curr_remote_ts = 0;

         static uint64_t prev_received_local_ts = 0, prev_received_remote_ts = 0;

239 static double cumulative_rate = INITIAL_RATE;

240 static uint32_t initial_sample_count = 1;

241 static uint32_t max_initial_sample_count = MAX_INITIAL_SAMPLE_COUNT;

242 static uint32_t skip_reset_count = MAX_SKIP_RESET_COUNT;

243 int recalculate_count = 1;

244 static bool reset = false;

245 bool sleep = false;

246 static bool skip_rcv_ts = false;

247 static uint64_t ts_pair_mismatch = 0;

248 static uint32_t ts_pair_mismatch_reset_count = 0;

249 spl_t s = splsched();

250 lck_spin_lock(&bt_spin_lock);

251 if (!received_remote_timestamp) {

                 if (PE_parse_boot_argn("rt_ini_count", &max_initial_sample_count,

                     sizeof(uint32_t)) == TRUE) {

254 if (max_initial_sample_count < MIN_INITIAL_SAMPLE_COUNT) {

255 max_initial_sample_count = MIN_INITIAL_SAMPLE_COUNT;

256 }

257 }

258 /* Nothing to do the first time */

259 goto block;

260 }

261 /*

262 * The values in bt_params are recalculated every time a new timestamp

263 * pair is received. Firstly, both timestamps are converted to nanoseconds.

264 * The current and previous timestamp pairs are used to compute the

265 * observed_rate of the two clocks w.r.t each other. For the first

266 * MIN_INITIAL_SAMPLE_COUNT number of pairs, the cumulative_rate is a simple

267 * average of the observed_rate. For the later pairs, the cumulative_rate

268 * is updated using exponential moving average of the observed_rate.

269 * The current and bt_params' base timestamp pairs are used to compute

270 * the rate_from_base. This value ensures that the bt_params base

271 * timestamp pair curve doesn't stay parallel to the observed timestamp

272 * pair curve, rather moves in the direction of the observed timestamp curve.

273 * The bt_params.rate is computed as a weighted average of the cumulative_rate

274 * and the rate_from_base. For each current local timestamp, the remote_time

275 * is predicted using the previous values of bt_params. After computing the new

276 * bt_params.rate, bt_params.base_remote_time is set to this predicted value

277 * and bt_params.base_local_time is set to the current local timestamp.

278 */

279 recalculate:

         assertf(recalculate_count <= MAX_RECALCULATE_COUNT, "bt_caliberation_thread: recalculate \

281 invocation exceeds MAX_RECALCULATE_COUNT");

282

         if ((received_remote_timestamp == BT_RESET_SENTINEL_TS) || (received_remote_timestamp == BT_WAKE_SENTINEL_TS)) {

                 KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_RESET_TS), received_local_timestamp, received_remote_timestamp, 1);

285 reset = true;

286 skip_reset_count = MAX_SKIP_RESET_COUNT;

287 ts_pair_mismatch_reset_count = 0;

288 goto block;

         } else if (received_remote_timestamp == BT_SLEEP_SENTINEL_TS) {

290 sleep = true;

291 } else if (!received_local_timestamp) {

292 /* If the local timestamp isn't accurately captured, the received value will be ignored */

293 skip_rcv_ts = true;

294 goto block;

295 }

296

297 /* Keep a copy of the prev timestamps to compute distance */

298 prev_received_local_ts = curr_local_ts;

299 prev_received_remote_ts = curr_remote_ts;

300

301 uint64_t curr_local_abs = received_local_timestamp;

302 absolutetime_to_nanoseconds(curr_local_abs, &curr_local_ts);

303 curr_remote_ts = received_remote_timestamp;

304

305 /* Prevent unusual rate changes caused by delayed timestamps */

306 uint64_t local_diff = curr_local_ts - prev_received_local_ts;

         if (!(reset || sleep) && ((local_diff < MIN_LOCAL_TS_DISTANCE_NS) ||

308 (!skip_rcv_ts && (local_diff > MAX_LOCAL_TS_DISTANCE_NS)))) {

309 /* Skip the current timestamp */

                 KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_SKIP_TS), curr_local_ts, curr_remote_ts,

311 prev_received_local_ts);

312 goto block;

313 } else {

314 skip_rcv_ts = false;

315 /* Use the prev copy of timestamps only if the distance is acceptable */

316 prev_local_ts = prev_received_local_ts;

317 prev_remote_ts = prev_received_remote_ts;

318 }

319 lck_spin_unlock(&bt_spin_lock);

320 splx(s);

321

322 struct bt_params bt_params = {};

323

324 lck_spin_lock(&bt_ts_conversion_lock);

325 if (reset) {

                 if (skip_reset_count > 0) {

                         KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_SKIP_TS), curr_local_ts, curr_remote_ts,

328 prev_local_ts, skip_reset_count);

329 skip_reset_count--;

330 goto skip_reset;

331 }

332 bt_params.base_local_ts = curr_local_ts;

333 bt_params.base_remote_ts = curr_remote_ts;

334 bt_params.rate = cumulative_rate;

335 prev_local_ts = 0;

336 prev_remote_ts = 0;

337 ts_pair_mismatch = 0;

338 initial_sample_count = 1;

339 reset = false;

                 KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_RESET_TS), curr_local_ts, curr_remote_ts, 2);

341 } else if (sleep) {

                 absolutetime_to_nanoseconds(mach_absolute_time(), &bt_params.base_local_ts);

343 bt_params.base_remote_ts = 0;

344 bt_params.rate = 0;

345 sleep = false;

346 } else {

347 struct bt_params bt_params_snapshot = {};

                 if (bt_params_idx >= 0) {

349 bt_params_snapshot = bt_params_hist[bt_params_idx];

350 }

351 lck_spin_unlock(&bt_ts_conversion_lock);

                 if (bt_params_snapshot.rate == 0.0) {

353 /*

354 * The rate should never be 0 because we always expect a reset/wake

355 * sentinel after sleep, followed by valid timestamp pair data that

356 * will be handled by the reset clause (above). However, we should

357 * not rely on a paired version of the remote OS - we could actually

358 * be running a completely different OS! Treat a timestamp after

359 * a sleep as a reset condition.

360 */

361 reset = true;

362 skip_reset_count = MAX_SKIP_RESET_COUNT;

363 ts_pair_mismatch_reset_count = 0;

                         KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_RESET_TS), curr_local_ts, curr_remote_ts, 3);

365 s = splsched();

366 lck_spin_lock(&bt_spin_lock);

367 goto block;

368 }

369

370 /* Check if the predicted remote timestamp is within the expected current remote timestamp range */

                 uint64_t pred_remote_ts = mach_bridge_compute_timestamp(curr_local_ts, &bt_params_snapshot);

372 uint64_t diff = 0;

373 if (initial_sample_count >= max_initial_sample_count) {

374 if (pred_remote_ts > curr_remote_ts) {

375 diff = pred_remote_ts - curr_remote_ts;

376 } else {

377 diff = curr_remote_ts - pred_remote_ts;

378 }

379 if (diff > TS_PAIR_MISMATCH_THRESHOLD_NS) {

380 ts_pair_mismatch++;

                                 KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_TS_MISMATCH), curr_local_ts,

382 curr_remote_ts, pred_remote_ts, ts_pair_mismatch);

383 } else {

384 ts_pair_mismatch = 0;

385 }

386 if (ts_pair_mismatch > MAX_TS_PAIR_MISMATCHES) {

387 #if (DEVELOPMENT || DEBUG)

388 if (ts_pair_mismatch_reset_count == MAX_TS_PAIR_MISMATCH_RESET_COUNT) {

389 panic("remote_time: timestamp pair mismatch exceeded limit");

390 }

391 #endif /* (DEVELOPMENT || DEBUG) */

392 reset = true;

393 ts_pair_mismatch_reset_count++;

                                 KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_RESET_TS), curr_local_ts, curr_remote_ts, 4);

395 s = splsched();

396 lck_spin_lock(&bt_spin_lock);

397 goto block;

398 }

399 }

400 double observed_rate, rate_from_base, new_rate;

                 observed_rate = mach_bridge_compute_rate(curr_local_ts, curr_remote_ts, prev_local_ts, prev_remote_ts);

402 /* Log bad observed rates and skip the timestamp pair */

                 if ((observed_rate < MIN_OBSERVED_RATE) || (observed_rate > MAX_OBSERVED_RATE)) {

                         KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_OBSV_RATE), *(uint64_t *)((void *)&observed_rate));

                         ts_pair_mismatch = ts_pair_mismatch > 0 ? (ts_pair_mismatch - 1) : 0;

406 s = splsched();

407 lck_spin_lock(&bt_spin_lock);

408 goto block;

409 }

410 if (initial_sample_count <= MIN_INITIAL_SAMPLE_COUNT) {

411 initial_sample_count++;

                         cumulative_rate = cumulative_rate + (observed_rate - cumulative_rate) / initial_sample_count;

413 } else {

414 if (initial_sample_count < max_initial_sample_count) {

415 initial_sample_count++;

416 }

                         cumulative_rate = cumulative_rate + CUMULATIVE_RATE_DECAY_CONSTANT * (observed_rate - cumulative_rate);

418 }

                 rate_from_base = mach_bridge_compute_rate(curr_local_ts, curr_remote_ts, bt_params_snapshot.base_local_ts,

420 bt_params_snapshot.base_remote_ts);

                 new_rate = CUMULATIVE_RATE_WEIGHT * cumulative_rate + (1 - CUMULATIVE_RATE_WEIGHT) * rate_from_base;

422 /*

423 * Acquire the lock first to ensure that bt_params.base_local_ts is always

424 * greater than the last value of now captured by mach_bridge_remote_time.

425 * This ensures that we always use the same parameters to compute remote

426 * timestamp for a given local timestamp.

427 */

428 lck_spin_lock(&bt_ts_conversion_lock);

                 absolutetime_to_nanoseconds(mach_absolute_time(), &bt_params.base_local_ts);

                 bt_params.base_remote_ts = mach_bridge_compute_timestamp(bt_params.base_local_ts, &bt_params_snapshot);

431 bt_params.rate = new_rate;

432 }

433 bt_params_add(&bt_params);

         commpage_set_remotetime_params(bt_params.rate, bt_params.base_local_ts, bt_params.base_remote_ts);

         KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_TS_PARAMS), bt_params.base_local_ts,

             bt_params.base_remote_ts, *(uint64_t *)((void *)&bt_params.rate));

437

438 skip_reset:

439 lck_spin_unlock(&bt_ts_conversion_lock);

440

441 s = splsched();

442 lck_spin_lock(&bt_spin_lock);

443 /* Check if a new timestamp pair was received */

444 if (received_local_timestamp != curr_local_abs) {

445 recalculate_count++;

446 goto recalculate;

447 }

448 block:

         assert_wait((event_t)bt_params_hist, THREAD_UNINT);

450 lck_spin_unlock(&bt_spin_lock);

451 splx(s);

452 thread_block((thread_continue_t)bt_calibration_thread);

453 }

454

455 void

456 bt_calibration_thread_start(void)

457 {

458 thread_t thread;

         kern_return_t result = kernel_thread_start_priority((thread_continue_t)bt_calibration_thread,

460 NULL, BASEPRI_KERNEL, &thread);

461 if (result != KERN_SUCCESS) {

462 panic("mach_bridge_add_timestamp: thread_timestamp_calibration");

463 }

464 thread_deallocate(thread);

465 }

466

467 #endif /* CONFIG_MACH_BRIDGE_RECV_TIME */

468

469 /**

470 * mach_bridge_remote_time

471 *

472 * This function is used to predict the remote CPU's clock time, given

473 * the local time.

474 *

475 * If local_timestamp = 0, then the remote_timestamp is calculated

476 * corresponding to the current mach_absolute_time.

477 *

478 * If XNU_TARGET_OS_BRIDGE is defined, then monotonicity of

479 * predicted time is guaranteed only for recent local_timestamp values

480 * lesser than the current mach_absolute_time upto 1 second.

481 *

482 * If CONFIG_MACH_BRIDGE_SEND_TIME is true, then the function is compiled

483 * for the remote CPU. If CONFIG_MACH_BRIDGE_RECV_TIME is true, then the

484 * the function is compiled for the local CPU. Both config options cannot

485 * be true simultaneously.

486 */

487 uint64_t

488 mach_bridge_remote_time(uint64_t local_timestamp)

489 {

490 #if defined(CONFIG_MACH_BRIDGE_SEND_TIME)

491 #if !defined(CONFIG_MACH_BRIDGE_RECV_TIME)

492 /* only send side of the bridge is defined: no translation needed */

493 if (!local_timestamp) {

494 return mach_absolute_time();

495 }

496 return 0;

497 #else

498 #error "You cannot define both sides of the bridge!"

499 #endif /* !defined(CONFIG_MACH_BRIDGE_RECV_TIME) */

500 #else

501 #if !defined(CONFIG_MACH_BRIDGE_RECV_TIME)

502 /* neither the send or receive side of the bridge is defined: echo the input */

503 return local_timestamp;

504 #else

         if (!os_atomic_load(&bt_init_flag, acquire)) {

506 return 0;

507 }

508

509 uint64_t remote_timestamp = 0;

510

511 lck_spin_lock(&bt_ts_conversion_lock);

512 uint64_t now = mach_absolute_time();

513 if (!local_timestamp) {

514 local_timestamp = now;

515 }

516 #if defined(XNU_TARGET_OS_BRIDGE)

517 uint64_t local_timestamp_ns = 0;

518 if (local_timestamp < now) {

519 absolutetime_to_nanoseconds(local_timestamp, &local_timestamp_ns);

                 struct bt_params *params = bt_params_find(local_timestamp_ns);

                 remote_timestamp = mach_bridge_compute_timestamp(local_timestamp_ns, params);

522 }

523 #else

524 struct bt_params params = bt_params_get_latest_locked();

         remote_timestamp = mach_bridge_compute_timestamp(local_timestamp, &params);

526 #endif /* defined(XNU_TARGET_OS_BRIDGE) */

527 lck_spin_unlock(&bt_ts_conversion_lock);

         KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_REMOTE_TIME), local_timestamp, remote_timestamp, now);

529

530 return remote_timestamp;

531 #endif /* !defined(CONFIG_MACH_BRIDGE_RECV_TIME) */

532 #endif /* defined(CONFIG_MACH_BRIDGE_SEND_TIME) */

533 }