Print this page
5042 stop using deprecated atomic functions
Split |
Close |
Expand all |
Collapse all |
--- old/usr/src/uts/sun4/os/machdep.c
+++ new/usr/src/uts/sun4/os/machdep.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
23 23 */
24 24
25 25 #include <sys/types.h>
26 26 #include <sys/kstat.h>
27 27 #include <sys/param.h>
28 28 #include <sys/stack.h>
29 29 #include <sys/regset.h>
30 30 #include <sys/thread.h>
31 31 #include <sys/proc.h>
32 32 #include <sys/procfs_isa.h>
33 33 #include <sys/kmem.h>
34 34 #include <sys/cpuvar.h>
35 35 #include <sys/systm.h>
36 36 #include <sys/machpcb.h>
37 37 #include <sys/machasi.h>
38 38 #include <sys/vis.h>
39 39 #include <sys/fpu/fpusystm.h>
40 40 #include <sys/cpu_module.h>
41 41 #include <sys/privregs.h>
42 42 #include <sys/archsystm.h>
43 43 #include <sys/atomic.h>
44 44 #include <sys/cmn_err.h>
45 45 #include <sys/time.h>
46 46 #include <sys/clock.h>
47 47 #include <sys/cmp.h>
48 48 #include <sys/platform_module.h>
49 49 #include <sys/bl.h>
50 50 #include <sys/nvpair.h>
51 51 #include <sys/kdi_impl.h>
52 52 #include <sys/machsystm.h>
53 53 #include <sys/sysmacros.h>
54 54 #include <sys/promif.h>
55 55 #include <sys/pool_pset.h>
56 56 #include <sys/mem.h>
57 57 #include <sys/dumphdr.h>
58 58 #include <vm/seg_kmem.h>
59 59 #include <sys/hold_page.h>
60 60 #include <sys/cpu.h>
61 61 #include <sys/ivintr.h>
62 62 #include <sys/clock_impl.h>
63 63 #include <sys/machclock.h>
64 64
65 65 int maxphys = MMU_PAGESIZE * 16; /* 128k */
66 66 int klustsize = MMU_PAGESIZE * 16; /* 128k */
67 67
68 68 /*
69 69 * Initialize kernel thread's stack.
70 70 */
71 71 caddr_t
72 72 thread_stk_init(caddr_t stk)
73 73 {
74 74 kfpu_t *fp;
75 75 ulong_t align;
76 76
77 77 /* allocate extra space for floating point state */
78 78 stk -= SA(sizeof (kfpu_t) + GSR_SIZE);
79 79 align = (uintptr_t)stk & 0x3f;
80 80 stk -= align; /* force v9_fpu to be 16 byte aligned */
81 81 fp = (kfpu_t *)stk;
82 82 fp->fpu_fprs = 0;
83 83
84 84 stk -= SA(MINFRAME);
85 85 return (stk);
86 86 }
87 87
88 88 #define WIN32_SIZE (MAXWIN * sizeof (struct rwindow32))
89 89 #define WIN64_SIZE (MAXWIN * sizeof (struct rwindow64))
90 90
91 91 kmem_cache_t *wbuf32_cache;
92 92 kmem_cache_t *wbuf64_cache;
93 93
94 94 void
95 95 lwp_stk_cache_init(void)
96 96 {
97 97 /*
98 98 * Window buffers are allocated from the static arena
99 99 * because they are accessed at TL>0. We also must use
100 100 * KMC_NOHASH to prevent them from straddling page
101 101 * boundaries as they are accessed by physical address.
102 102 */
103 103 wbuf32_cache = kmem_cache_create("wbuf32_cache", WIN32_SIZE,
104 104 0, NULL, NULL, NULL, NULL, static_arena, KMC_NOHASH);
105 105 wbuf64_cache = kmem_cache_create("wbuf64_cache", WIN64_SIZE,
106 106 0, NULL, NULL, NULL, NULL, static_arena, KMC_NOHASH);
107 107 }
108 108
109 109 /*
110 110 * Initialize lwp's kernel stack.
111 111 * Note that now that the floating point register save area (kfpu_t)
112 112 * has been broken out from machpcb and aligned on a 64 byte boundary so that
113 113 * we can do block load/stores to/from it, there are a couple of potential
114 114 * optimizations to save stack space. 1. The floating point register save
115 115 * area could be aligned on a 16 byte boundary, and the floating point code
116 116 * changed to (a) check the alignment and (b) use different save/restore
117 117 * macros depending upon the alignment. 2. The lwp_stk_init code below
118 118 * could be changed to calculate if less space would be wasted if machpcb
119 119 * was first instead of second. However there is a REGOFF macro used in
120 120 * locore, syscall_trap, machdep and mlsetup that assumes that the saved
121 121 * register area is a fixed distance from the %sp, and would have to be
122 122 * changed to a pointer or something...JJ said later.
123 123 */
124 124 caddr_t
125 125 lwp_stk_init(klwp_t *lwp, caddr_t stk)
126 126 {
127 127 struct machpcb *mpcb;
128 128 kfpu_t *fp;
129 129 uintptr_t aln;
130 130
131 131 stk -= SA(sizeof (kfpu_t) + GSR_SIZE);
132 132 aln = (uintptr_t)stk & 0x3F;
133 133 stk -= aln;
134 134 fp = (kfpu_t *)stk;
135 135 stk -= SA(sizeof (struct machpcb));
136 136 mpcb = (struct machpcb *)stk;
137 137 bzero(mpcb, sizeof (struct machpcb));
138 138 bzero(fp, sizeof (kfpu_t) + GSR_SIZE);
139 139 lwp->lwp_regs = (void *)&mpcb->mpcb_regs;
140 140 lwp->lwp_fpu = (void *)fp;
141 141 mpcb->mpcb_fpu = fp;
142 142 mpcb->mpcb_fpu->fpu_q = mpcb->mpcb_fpu_q;
143 143 mpcb->mpcb_thread = lwp->lwp_thread;
144 144 mpcb->mpcb_wbcnt = 0;
145 145 if (lwp->lwp_procp->p_model == DATAMODEL_ILP32) {
146 146 mpcb->mpcb_wstate = WSTATE_USER32;
147 147 mpcb->mpcb_wbuf = kmem_cache_alloc(wbuf32_cache, KM_SLEEP);
148 148 } else {
149 149 mpcb->mpcb_wstate = WSTATE_USER64;
150 150 mpcb->mpcb_wbuf = kmem_cache_alloc(wbuf64_cache, KM_SLEEP);
151 151 }
152 152 ASSERT(((uintptr_t)mpcb->mpcb_wbuf & 7) == 0);
153 153 mpcb->mpcb_wbuf_pa = va_to_pa(mpcb->mpcb_wbuf);
154 154 mpcb->mpcb_pa = va_to_pa(mpcb);
155 155 return (stk);
156 156 }
157 157
158 158 void
159 159 lwp_stk_fini(klwp_t *lwp)
160 160 {
161 161 struct machpcb *mpcb = lwptompcb(lwp);
162 162
163 163 /*
164 164 * there might be windows still in the wbuf due to unmapped
165 165 * stack, misaligned stack pointer, etc. We just free it.
166 166 */
167 167 mpcb->mpcb_wbcnt = 0;
168 168 if (mpcb->mpcb_wstate == WSTATE_USER32)
169 169 kmem_cache_free(wbuf32_cache, mpcb->mpcb_wbuf);
170 170 else
171 171 kmem_cache_free(wbuf64_cache, mpcb->mpcb_wbuf);
172 172 mpcb->mpcb_wbuf = NULL;
173 173 mpcb->mpcb_wbuf_pa = -1;
174 174 }
175 175
176 176
177 177 /*
178 178 * Copy regs from parent to child.
179 179 */
180 180 void
181 181 lwp_forkregs(klwp_t *lwp, klwp_t *clwp)
182 182 {
183 183 kthread_t *t, *pt = lwptot(lwp);
184 184 struct machpcb *mpcb = lwptompcb(clwp);
185 185 struct machpcb *pmpcb = lwptompcb(lwp);
186 186 kfpu_t *fp, *pfp = lwptofpu(lwp);
187 187 caddr_t wbuf;
188 188 uint_t wstate;
189 189
190 190 t = mpcb->mpcb_thread;
191 191 /*
192 192 * remember child's fp and wbuf since they will get erased during
193 193 * the bcopy.
194 194 */
195 195 fp = mpcb->mpcb_fpu;
196 196 wbuf = mpcb->mpcb_wbuf;
197 197 wstate = mpcb->mpcb_wstate;
198 198 /*
199 199 * Don't copy mpcb_frame since we hand-crafted it
200 200 * in thread_load().
201 201 */
202 202 bcopy(lwp->lwp_regs, clwp->lwp_regs, sizeof (struct machpcb) - REGOFF);
203 203 mpcb->mpcb_thread = t;
204 204 mpcb->mpcb_fpu = fp;
205 205 fp->fpu_q = mpcb->mpcb_fpu_q;
206 206
207 207 /*
208 208 * It is theoretically possibly for the lwp's wstate to
209 209 * be different from its value assigned in lwp_stk_init,
210 210 * since lwp_stk_init assumed the data model of the process.
211 211 * Here, we took on the data model of the cloned lwp.
212 212 */
213 213 if (mpcb->mpcb_wstate != wstate) {
214 214 if (wstate == WSTATE_USER32) {
215 215 kmem_cache_free(wbuf32_cache, wbuf);
216 216 wbuf = kmem_cache_alloc(wbuf64_cache, KM_SLEEP);
217 217 wstate = WSTATE_USER64;
218 218 } else {
219 219 kmem_cache_free(wbuf64_cache, wbuf);
220 220 wbuf = kmem_cache_alloc(wbuf32_cache, KM_SLEEP);
221 221 wstate = WSTATE_USER32;
222 222 }
223 223 }
224 224
225 225 mpcb->mpcb_pa = va_to_pa(mpcb);
226 226 mpcb->mpcb_wbuf = wbuf;
227 227 mpcb->mpcb_wbuf_pa = va_to_pa(wbuf);
228 228
229 229 ASSERT(mpcb->mpcb_wstate == wstate);
230 230
231 231 if (mpcb->mpcb_wbcnt != 0) {
232 232 bcopy(pmpcb->mpcb_wbuf, mpcb->mpcb_wbuf,
233 233 mpcb->mpcb_wbcnt * ((mpcb->mpcb_wstate == WSTATE_USER32) ?
234 234 sizeof (struct rwindow32) : sizeof (struct rwindow64)));
235 235 }
236 236
237 237 if (pt == curthread)
238 238 pfp->fpu_fprs = _fp_read_fprs();
239 239 if ((pfp->fpu_en) || (pfp->fpu_fprs & FPRS_FEF)) {
240 240 if (pt == curthread && fpu_exists) {
241 241 save_gsr(clwp->lwp_fpu);
242 242 } else {
243 243 uint64_t gsr;
244 244 gsr = get_gsr(lwp->lwp_fpu);
245 245 set_gsr(gsr, clwp->lwp_fpu);
246 246 }
247 247 fp_fork(lwp, clwp);
248 248 }
249 249 }
250 250
251 251 /*
252 252 * Free lwp fpu regs.
253 253 */
254 254 void
255 255 lwp_freeregs(klwp_t *lwp, int isexec)
256 256 {
257 257 kfpu_t *fp = lwptofpu(lwp);
258 258
259 259 if (lwptot(lwp) == curthread)
260 260 fp->fpu_fprs = _fp_read_fprs();
261 261 if ((fp->fpu_en) || (fp->fpu_fprs & FPRS_FEF))
262 262 fp_free(fp, isexec);
263 263 }
264 264
265 265 /*
266 266 * These function are currently unused on sparc.
267 267 */
268 268 /*ARGSUSED*/
269 269 void
270 270 lwp_attach_brand_hdlrs(klwp_t *lwp)
271 271 {}
272 272
273 273 /*ARGSUSED*/
274 274 void
275 275 lwp_detach_brand_hdlrs(klwp_t *lwp)
276 276 {}
277 277
278 278 /*
279 279 * fill in the extra register state area specified with the
280 280 * specified lwp's platform-dependent non-floating-point extra
281 281 * register state information
282 282 */
283 283 /* ARGSUSED */
284 284 void
285 285 xregs_getgfiller(klwp_id_t lwp, caddr_t xrp)
286 286 {
287 287 /* for sun4u nothing to do here, added for symmetry */
288 288 }
289 289
290 290 /*
291 291 * fill in the extra register state area specified with the specified lwp's
292 292 * platform-dependent floating-point extra register state information.
293 293 * NOTE: 'lwp' might not correspond to 'curthread' since this is
294 294 * called from code in /proc to get the registers of another lwp.
295 295 */
296 296 void
297 297 xregs_getfpfiller(klwp_id_t lwp, caddr_t xrp)
298 298 {
299 299 prxregset_t *xregs = (prxregset_t *)xrp;
300 300 kfpu_t *fp = lwptofpu(lwp);
301 301 uint32_t fprs = (FPRS_FEF|FPRS_DU|FPRS_DL);
302 302 uint64_t gsr;
303 303
304 304 /*
305 305 * fp_fksave() does not flush the GSR register into
306 306 * the lwp area, so do it now
307 307 */
308 308 kpreempt_disable();
309 309 if (ttolwp(curthread) == lwp && fpu_exists) {
310 310 fp->fpu_fprs = _fp_read_fprs();
311 311 if ((fp->fpu_fprs & FPRS_FEF) != FPRS_FEF) {
312 312 _fp_write_fprs(fprs);
313 313 fp->fpu_fprs = (V9_FPU_FPRS_TYPE)fprs;
314 314 }
315 315 save_gsr(fp);
316 316 }
317 317 gsr = get_gsr(fp);
318 318 kpreempt_enable();
319 319 PRXREG_GSR(xregs) = gsr;
320 320 }
321 321
322 322 /*
323 323 * set the specified lwp's platform-dependent non-floating-point
324 324 * extra register state based on the specified input
325 325 */
326 326 /* ARGSUSED */
327 327 void
328 328 xregs_setgfiller(klwp_id_t lwp, caddr_t xrp)
329 329 {
330 330 /* for sun4u nothing to do here, added for symmetry */
331 331 }
332 332
333 333 /*
334 334 * set the specified lwp's platform-dependent floating-point
335 335 * extra register state based on the specified input
336 336 */
337 337 void
338 338 xregs_setfpfiller(klwp_id_t lwp, caddr_t xrp)
339 339 {
340 340 prxregset_t *xregs = (prxregset_t *)xrp;
341 341 kfpu_t *fp = lwptofpu(lwp);
342 342 uint32_t fprs = (FPRS_FEF|FPRS_DU|FPRS_DL);
343 343 uint64_t gsr = PRXREG_GSR(xregs);
344 344
345 345 kpreempt_disable();
346 346 set_gsr(gsr, lwptofpu(lwp));
347 347
348 348 if ((lwp == ttolwp(curthread)) && fpu_exists) {
349 349 fp->fpu_fprs = _fp_read_fprs();
350 350 if ((fp->fpu_fprs & FPRS_FEF) != FPRS_FEF) {
351 351 _fp_write_fprs(fprs);
352 352 fp->fpu_fprs = (V9_FPU_FPRS_TYPE)fprs;
353 353 }
354 354 restore_gsr(lwptofpu(lwp));
355 355 }
356 356 kpreempt_enable();
357 357 }
358 358
359 359 /*
360 360 * fill in the sun4u asrs, ie, the lwp's platform-dependent
361 361 * non-floating-point extra register state information
362 362 */
363 363 /* ARGSUSED */
364 364 void
365 365 getasrs(klwp_t *lwp, asrset_t asr)
366 366 {
367 367 /* for sun4u nothing to do here, added for symmetry */
368 368 }
369 369
370 370 /*
371 371 * fill in the sun4u asrs, ie, the lwp's platform-dependent
372 372 * floating-point extra register state information
373 373 */
374 374 void
375 375 getfpasrs(klwp_t *lwp, asrset_t asr)
376 376 {
377 377 kfpu_t *fp = lwptofpu(lwp);
378 378 uint32_t fprs = (FPRS_FEF|FPRS_DU|FPRS_DL);
379 379
380 380 kpreempt_disable();
381 381 if (ttolwp(curthread) == lwp)
382 382 fp->fpu_fprs = _fp_read_fprs();
383 383 if ((fp->fpu_en) || (fp->fpu_fprs & FPRS_FEF)) {
384 384 if (fpu_exists && ttolwp(curthread) == lwp) {
385 385 if ((fp->fpu_fprs & FPRS_FEF) != FPRS_FEF) {
386 386 _fp_write_fprs(fprs);
387 387 fp->fpu_fprs = (V9_FPU_FPRS_TYPE)fprs;
388 388 }
389 389 save_gsr(fp);
390 390 }
391 391 asr[ASR_GSR] = (int64_t)get_gsr(fp);
392 392 }
393 393 kpreempt_enable();
394 394 }
395 395
396 396 /*
397 397 * set the sun4u asrs, ie, the lwp's platform-dependent
398 398 * non-floating-point extra register state information
399 399 */
400 400 /* ARGSUSED */
401 401 void
402 402 setasrs(klwp_t *lwp, asrset_t asr)
403 403 {
404 404 /* for sun4u nothing to do here, added for symmetry */
405 405 }
406 406
407 407 void
408 408 setfpasrs(klwp_t *lwp, asrset_t asr)
409 409 {
410 410 kfpu_t *fp = lwptofpu(lwp);
411 411 uint32_t fprs = (FPRS_FEF|FPRS_DU|FPRS_DL);
412 412
413 413 kpreempt_disable();
414 414 if (ttolwp(curthread) == lwp)
415 415 fp->fpu_fprs = _fp_read_fprs();
416 416 if ((fp->fpu_en) || (fp->fpu_fprs & FPRS_FEF)) {
417 417 set_gsr(asr[ASR_GSR], fp);
418 418 if (fpu_exists && ttolwp(curthread) == lwp) {
419 419 if ((fp->fpu_fprs & FPRS_FEF) != FPRS_FEF) {
420 420 _fp_write_fprs(fprs);
421 421 fp->fpu_fprs = (V9_FPU_FPRS_TYPE)fprs;
422 422 }
423 423 restore_gsr(fp);
424 424 }
425 425 }
426 426 kpreempt_enable();
427 427 }
428 428
429 429 /*
430 430 * Create interrupt kstats for this CPU.
431 431 */
432 432 void
433 433 cpu_create_intrstat(cpu_t *cp)
434 434 {
435 435 int i;
436 436 kstat_t *intr_ksp;
437 437 kstat_named_t *knp;
438 438 char name[KSTAT_STRLEN];
439 439 zoneid_t zoneid;
440 440
441 441 ASSERT(MUTEX_HELD(&cpu_lock));
442 442
443 443 if (pool_pset_enabled())
444 444 zoneid = GLOBAL_ZONEID;
445 445 else
446 446 zoneid = ALL_ZONES;
447 447
448 448 intr_ksp = kstat_create_zone("cpu", cp->cpu_id, "intrstat", "misc",
449 449 KSTAT_TYPE_NAMED, PIL_MAX * 2, NULL, zoneid);
450 450
451 451 /*
452 452 * Initialize each PIL's named kstat
453 453 */
454 454 if (intr_ksp != NULL) {
455 455 intr_ksp->ks_update = cpu_kstat_intrstat_update;
456 456 knp = (kstat_named_t *)intr_ksp->ks_data;
457 457 intr_ksp->ks_private = cp;
458 458 for (i = 0; i < PIL_MAX; i++) {
459 459 (void) snprintf(name, KSTAT_STRLEN, "level-%d-time",
460 460 i + 1);
461 461 kstat_named_init(&knp[i * 2], name, KSTAT_DATA_UINT64);
462 462 (void) snprintf(name, KSTAT_STRLEN, "level-%d-count",
463 463 i + 1);
464 464 kstat_named_init(&knp[(i * 2) + 1], name,
465 465 KSTAT_DATA_UINT64);
466 466 }
467 467 kstat_install(intr_ksp);
468 468 }
469 469 }
470 470
471 471 /*
472 472 * Delete interrupt kstats for this CPU.
473 473 */
474 474 void
475 475 cpu_delete_intrstat(cpu_t *cp)
476 476 {
477 477 kstat_delete_byname_zone("cpu", cp->cpu_id, "intrstat", ALL_ZONES);
478 478 }
479 479
480 480 /*
481 481 * Convert interrupt statistics from CPU ticks to nanoseconds and
482 482 * update kstat.
483 483 */
484 484 int
485 485 cpu_kstat_intrstat_update(kstat_t *ksp, int rw)
486 486 {
487 487 kstat_named_t *knp = ksp->ks_data;
488 488 cpu_t *cpup = (cpu_t *)ksp->ks_private;
489 489 int i;
490 490
491 491 if (rw == KSTAT_WRITE)
492 492 return (EACCES);
493 493
494 494 /*
495 495 * We use separate passes to copy and convert the statistics to
496 496 * nanoseconds. This assures that the snapshot of the data is as
497 497 * self-consistent as possible.
498 498 */
499 499
500 500 for (i = 0; i < PIL_MAX; i++) {
501 501 knp[i * 2].value.ui64 = cpup->cpu_m.intrstat[i + 1][0];
502 502 knp[(i * 2) + 1].value.ui64 = cpup->cpu_stats.sys.intr[i];
503 503 }
504 504
505 505 for (i = 0; i < PIL_MAX; i++) {
506 506 knp[i * 2].value.ui64 =
507 507 (uint64_t)tick2ns((hrtime_t)knp[i * 2].value.ui64,
508 508 cpup->cpu_id);
509 509 }
510 510
511 511 return (0);
512 512 }
513 513
514 514 /*
515 515 * Called by common/os/cpu.c for psrinfo(1m) kstats
516 516 */
517 517 char *
518 518 cpu_fru_fmri(cpu_t *cp)
519 519 {
520 520 return (cpunodes[cp->cpu_id].fru_fmri);
521 521 }
522 522
523 523 /*
524 524 * An interrupt thread is ending a time slice, so compute the interval it
525 525 * ran for and update the statistic for its PIL.
526 526 */
527 527 void
528 528 cpu_intr_swtch_enter(kthread_id_t t)
529 529 {
530 530 uint64_t interval;
531 531 uint64_t start;
532 532 cpu_t *cpu;
533 533
534 534 ASSERT((t->t_flag & T_INTR_THREAD) != 0);
535 535 ASSERT(t->t_pil > 0 && t->t_pil <= LOCK_LEVEL);
536 536
537 537 /*
538 538 * We could be here with a zero timestamp. This could happen if:
539 539 * an interrupt thread which no longer has a pinned thread underneath
540 540 * it (i.e. it blocked at some point in its past) has finished running
541 541 * its handler. intr_thread() updated the interrupt statistic for its
542 542 * PIL and zeroed its timestamp. Since there was no pinned thread to
↓ open down ↓ |
542 lines elided |
↑ open up ↑ |
543 543 * return to, swtch() gets called and we end up here.
544 544 *
545 545 * It can also happen if an interrupt thread in intr_thread() calls
546 546 * preempt. It will have already taken care of updating stats. In
547 547 * this event, the interrupt thread will be runnable.
548 548 */
549 549 if (t->t_intr_start) {
550 550 do {
551 551 start = t->t_intr_start;
552 552 interval = CLOCK_TICK_COUNTER() - start;
553 - } while (cas64(&t->t_intr_start, start, 0) != start);
553 + } while (atomic_cas_64(&t->t_intr_start, start, 0) != start);
554 554 cpu = CPU;
555 555 if (cpu->cpu_m.divisor > 1)
556 556 interval *= cpu->cpu_m.divisor;
557 557 cpu->cpu_m.intrstat[t->t_pil][0] += interval;
558 558
559 559 atomic_add_64((uint64_t *)&cpu->cpu_intracct[cpu->cpu_mstate],
560 560 interval);
561 561 } else
562 562 ASSERT(t->t_intr == NULL || t->t_state == TS_RUN);
563 563 }
564 564
565 565
566 566 /*
567 567 * An interrupt thread is returning from swtch(). Place a starting timestamp
568 568 * in its thread structure.
569 569 */
↓ open down ↓ |
6 lines elided |
↑ open up ↑ |
570 570 void
571 571 cpu_intr_swtch_exit(kthread_id_t t)
572 572 {
573 573 uint64_t ts;
574 574
575 575 ASSERT((t->t_flag & T_INTR_THREAD) != 0);
576 576 ASSERT(t->t_pil > 0 && t->t_pil <= LOCK_LEVEL);
577 577
578 578 do {
579 579 ts = t->t_intr_start;
580 - } while (cas64(&t->t_intr_start, ts, CLOCK_TICK_COUNTER()) != ts);
580 + } while (atomic_cas_64(&t->t_intr_start, ts, CLOCK_TICK_COUNTER()) !=
581 + ts);
581 582 }
582 583
583 584
584 585 int
585 586 blacklist(int cmd, const char *scheme, nvlist_t *fmri, const char *class)
586 587 {
587 588 if (&plat_blacklist)
588 589 return (plat_blacklist(cmd, scheme, fmri, class));
589 590
590 591 return (ENOTSUP);
591 592 }
592 593
593 594 int
594 595 kdi_pread(caddr_t buf, size_t nbytes, uint64_t addr, size_t *ncopiedp)
595 596 {
596 597 extern void kdi_flush_caches(void);
597 598 size_t nread = 0;
598 599 uint32_t word;
599 600 int slop, i;
600 601
601 602 kdi_flush_caches();
602 603 membar_enter();
603 604
604 605 /* We might not begin on a word boundary. */
605 606 if ((slop = addr & 3) != 0) {
606 607 word = ldphys(addr & ~3);
607 608 for (i = slop; i < 4 && nbytes > 0; i++, nbytes--, nread++)
608 609 *buf++ = ((uchar_t *)&word)[i];
609 610 addr = roundup(addr, 4);
610 611 }
611 612
612 613 while (nbytes > 0) {
613 614 word = ldphys(addr);
614 615 for (i = 0; i < 4 && nbytes > 0; i++, nbytes--, nread++, addr++)
615 616 *buf++ = ((uchar_t *)&word)[i];
616 617 }
617 618
618 619 kdi_flush_caches();
619 620
620 621 *ncopiedp = nread;
621 622 return (0);
622 623 }
623 624
624 625 int
625 626 kdi_pwrite(caddr_t buf, size_t nbytes, uint64_t addr, size_t *ncopiedp)
626 627 {
627 628 extern void kdi_flush_caches(void);
628 629 size_t nwritten = 0;
629 630 uint32_t word;
630 631 int slop, i;
631 632
632 633 kdi_flush_caches();
633 634
634 635 /* We might not begin on a word boundary. */
635 636 if ((slop = addr & 3) != 0) {
636 637 word = ldphys(addr & ~3);
637 638 for (i = slop; i < 4 && nbytes > 0; i++, nbytes--, nwritten++)
638 639 ((uchar_t *)&word)[i] = *buf++;
639 640 stphys(addr & ~3, word);
640 641 addr = roundup(addr, 4);
641 642 }
642 643
643 644 while (nbytes > 3) {
644 645 for (word = 0, i = 0; i < 4; i++, nbytes--, nwritten++)
645 646 ((uchar_t *)&word)[i] = *buf++;
646 647 stphys(addr, word);
647 648 addr += 4;
648 649 }
649 650
650 651 /* We might not end with a whole word. */
651 652 if (nbytes > 0) {
652 653 word = ldphys(addr);
653 654 for (i = 0; nbytes > 0; i++, nbytes--, nwritten++)
654 655 ((uchar_t *)&word)[i] = *buf++;
655 656 stphys(addr, word);
656 657 }
657 658
658 659 membar_enter();
659 660 kdi_flush_caches();
660 661
661 662 *ncopiedp = nwritten;
662 663 return (0);
663 664 }
664 665
665 666 static void
666 667 kdi_kernpanic(struct regs *regs, uint_t tt)
667 668 {
668 669 sync_reg_buf = *regs;
669 670 sync_tt = tt;
670 671
671 672 sync_handler();
672 673 }
673 674
674 675 static void
675 676 kdi_plat_call(void (*platfn)(void))
676 677 {
677 678 if (platfn != NULL) {
678 679 prom_suspend_prepost();
679 680 platfn();
680 681 prom_resume_prepost();
681 682 }
682 683 }
683 684
684 685 /*
685 686 * kdi_system_claim and release are defined here for all sun4 platforms and
686 687 * pointed to by mach_kdi_init() to provide default callbacks for such systems.
687 688 * Specific sun4u or sun4v platforms may implement their own claim and release
688 689 * routines, at which point their respective callbacks will be updated.
689 690 */
690 691 static void
691 692 kdi_system_claim(void)
692 693 {
693 694 lbolt_debug_entry();
694 695 }
695 696
696 697 static void
697 698 kdi_system_release(void)
698 699 {
699 700 lbolt_debug_return();
700 701 }
701 702
702 703 void
703 704 mach_kdi_init(kdi_t *kdi)
704 705 {
705 706 kdi->kdi_plat_call = kdi_plat_call;
706 707 kdi->kdi_kmdb_enter = kmdb_enter;
707 708 kdi->pkdi_system_claim = kdi_system_claim;
708 709 kdi->pkdi_system_release = kdi_system_release;
709 710 kdi->mkdi_cpu_index = kdi_cpu_index;
710 711 kdi->mkdi_trap_vatotte = kdi_trap_vatotte;
711 712 kdi->mkdi_kernpanic = kdi_kernpanic;
712 713 }
713 714
714 715
715 716 /*
716 717 * get_cpu_mstate() is passed an array of timestamps, NCMSTATES
717 718 * long, and it fills in the array with the time spent on cpu in
718 719 * each of the mstates, where time is returned in nsec.
719 720 *
720 721 * No guarantee is made that the returned values in times[] will
721 722 * monotonically increase on sequential calls, although this will
722 723 * be true in the long run. Any such guarantee must be handled by
723 724 * the caller, if needed. This can happen if we fail to account
724 725 * for elapsed time due to a generation counter conflict, yet we
725 726 * did account for it on a prior call (see below).
726 727 *
727 728 * The complication is that the cpu in question may be updating
728 729 * its microstate at the same time that we are reading it.
729 730 * Because the microstate is only updated when the CPU's state
730 731 * changes, the values in cpu_intracct[] can be indefinitely out
731 732 * of date. To determine true current values, it is necessary to
732 733 * compare the current time with cpu_mstate_start, and add the
733 734 * difference to times[cpu_mstate].
734 735 *
735 736 * This can be a problem if those values are changing out from
736 737 * under us. Because the code path in new_cpu_mstate() is
737 738 * performance critical, we have not added a lock to it. Instead,
738 739 * we have added a generation counter. Before beginning
739 740 * modifications, the counter is set to 0. After modifications,
740 741 * it is set to the old value plus one.
741 742 *
742 743 * get_cpu_mstate() will not consider the values of cpu_mstate
743 744 * and cpu_mstate_start to be usable unless the value of
744 745 * cpu_mstate_gen is both non-zero and unchanged, both before and
745 746 * after reading the mstate information. Note that we must
746 747 * protect against out-of-order loads around accesses to the
747 748 * generation counter. Also, this is a best effort approach in
748 749 * that we do not retry should the counter be found to have
749 750 * changed.
750 751 *
751 752 * cpu_intracct[] is used to identify time spent in each CPU
752 753 * mstate while handling interrupts. Such time should be reported
753 754 * against system time, and so is subtracted out from its
754 755 * corresponding cpu_acct[] time and added to
755 756 * cpu_acct[CMS_SYSTEM]. Additionally, intracct time is stored in
756 757 * %ticks, but acct time may be stored as %sticks, thus requiring
757 758 * different conversions before they can be compared.
758 759 */
759 760
760 761 void
761 762 get_cpu_mstate(cpu_t *cpu, hrtime_t *times)
762 763 {
763 764 int i;
764 765 hrtime_t now, start;
765 766 uint16_t gen;
766 767 uint16_t state;
767 768 hrtime_t intracct[NCMSTATES];
768 769
769 770 /*
770 771 * Load all volatile state under the protection of membar.
771 772 * cpu_acct[cpu_mstate] must be loaded to avoid double counting
772 773 * of (now - cpu_mstate_start) by a change in CPU mstate that
773 774 * arrives after we make our last check of cpu_mstate_gen.
774 775 */
775 776
776 777 now = gethrtime_unscaled();
777 778 gen = cpu->cpu_mstate_gen;
778 779
779 780 membar_consumer(); /* guarantee load ordering */
780 781 start = cpu->cpu_mstate_start;
781 782 state = cpu->cpu_mstate;
782 783 for (i = 0; i < NCMSTATES; i++) {
783 784 intracct[i] = cpu->cpu_intracct[i];
784 785 times[i] = cpu->cpu_acct[i];
785 786 }
786 787 membar_consumer(); /* guarantee load ordering */
787 788
788 789 if (gen != 0 && gen == cpu->cpu_mstate_gen && now > start)
789 790 times[state] += now - start;
790 791
791 792 for (i = 0; i < NCMSTATES; i++) {
792 793 scalehrtime(×[i]);
793 794 intracct[i] = tick2ns((hrtime_t)intracct[i], cpu->cpu_id);
794 795 }
795 796
796 797 for (i = 0; i < NCMSTATES; i++) {
797 798 if (i == CMS_SYSTEM)
798 799 continue;
799 800 times[i] -= intracct[i];
800 801 if (times[i] < 0) {
801 802 intracct[i] += times[i];
802 803 times[i] = 0;
803 804 }
804 805 times[CMS_SYSTEM] += intracct[i];
805 806 }
806 807 }
807 808
808 809 void
809 810 mach_cpu_pause(volatile char *safe)
810 811 {
811 812 /*
812 813 * This cpu is now safe.
813 814 */
814 815 *safe = PAUSE_WAIT;
815 816 membar_enter(); /* make sure stores are flushed */
816 817
817 818 /*
818 819 * Now we wait. When we are allowed to continue, safe
819 820 * will be set to PAUSE_IDLE.
820 821 */
821 822 while (*safe != PAUSE_IDLE)
822 823 SMT_PAUSE();
823 824 }
824 825
825 826 /*ARGSUSED*/
826 827 int
827 828 plat_mem_do_mmio(struct uio *uio, enum uio_rw rw)
828 829 {
829 830 return (ENOTSUP);
830 831 }
831 832
832 833 /* cpu threshold for compressed dumps */
833 834 #ifdef sun4v
834 835 uint_t dump_plat_mincpu_default = DUMP_PLAT_SUN4V_MINCPU;
835 836 #else
836 837 uint_t dump_plat_mincpu_default = DUMP_PLAT_SUN4U_MINCPU;
837 838 #endif
838 839
839 840 int
840 841 dump_plat_addr()
841 842 {
842 843 return (0);
843 844 }
844 845
845 846 void
846 847 dump_plat_pfn()
847 848 {
848 849 }
849 850
850 851 /* ARGSUSED */
851 852 int
852 853 dump_plat_data(void *dump_cdata)
853 854 {
854 855 return (0);
855 856 }
856 857
857 858 /* ARGSUSED */
858 859 int
859 860 plat_hold_page(pfn_t pfn, int lock, page_t **pp_ret)
860 861 {
861 862 return (PLAT_HOLD_OK);
862 863 }
863 864
864 865 /* ARGSUSED */
865 866 void
866 867 plat_release_page(page_t *pp)
867 868 {
868 869 }
869 870
870 871 /* ARGSUSED */
871 872 void
872 873 progressbar_key_abort(ldi_ident_t li)
873 874 {
874 875 }
875 876
876 877 /*
877 878 * We need to post a soft interrupt to reprogram the lbolt cyclic when
878 879 * switching from event to cyclic driven lbolt. The following code adds
879 880 * and posts the softint for sun4 platforms.
880 881 */
881 882 static uint64_t lbolt_softint_inum;
882 883
883 884 void
884 885 lbolt_softint_add(void)
885 886 {
886 887 lbolt_softint_inum = add_softintr(LOCK_LEVEL,
887 888 (softintrfunc)lbolt_ev_to_cyclic, NULL, SOFTINT_MT);
888 889 }
889 890
890 891 void
891 892 lbolt_softint_post(void)
892 893 {
893 894 setsoftint(lbolt_softint_inum);
894 895 }
↓ open down ↓ |
304 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX