Print this page
5042 stop using deprecated atomic functions
Split |
Close |
Expand all |
Collapse all |
--- old/usr/src/uts/common/disp/fss.c
+++ new/usr/src/uts/common/disp/fss.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright (c) 1994, 2010, Oracle and/or its affiliates. All rights reserved.
24 24 * Copyright 2013, Joyent, Inc. All rights reserved.
25 25 */
26 26
27 27 #include <sys/types.h>
28 28 #include <sys/param.h>
29 29 #include <sys/sysmacros.h>
30 30 #include <sys/cred.h>
31 31 #include <sys/proc.h>
32 32 #include <sys/strsubr.h>
33 33 #include <sys/priocntl.h>
34 34 #include <sys/class.h>
35 35 #include <sys/disp.h>
36 36 #include <sys/procset.h>
37 37 #include <sys/debug.h>
38 38 #include <sys/kmem.h>
39 39 #include <sys/errno.h>
40 40 #include <sys/systm.h>
41 41 #include <sys/schedctl.h>
42 42 #include <sys/vmsystm.h>
43 43 #include <sys/atomic.h>
44 44 #include <sys/project.h>
45 45 #include <sys/modctl.h>
46 46 #include <sys/fss.h>
47 47 #include <sys/fsspriocntl.h>
48 48 #include <sys/cpupart.h>
49 49 #include <sys/zone.h>
50 50 #include <vm/rm.h>
51 51 #include <vm/seg_kmem.h>
52 52 #include <sys/tnf_probe.h>
53 53 #include <sys/policy.h>
54 54 #include <sys/sdt.h>
55 55 #include <sys/cpucaps.h>
56 56
57 57 /*
58 58 * FSS Data Structures:
59 59 *
60 60 * fsszone
61 61 * ----- -----
62 62 * ----- | | | |
63 63 * | |-------->| |<------->| |<---->...
64 64 * | | ----- -----
65 65 * | | ^ ^ ^
66 66 * | |--- | \ \
67 67 * ----- | | \ \
68 68 * fsspset | | \ \
69 69 * | | \ \
70 70 * | ----- ----- -----
71 71 * -->| |<--->| |<--->| |
72 72 * | | | | | |
73 73 * ----- ----- -----
74 74 * fssproj
75 75 *
76 76 *
77 77 * That is, fsspsets contain a list of fsszone's that are currently active in
78 78 * the pset, and a list of fssproj's, corresponding to projects with runnable
79 79 * threads on the pset. fssproj's in turn point to the fsszone which they
80 80 * are a member of.
81 81 *
82 82 * An fssproj_t is removed when there are no threads in it.
83 83 *
84 84 * An fsszone_t is removed when there are no projects with threads in it.
85 85 *
86 86 * Projects in a zone compete with each other for cpu time, receiving cpu
87 87 * allocation within a zone proportional to fssproj->fssp_shares
88 88 * (project.cpu-shares); at a higher level zones compete with each other,
89 89 * receiving allocation in a pset proportional to fsszone->fssz_shares
90 90 * (zone.cpu-shares). See fss_decay_usage() for the precise formula.
91 91 */
92 92
93 93 static pri_t fss_init(id_t, int, classfuncs_t **);
94 94
95 95 static struct sclass fss = {
96 96 "FSS",
97 97 fss_init,
98 98 0
99 99 };
100 100
101 101 extern struct mod_ops mod_schedops;
102 102
103 103 /*
104 104 * Module linkage information for the kernel.
105 105 */
106 106 static struct modlsched modlsched = {
107 107 &mod_schedops, "fair share scheduling class", &fss
108 108 };
109 109
110 110 static struct modlinkage modlinkage = {
111 111 MODREV_1, (void *)&modlsched, NULL
112 112 };
113 113
114 114 #define FSS_MAXUPRI 60
115 115
116 116 /*
117 117 * The fssproc_t structures are kept in an array of circular doubly linked
118 118 * lists. A hash on the thread pointer is used to determine which list each
119 119 * thread should be placed in. Each list has a dummy "head" which is never
120 120 * removed, so the list is never empty. fss_update traverses these lists to
121 121 * update the priorities of threads that have been waiting on the run queue.
122 122 */
123 123 #define FSS_LISTS 16 /* number of lists, must be power of 2 */
124 124 #define FSS_LIST_HASH(t) (((uintptr_t)(t) >> 9) & (FSS_LISTS - 1))
125 125 #define FSS_LIST_NEXT(i) (((i) + 1) & (FSS_LISTS - 1))
126 126
127 127 #define FSS_LIST_INSERT(fssproc) \
128 128 { \
129 129 int index = FSS_LIST_HASH(fssproc->fss_tp); \
130 130 kmutex_t *lockp = &fss_listlock[index]; \
131 131 fssproc_t *headp = &fss_listhead[index]; \
132 132 mutex_enter(lockp); \
133 133 fssproc->fss_next = headp->fss_next; \
134 134 fssproc->fss_prev = headp; \
135 135 headp->fss_next->fss_prev = fssproc; \
136 136 headp->fss_next = fssproc; \
137 137 mutex_exit(lockp); \
138 138 }
139 139
140 140 #define FSS_LIST_DELETE(fssproc) \
141 141 { \
142 142 int index = FSS_LIST_HASH(fssproc->fss_tp); \
143 143 kmutex_t *lockp = &fss_listlock[index]; \
144 144 mutex_enter(lockp); \
145 145 fssproc->fss_prev->fss_next = fssproc->fss_next; \
146 146 fssproc->fss_next->fss_prev = fssproc->fss_prev; \
147 147 mutex_exit(lockp); \
148 148 }
149 149
150 150 #define FSS_TICK_COST 1000 /* tick cost for threads with nice level = 0 */
151 151
152 152 /*
153 153 * Decay rate percentages are based on n/128 rather than n/100 so that
154 154 * calculations can avoid having to do an integer divide by 100 (divide
155 155 * by FSS_DECAY_BASE == 128 optimizes to an arithmetic shift).
156 156 *
157 157 * FSS_DECAY_MIN = 83/128 ~= 65%
158 158 * FSS_DECAY_MAX = 108/128 ~= 85%
159 159 * FSS_DECAY_USG = 96/128 ~= 75%
160 160 */
161 161 #define FSS_DECAY_MIN 83 /* fsspri decay pct for threads w/ nice -20 */
162 162 #define FSS_DECAY_MAX 108 /* fsspri decay pct for threads w/ nice +19 */
163 163 #define FSS_DECAY_USG 96 /* fssusage decay pct for projects */
164 164 #define FSS_DECAY_BASE 128 /* base for decay percentages above */
165 165
166 166 #define FSS_NICE_MIN 0
167 167 #define FSS_NICE_MAX (2 * NZERO - 1)
168 168 #define FSS_NICE_RANGE (FSS_NICE_MAX - FSS_NICE_MIN + 1)
169 169
170 170 static int fss_nice_tick[FSS_NICE_RANGE];
171 171 static int fss_nice_decay[FSS_NICE_RANGE];
172 172
173 173 static pri_t fss_maxupri = FSS_MAXUPRI; /* maximum FSS user priority */
174 174 static pri_t fss_maxumdpri; /* maximum user mode fss priority */
175 175 static pri_t fss_maxglobpri; /* maximum global priority used by fss class */
176 176 static pri_t fss_minglobpri; /* minimum global priority */
177 177
178 178 static fssproc_t fss_listhead[FSS_LISTS];
179 179 static kmutex_t fss_listlock[FSS_LISTS];
180 180
181 181 static fsspset_t *fsspsets;
182 182 static kmutex_t fsspsets_lock; /* protects fsspsets */
183 183
184 184 static id_t fss_cid;
185 185
186 186 static time_t fss_minrun = 2; /* t_pri becomes 59 within 2 secs */
187 187 static time_t fss_minslp = 2; /* min time on sleep queue for hardswap */
188 188 static int fss_quantum = 11;
189 189
190 190 static void fss_newpri(fssproc_t *);
191 191 static void fss_update(void *);
192 192 static int fss_update_list(int);
193 193 static void fss_change_priority(kthread_t *, fssproc_t *);
194 194
195 195 static int fss_admin(caddr_t, cred_t *);
196 196 static int fss_getclinfo(void *);
197 197 static int fss_parmsin(void *);
198 198 static int fss_parmsout(void *, pc_vaparms_t *);
199 199 static int fss_vaparmsin(void *, pc_vaparms_t *);
200 200 static int fss_vaparmsout(void *, pc_vaparms_t *);
201 201 static int fss_getclpri(pcpri_t *);
202 202 static int fss_alloc(void **, int);
203 203 static void fss_free(void *);
204 204
205 205 static int fss_enterclass(kthread_t *, id_t, void *, cred_t *, void *);
206 206 static void fss_exitclass(void *);
207 207 static int fss_canexit(kthread_t *, cred_t *);
208 208 static int fss_fork(kthread_t *, kthread_t *, void *);
209 209 static void fss_forkret(kthread_t *, kthread_t *);
210 210 static void fss_parmsget(kthread_t *, void *);
211 211 static int fss_parmsset(kthread_t *, void *, id_t, cred_t *);
212 212 static void fss_stop(kthread_t *, int, int);
213 213 static void fss_exit(kthread_t *);
214 214 static void fss_active(kthread_t *);
215 215 static void fss_inactive(kthread_t *);
216 216 static pri_t fss_swapin(kthread_t *, int);
217 217 static pri_t fss_swapout(kthread_t *, int);
218 218 static void fss_trapret(kthread_t *);
219 219 static void fss_preempt(kthread_t *);
220 220 static void fss_setrun(kthread_t *);
221 221 static void fss_sleep(kthread_t *);
222 222 static void fss_tick(kthread_t *);
223 223 static void fss_wakeup(kthread_t *);
224 224 static int fss_donice(kthread_t *, cred_t *, int, int *);
225 225 static int fss_doprio(kthread_t *, cred_t *, int, int *);
226 226 static pri_t fss_globpri(kthread_t *);
227 227 static void fss_yield(kthread_t *);
228 228 static void fss_nullsys();
229 229
230 230 static struct classfuncs fss_classfuncs = {
231 231 /* class functions */
232 232 fss_admin,
233 233 fss_getclinfo,
234 234 fss_parmsin,
235 235 fss_parmsout,
236 236 fss_vaparmsin,
237 237 fss_vaparmsout,
238 238 fss_getclpri,
239 239 fss_alloc,
240 240 fss_free,
241 241
242 242 /* thread functions */
243 243 fss_enterclass,
244 244 fss_exitclass,
245 245 fss_canexit,
246 246 fss_fork,
247 247 fss_forkret,
248 248 fss_parmsget,
249 249 fss_parmsset,
250 250 fss_stop,
251 251 fss_exit,
252 252 fss_active,
253 253 fss_inactive,
254 254 fss_swapin,
255 255 fss_swapout,
256 256 fss_trapret,
257 257 fss_preempt,
258 258 fss_setrun,
259 259 fss_sleep,
260 260 fss_tick,
261 261 fss_wakeup,
262 262 fss_donice,
263 263 fss_globpri,
264 264 fss_nullsys, /* set_process_group */
265 265 fss_yield,
266 266 fss_doprio,
267 267 };
268 268
269 269 int
270 270 _init()
271 271 {
272 272 return (mod_install(&modlinkage));
273 273 }
274 274
275 275 int
276 276 _fini()
277 277 {
278 278 return (EBUSY);
279 279 }
280 280
281 281 int
282 282 _info(struct modinfo *modinfop)
283 283 {
284 284 return (mod_info(&modlinkage, modinfop));
285 285 }
286 286
287 287 /*ARGSUSED*/
288 288 static int
289 289 fss_project_walker(kproject_t *kpj, void *buf)
290 290 {
291 291 return (0);
292 292 }
293 293
294 294 void *
295 295 fss_allocbuf(int op, int type)
296 296 {
297 297 fssbuf_t *fssbuf;
298 298 void **fsslist;
299 299 int cnt;
300 300 int i;
301 301 size_t size;
302 302
303 303 ASSERT(op == FSS_NPSET_BUF || op == FSS_NPROJ_BUF || op == FSS_ONE_BUF);
304 304 ASSERT(type == FSS_ALLOC_PROJ || type == FSS_ALLOC_ZONE);
305 305 ASSERT(MUTEX_HELD(&cpu_lock));
306 306
307 307 fssbuf = kmem_zalloc(sizeof (fssbuf_t), KM_SLEEP);
308 308 switch (op) {
309 309 case FSS_NPSET_BUF:
310 310 cnt = cpupart_list(NULL, 0, CP_NONEMPTY);
311 311 break;
312 312 case FSS_NPROJ_BUF:
313 313 cnt = project_walk_all(ALL_ZONES, fss_project_walker, NULL);
314 314 break;
315 315 case FSS_ONE_BUF:
316 316 cnt = 1;
317 317 break;
318 318 }
319 319
320 320 switch (type) {
321 321 case FSS_ALLOC_PROJ:
322 322 size = sizeof (fssproj_t);
323 323 break;
324 324 case FSS_ALLOC_ZONE:
325 325 size = sizeof (fsszone_t);
326 326 break;
327 327 }
328 328 fsslist = kmem_zalloc(cnt * sizeof (void *), KM_SLEEP);
329 329 fssbuf->fssb_size = cnt;
330 330 fssbuf->fssb_list = fsslist;
331 331 for (i = 0; i < cnt; i++)
332 332 fsslist[i] = kmem_zalloc(size, KM_SLEEP);
333 333 return (fssbuf);
334 334 }
335 335
336 336 void
337 337 fss_freebuf(fssbuf_t *fssbuf, int type)
338 338 {
339 339 void **fsslist;
340 340 int i;
341 341 size_t size;
342 342
343 343 ASSERT(fssbuf != NULL);
344 344 ASSERT(type == FSS_ALLOC_PROJ || type == FSS_ALLOC_ZONE);
345 345 fsslist = fssbuf->fssb_list;
346 346
347 347 switch (type) {
348 348 case FSS_ALLOC_PROJ:
349 349 size = sizeof (fssproj_t);
350 350 break;
351 351 case FSS_ALLOC_ZONE:
352 352 size = sizeof (fsszone_t);
353 353 break;
354 354 }
355 355
356 356 for (i = 0; i < fssbuf->fssb_size; i++) {
357 357 if (fsslist[i] != NULL)
358 358 kmem_free(fsslist[i], size);
359 359 }
360 360 kmem_free(fsslist, sizeof (void *) * fssbuf->fssb_size);
361 361 kmem_free(fssbuf, sizeof (fssbuf_t));
362 362 }
363 363
364 364 static fsspset_t *
365 365 fss_find_fsspset(cpupart_t *cpupart)
366 366 {
367 367 int i;
368 368 fsspset_t *fsspset = NULL;
369 369 int found = 0;
370 370
371 371 ASSERT(cpupart != NULL);
372 372 ASSERT(MUTEX_HELD(&fsspsets_lock));
373 373
374 374 /*
375 375 * Search for the cpupart pointer in the array of fsspsets.
376 376 */
377 377 for (i = 0; i < max_ncpus; i++) {
378 378 fsspset = &fsspsets[i];
379 379 if (fsspset->fssps_cpupart == cpupart) {
380 380 ASSERT(fsspset->fssps_nproj > 0);
381 381 found = 1;
382 382 break;
383 383 }
384 384 }
385 385 if (found == 0) {
386 386 /*
387 387 * If we didn't find anything, then use the first
388 388 * available slot in the fsspsets array.
389 389 */
390 390 for (i = 0; i < max_ncpus; i++) {
391 391 fsspset = &fsspsets[i];
392 392 if (fsspset->fssps_cpupart == NULL) {
393 393 ASSERT(fsspset->fssps_nproj == 0);
394 394 found = 1;
395 395 break;
396 396 }
397 397 }
398 398 fsspset->fssps_cpupart = cpupart;
399 399 }
400 400 ASSERT(found == 1);
401 401 return (fsspset);
402 402 }
403 403
404 404 static void
405 405 fss_del_fsspset(fsspset_t *fsspset)
406 406 {
407 407 ASSERT(MUTEX_HELD(&fsspsets_lock));
408 408 ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
409 409 ASSERT(fsspset->fssps_nproj == 0);
410 410 ASSERT(fsspset->fssps_list == NULL);
411 411 ASSERT(fsspset->fssps_zones == NULL);
412 412 fsspset->fssps_cpupart = NULL;
413 413 fsspset->fssps_maxfsspri = 0;
414 414 fsspset->fssps_shares = 0;
415 415 }
416 416
417 417 /*
418 418 * The following routine returns a pointer to the fsszone structure which
419 419 * belongs to zone "zone" and cpu partition fsspset, if such structure exists.
420 420 */
421 421 static fsszone_t *
422 422 fss_find_fsszone(fsspset_t *fsspset, zone_t *zone)
423 423 {
424 424 fsszone_t *fsszone;
425 425
426 426 ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
427 427
428 428 if (fsspset->fssps_list != NULL) {
429 429 /*
430 430 * There are projects/zones active on this cpu partition
431 431 * already. Try to find our zone among them.
432 432 */
433 433 fsszone = fsspset->fssps_zones;
434 434 do {
435 435 if (fsszone->fssz_zone == zone) {
436 436 return (fsszone);
437 437 }
438 438 fsszone = fsszone->fssz_next;
439 439 } while (fsszone != fsspset->fssps_zones);
440 440 }
441 441 return (NULL);
442 442 }
443 443
444 444 /*
445 445 * The following routine links new fsszone structure into doubly linked list of
446 446 * zones active on the specified cpu partition.
447 447 */
448 448 static void
449 449 fss_insert_fsszone(fsspset_t *fsspset, zone_t *zone, fsszone_t *fsszone)
450 450 {
451 451 ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
452 452
453 453 fsszone->fssz_zone = zone;
454 454 fsszone->fssz_rshares = zone->zone_shares;
455 455
456 456 if (fsspset->fssps_zones == NULL) {
457 457 /*
458 458 * This will be the first fsszone for this fsspset
459 459 */
460 460 fsszone->fssz_next = fsszone->fssz_prev = fsszone;
461 461 fsspset->fssps_zones = fsszone;
462 462 } else {
463 463 /*
464 464 * Insert this fsszone to the doubly linked list.
465 465 */
466 466 fsszone_t *fssz_head = fsspset->fssps_zones;
467 467
468 468 fsszone->fssz_next = fssz_head;
469 469 fsszone->fssz_prev = fssz_head->fssz_prev;
470 470 fssz_head->fssz_prev->fssz_next = fsszone;
471 471 fssz_head->fssz_prev = fsszone;
472 472 fsspset->fssps_zones = fsszone;
473 473 }
474 474 }
475 475
476 476 /*
477 477 * The following routine removes a single fsszone structure from the doubly
478 478 * linked list of zones active on the specified cpu partition. Note that
479 479 * global fsspsets_lock must be held in case this fsszone structure is the last
480 480 * on the above mentioned list. Also note that the fsszone structure is not
481 481 * freed here, it is the responsibility of the caller to call kmem_free for it.
482 482 */
483 483 static void
484 484 fss_remove_fsszone(fsspset_t *fsspset, fsszone_t *fsszone)
485 485 {
486 486 ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
487 487 ASSERT(fsszone->fssz_nproj == 0);
488 488 ASSERT(fsszone->fssz_shares == 0);
489 489 ASSERT(fsszone->fssz_runnable == 0);
490 490
491 491 if (fsszone->fssz_next != fsszone) {
492 492 /*
493 493 * This is not the last zone in the list.
494 494 */
495 495 fsszone->fssz_prev->fssz_next = fsszone->fssz_next;
496 496 fsszone->fssz_next->fssz_prev = fsszone->fssz_prev;
497 497 if (fsspset->fssps_zones == fsszone)
498 498 fsspset->fssps_zones = fsszone->fssz_next;
499 499 } else {
500 500 /*
501 501 * This was the last zone active in this cpu partition.
502 502 */
503 503 fsspset->fssps_zones = NULL;
504 504 }
505 505 }
506 506
507 507 /*
508 508 * The following routine returns a pointer to the fssproj structure
509 509 * which belongs to project kpj and cpu partition fsspset, if such structure
510 510 * exists.
511 511 */
512 512 static fssproj_t *
513 513 fss_find_fssproj(fsspset_t *fsspset, kproject_t *kpj)
514 514 {
515 515 fssproj_t *fssproj;
516 516
517 517 ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
518 518
519 519 if (fsspset->fssps_list != NULL) {
520 520 /*
521 521 * There are projects running on this cpu partition already.
522 522 * Try to find our project among them.
523 523 */
524 524 fssproj = fsspset->fssps_list;
525 525 do {
526 526 if (fssproj->fssp_proj == kpj) {
527 527 ASSERT(fssproj->fssp_pset == fsspset);
528 528 return (fssproj);
529 529 }
530 530 fssproj = fssproj->fssp_next;
531 531 } while (fssproj != fsspset->fssps_list);
532 532 }
533 533 return (NULL);
534 534 }
535 535
536 536 /*
537 537 * The following routine links new fssproj structure into doubly linked list
538 538 * of projects running on the specified cpu partition.
539 539 */
540 540 static void
541 541 fss_insert_fssproj(fsspset_t *fsspset, kproject_t *kpj, fsszone_t *fsszone,
542 542 fssproj_t *fssproj)
543 543 {
544 544 ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
545 545
546 546 fssproj->fssp_pset = fsspset;
547 547 fssproj->fssp_proj = kpj;
548 548 fssproj->fssp_shares = kpj->kpj_shares;
549 549
550 550 fsspset->fssps_nproj++;
551 551
552 552 if (fsspset->fssps_list == NULL) {
553 553 /*
554 554 * This will be the first fssproj for this fsspset
555 555 */
556 556 fssproj->fssp_next = fssproj->fssp_prev = fssproj;
557 557 fsspset->fssps_list = fssproj;
558 558 } else {
559 559 /*
560 560 * Insert this fssproj to the doubly linked list.
561 561 */
562 562 fssproj_t *fssp_head = fsspset->fssps_list;
563 563
564 564 fssproj->fssp_next = fssp_head;
565 565 fssproj->fssp_prev = fssp_head->fssp_prev;
566 566 fssp_head->fssp_prev->fssp_next = fssproj;
567 567 fssp_head->fssp_prev = fssproj;
568 568 fsspset->fssps_list = fssproj;
569 569 }
570 570 fssproj->fssp_fsszone = fsszone;
571 571 fsszone->fssz_nproj++;
572 572 ASSERT(fsszone->fssz_nproj != 0);
573 573 }
574 574
575 575 /*
576 576 * The following routine removes a single fssproj structure from the doubly
577 577 * linked list of projects running on the specified cpu partition. Note that
578 578 * global fsspsets_lock must be held in case if this fssproj structure is the
579 579 * last on the above mentioned list. Also note that the fssproj structure is
580 580 * not freed here, it is the responsibility of the caller to call kmem_free
581 581 * for it.
582 582 */
583 583 static void
584 584 fss_remove_fssproj(fsspset_t *fsspset, fssproj_t *fssproj)
585 585 {
586 586 fsszone_t *fsszone;
587 587
588 588 ASSERT(MUTEX_HELD(&fsspsets_lock));
589 589 ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
590 590 ASSERT(fssproj->fssp_runnable == 0);
591 591
592 592 fsspset->fssps_nproj--;
593 593
594 594 fsszone = fssproj->fssp_fsszone;
595 595 fsszone->fssz_nproj--;
596 596
597 597 if (fssproj->fssp_next != fssproj) {
598 598 /*
599 599 * This is not the last part in the list.
600 600 */
601 601 fssproj->fssp_prev->fssp_next = fssproj->fssp_next;
602 602 fssproj->fssp_next->fssp_prev = fssproj->fssp_prev;
603 603 if (fsspset->fssps_list == fssproj)
604 604 fsspset->fssps_list = fssproj->fssp_next;
605 605 if (fsszone->fssz_nproj == 0)
606 606 fss_remove_fsszone(fsspset, fsszone);
607 607 } else {
608 608 /*
609 609 * This was the last project part running
610 610 * at this cpu partition.
611 611 */
612 612 fsspset->fssps_list = NULL;
613 613 ASSERT(fsspset->fssps_nproj == 0);
614 614 ASSERT(fsszone->fssz_nproj == 0);
615 615 fss_remove_fsszone(fsspset, fsszone);
616 616 fss_del_fsspset(fsspset);
617 617 }
618 618 }
619 619
620 620 static void
621 621 fss_inactive(kthread_t *t)
622 622 {
623 623 fssproc_t *fssproc;
624 624 fssproj_t *fssproj;
625 625 fsspset_t *fsspset;
626 626 fsszone_t *fsszone;
627 627
628 628 ASSERT(THREAD_LOCK_HELD(t));
629 629 fssproc = FSSPROC(t);
630 630 fssproj = FSSPROC2FSSPROJ(fssproc);
631 631 if (fssproj == NULL) /* if this thread already exited */
632 632 return;
633 633 fsspset = FSSPROJ2FSSPSET(fssproj);
634 634 fsszone = fssproj->fssp_fsszone;
635 635 disp_lock_enter_high(&fsspset->fssps_displock);
636 636 ASSERT(fssproj->fssp_runnable > 0);
637 637 if (--fssproj->fssp_runnable == 0) {
638 638 fsszone->fssz_shares -= fssproj->fssp_shares;
639 639 if (--fsszone->fssz_runnable == 0)
640 640 fsspset->fssps_shares -= fsszone->fssz_rshares;
641 641 }
642 642 ASSERT(fssproc->fss_runnable == 1);
643 643 fssproc->fss_runnable = 0;
644 644 disp_lock_exit_high(&fsspset->fssps_displock);
645 645 }
646 646
647 647 static void
648 648 fss_active(kthread_t *t)
649 649 {
650 650 fssproc_t *fssproc;
651 651 fssproj_t *fssproj;
652 652 fsspset_t *fsspset;
653 653 fsszone_t *fsszone;
654 654
655 655 ASSERT(THREAD_LOCK_HELD(t));
656 656 fssproc = FSSPROC(t);
657 657 fssproj = FSSPROC2FSSPROJ(fssproc);
658 658 if (fssproj == NULL) /* if this thread already exited */
659 659 return;
660 660 fsspset = FSSPROJ2FSSPSET(fssproj);
661 661 fsszone = fssproj->fssp_fsszone;
662 662 disp_lock_enter_high(&fsspset->fssps_displock);
663 663 if (++fssproj->fssp_runnable == 1) {
664 664 fsszone->fssz_shares += fssproj->fssp_shares;
665 665 if (++fsszone->fssz_runnable == 1)
666 666 fsspset->fssps_shares += fsszone->fssz_rshares;
667 667 }
668 668 ASSERT(fssproc->fss_runnable == 0);
669 669 fssproc->fss_runnable = 1;
670 670 disp_lock_exit_high(&fsspset->fssps_displock);
671 671 }
672 672
673 673 /*
674 674 * Fair share scheduler initialization. Called by dispinit() at boot time.
675 675 * We can ignore clparmsz argument since we know that the smallest possible
676 676 * parameter buffer is big enough for us.
677 677 */
678 678 /*ARGSUSED*/
679 679 static pri_t
680 680 fss_init(id_t cid, int clparmsz, classfuncs_t **clfuncspp)
681 681 {
682 682 int i;
683 683
684 684 ASSERT(MUTEX_HELD(&cpu_lock));
685 685
686 686 fss_cid = cid;
687 687 fss_maxumdpri = minclsyspri - 1;
688 688 fss_maxglobpri = minclsyspri;
689 689 fss_minglobpri = 0;
690 690 fsspsets = kmem_zalloc(sizeof (fsspset_t) * max_ncpus, KM_SLEEP);
691 691
692 692 /*
693 693 * Initialize the fssproc hash table.
694 694 */
695 695 for (i = 0; i < FSS_LISTS; i++)
696 696 fss_listhead[i].fss_next = fss_listhead[i].fss_prev =
697 697 &fss_listhead[i];
698 698
699 699 *clfuncspp = &fss_classfuncs;
700 700
701 701 /*
702 702 * Fill in fss_nice_tick and fss_nice_decay arrays:
703 703 * The cost of a tick is lower at positive nice values (so that it
704 704 * will not increase its project's usage as much as normal) with 50%
705 705 * drop at the maximum level and 50% increase at the minimum level.
706 706 * The fsspri decay is slower at positive nice values. fsspri values
707 707 * of processes with negative nice levels must decay faster to receive
708 708 * time slices more frequently than normal.
709 709 */
710 710 for (i = 0; i < FSS_NICE_RANGE; i++) {
711 711 fss_nice_tick[i] = (FSS_TICK_COST * (((3 * FSS_NICE_RANGE) / 2)
712 712 - i)) / FSS_NICE_RANGE;
713 713 fss_nice_decay[i] = FSS_DECAY_MIN +
714 714 ((FSS_DECAY_MAX - FSS_DECAY_MIN) * i) /
715 715 (FSS_NICE_RANGE - 1);
716 716 }
717 717
718 718 return (fss_maxglobpri);
719 719 }
720 720
721 721 /*
722 722 * Calculate the new cpupri based on the usage, the number of shares and
723 723 * the number of active threads. Reset the tick counter for this thread.
724 724 */
725 725 static void
726 726 fss_newpri(fssproc_t *fssproc)
727 727 {
728 728 kthread_t *tp;
729 729 fssproj_t *fssproj;
730 730 fsspset_t *fsspset;
731 731 fsszone_t *fsszone;
732 732 fsspri_t fsspri, maxfsspri;
733 733 pri_t invpri;
734 734 uint32_t ticks;
735 735
736 736 tp = fssproc->fss_tp;
737 737 ASSERT(tp != NULL);
738 738
739 739 if (tp->t_cid != fss_cid)
740 740 return;
741 741
742 742 ASSERT(THREAD_LOCK_HELD(tp));
743 743
744 744 fssproj = FSSPROC2FSSPROJ(fssproc);
745 745 fsszone = FSSPROJ2FSSZONE(fssproj);
746 746 if (fssproj == NULL)
747 747 /*
748 748 * No need to change priority of exited threads.
749 749 */
750 750 return;
751 751
752 752 fsspset = FSSPROJ2FSSPSET(fssproj);
753 753 disp_lock_enter_high(&fsspset->fssps_displock);
754 754
755 755 if (fssproj->fssp_shares == 0 || fsszone->fssz_rshares == 0) {
756 756 /*
757 757 * Special case: threads with no shares.
758 758 */
759 759 fssproc->fss_umdpri = fss_minglobpri;
760 760 fssproc->fss_ticks = 0;
761 761 disp_lock_exit_high(&fsspset->fssps_displock);
762 762 return;
763 763 }
764 764
765 765 /*
766 766 * fsspri += shusage * nrunnable * ticks
767 767 */
768 768 ticks = fssproc->fss_ticks;
769 769 fssproc->fss_ticks = 0;
770 770 fsspri = fssproc->fss_fsspri;
771 771 fsspri += fssproj->fssp_shusage * fssproj->fssp_runnable * ticks;
772 772 fssproc->fss_fsspri = fsspri;
773 773
774 774 if (fsspri < fss_maxumdpri)
775 775 fsspri = fss_maxumdpri; /* so that maxfsspri is != 0 */
776 776
777 777 /*
778 778 * The general priority formula:
779 779 *
780 780 * (fsspri * umdprirange)
781 781 * pri = maxumdpri - ------------------------
782 782 * maxfsspri
783 783 *
784 784 * If this thread's fsspri is greater than the previous largest
785 785 * fsspri, then record it as the new high and priority for this
786 786 * thread will be one (the lowest priority assigned to a thread
787 787 * that has non-zero shares).
788 788 * Note that this formula cannot produce out of bounds priority
789 789 * values; if it is changed, additional checks may need to be
790 790 * added.
791 791 */
792 792 maxfsspri = fsspset->fssps_maxfsspri;
793 793 if (fsspri >= maxfsspri) {
794 794 fsspset->fssps_maxfsspri = fsspri;
795 795 disp_lock_exit_high(&fsspset->fssps_displock);
796 796 fssproc->fss_umdpri = 1;
797 797 } else {
798 798 disp_lock_exit_high(&fsspset->fssps_displock);
799 799 invpri = (fsspri * (fss_maxumdpri - 1)) / maxfsspri;
800 800 fssproc->fss_umdpri = fss_maxumdpri - invpri;
801 801 }
802 802 }
803 803
804 804 /*
805 805 * Decays usages of all running projects and resets their tick counters.
806 806 * Called once per second from fss_update() after updating priorities.
807 807 */
808 808 static void
809 809 fss_decay_usage()
810 810 {
811 811 uint32_t zone_ext_shares, zone_int_shares;
812 812 uint32_t kpj_shares, pset_shares;
813 813 fsspset_t *fsspset;
814 814 fssproj_t *fssproj;
815 815 fsszone_t *fsszone;
816 816 fsspri_t maxfsspri;
817 817 int psetid;
818 818
819 819 mutex_enter(&fsspsets_lock);
820 820 /*
821 821 * Go through all active processor sets and decay usages of projects
822 822 * running on them.
823 823 */
824 824 for (psetid = 0; psetid < max_ncpus; psetid++) {
825 825 fsspset = &fsspsets[psetid];
826 826 mutex_enter(&fsspset->fssps_lock);
827 827
828 828 if (fsspset->fssps_cpupart == NULL ||
829 829 (fssproj = fsspset->fssps_list) == NULL) {
830 830 mutex_exit(&fsspset->fssps_lock);
831 831 continue;
832 832 }
833 833
834 834 /*
835 835 * Decay maxfsspri for this cpu partition with the
836 836 * fastest possible decay rate.
837 837 */
838 838 disp_lock_enter(&fsspset->fssps_displock);
839 839
840 840 maxfsspri = (fsspset->fssps_maxfsspri *
841 841 fss_nice_decay[NZERO]) / FSS_DECAY_BASE;
842 842 if (maxfsspri < fss_maxumdpri)
843 843 maxfsspri = fss_maxumdpri;
844 844 fsspset->fssps_maxfsspri = maxfsspri;
845 845
846 846 do {
847 847 /*
848 848 * Decay usage for each project running on
849 849 * this cpu partition.
850 850 */
851 851 fssproj->fssp_usage =
852 852 (fssproj->fssp_usage * FSS_DECAY_USG) /
853 853 FSS_DECAY_BASE + fssproj->fssp_ticks;
854 854 fssproj->fssp_ticks = 0;
855 855
856 856 fsszone = fssproj->fssp_fsszone;
857 857 /*
858 858 * Readjust the project's number of shares if it has
859 859 * changed since we checked it last time.
860 860 */
861 861 kpj_shares = fssproj->fssp_proj->kpj_shares;
862 862 if (fssproj->fssp_shares != kpj_shares) {
863 863 if (fssproj->fssp_runnable != 0) {
864 864 fsszone->fssz_shares -=
865 865 fssproj->fssp_shares;
866 866 fsszone->fssz_shares += kpj_shares;
867 867 }
868 868 fssproj->fssp_shares = kpj_shares;
869 869 }
870 870
871 871 /*
872 872 * Readjust the zone's number of shares if it
873 873 * has changed since we checked it last time.
874 874 */
875 875 zone_ext_shares = fsszone->fssz_zone->zone_shares;
876 876 if (fsszone->fssz_rshares != zone_ext_shares) {
877 877 if (fsszone->fssz_runnable != 0) {
878 878 fsspset->fssps_shares -=
879 879 fsszone->fssz_rshares;
880 880 fsspset->fssps_shares +=
881 881 zone_ext_shares;
882 882 }
883 883 fsszone->fssz_rshares = zone_ext_shares;
884 884 }
885 885 zone_int_shares = fsszone->fssz_shares;
886 886 pset_shares = fsspset->fssps_shares;
887 887 /*
888 888 * Calculate fssp_shusage value to be used
889 889 * for fsspri increments for the next second.
890 890 */
891 891 if (kpj_shares == 0 || zone_ext_shares == 0) {
892 892 fssproj->fssp_shusage = 0;
893 893 } else if (FSSPROJ2KPROJ(fssproj) == proj0p) {
894 894 /*
895 895 * Project 0 in the global zone has 50%
896 896 * of its zone.
897 897 */
898 898 fssproj->fssp_shusage = (fssproj->fssp_usage *
899 899 zone_int_shares * zone_int_shares) /
900 900 (zone_ext_shares * zone_ext_shares);
901 901 } else {
902 902 /*
903 903 * Thread's priority is based on its project's
904 904 * normalized usage (shusage) value which gets
905 905 * calculated this way:
906 906 *
907 907 * pset_shares^2 zone_int_shares^2
908 908 * usage * ------------- * ------------------
909 909 * kpj_shares^2 zone_ext_shares^2
910 910 *
911 911 * Where zone_int_shares is the sum of shares
912 912 * of all active projects within the zone (and
913 913 * the pset), and zone_ext_shares is the number
914 914 * of zone shares (ie, zone.cpu-shares).
915 915 *
916 916 * If there is only one zone active on the pset
917 917 * the above reduces to:
918 918 *
919 919 * zone_int_shares^2
920 920 * shusage = usage * ---------------------
921 921 * kpj_shares^2
922 922 *
923 923 * If there's only one project active in the
924 924 * zone this formula reduces to:
925 925 *
926 926 * pset_shares^2
927 927 * shusage = usage * ----------------------
928 928 * zone_ext_shares^2
929 929 */
930 930 fssproj->fssp_shusage = fssproj->fssp_usage *
931 931 pset_shares * zone_int_shares;
932 932 fssproj->fssp_shusage /=
933 933 kpj_shares * zone_ext_shares;
934 934 fssproj->fssp_shusage *=
935 935 pset_shares * zone_int_shares;
936 936 fssproj->fssp_shusage /=
937 937 kpj_shares * zone_ext_shares;
938 938 }
939 939 fssproj = fssproj->fssp_next;
940 940 } while (fssproj != fsspset->fssps_list);
941 941
942 942 disp_lock_exit(&fsspset->fssps_displock);
943 943 mutex_exit(&fsspset->fssps_lock);
944 944 }
945 945 mutex_exit(&fsspsets_lock);
946 946 }
947 947
948 948 static void
949 949 fss_change_priority(kthread_t *t, fssproc_t *fssproc)
950 950 {
951 951 pri_t new_pri;
952 952
953 953 ASSERT(THREAD_LOCK_HELD(t));
954 954 new_pri = fssproc->fss_umdpri;
955 955 ASSERT(new_pri >= 0 && new_pri <= fss_maxglobpri);
956 956
957 957 t->t_cpri = fssproc->fss_upri;
958 958 fssproc->fss_flags &= ~FSSRESTORE;
959 959 if (t == curthread || t->t_state == TS_ONPROC) {
960 960 /*
961 961 * curthread is always onproc
962 962 */
963 963 cpu_t *cp = t->t_disp_queue->disp_cpu;
964 964 THREAD_CHANGE_PRI(t, new_pri);
965 965 if (t == cp->cpu_dispthread)
966 966 cp->cpu_dispatch_pri = DISP_PRIO(t);
967 967 if (DISP_MUST_SURRENDER(t)) {
968 968 fssproc->fss_flags |= FSSBACKQ;
969 969 cpu_surrender(t);
970 970 } else {
971 971 fssproc->fss_timeleft = fss_quantum;
972 972 }
973 973 } else {
974 974 /*
975 975 * When the priority of a thread is changed, it may be
976 976 * necessary to adjust its position on a sleep queue or
977 977 * dispatch queue. The function thread_change_pri accomplishes
978 978 * this.
979 979 */
980 980 if (thread_change_pri(t, new_pri, 0)) {
981 981 /*
982 982 * The thread was on a run queue.
983 983 */
984 984 fssproc->fss_timeleft = fss_quantum;
985 985 } else {
986 986 fssproc->fss_flags |= FSSBACKQ;
987 987 }
988 988 }
989 989 }
990 990
991 991 /*
992 992 * Update priorities of all fair-sharing threads that are currently runnable
993 993 * at a user mode priority based on the number of shares and current usage.
994 994 * Called once per second via timeout which we reset here.
995 995 *
996 996 * There are several lists of fair-sharing threads broken up by a hash on the
997 997 * thread pointer. Each list has its own lock. This avoids blocking all
998 998 * fss_enterclass, fss_fork, and fss_exitclass operations while fss_update runs.
999 999 * fss_update traverses each list in turn.
1000 1000 */
1001 1001 static void
1002 1002 fss_update(void *arg)
1003 1003 {
1004 1004 int i;
1005 1005 int new_marker = -1;
1006 1006 static int fss_update_marker;
1007 1007
1008 1008 /*
1009 1009 * Decay and update usages for all projects.
1010 1010 */
1011 1011 fss_decay_usage();
1012 1012
1013 1013 /*
1014 1014 * Start with the fss_update_marker list, then do the rest.
1015 1015 */
1016 1016 i = fss_update_marker;
1017 1017
1018 1018 /*
1019 1019 * Go around all threads, set new priorities and decay
1020 1020 * per-thread CPU usages.
1021 1021 */
1022 1022 do {
1023 1023 /*
1024 1024 * If this is the first list after the current marker to have
1025 1025 * threads with priorities updates, advance the marker to this
1026 1026 * list for the next time fss_update runs.
1027 1027 */
1028 1028 if (fss_update_list(i) &&
1029 1029 new_marker == -1 && i != fss_update_marker)
1030 1030 new_marker = i;
1031 1031 } while ((i = FSS_LIST_NEXT(i)) != fss_update_marker);
1032 1032
1033 1033 /*
1034 1034 * Advance marker for the next fss_update call
1035 1035 */
1036 1036 if (new_marker != -1)
1037 1037 fss_update_marker = new_marker;
1038 1038
1039 1039 (void) timeout(fss_update, arg, hz);
1040 1040 }
1041 1041
1042 1042 /*
1043 1043 * Updates priority for a list of threads. Returns 1 if the priority of one
1044 1044 * of the threads was actually updated, 0 if none were for various reasons
1045 1045 * (thread is no longer in the FSS class, is not runnable, has the preemption
1046 1046 * control no-preempt bit set, etc.)
1047 1047 */
1048 1048 static int
1049 1049 fss_update_list(int i)
1050 1050 {
1051 1051 fssproc_t *fssproc;
1052 1052 fssproj_t *fssproj;
1053 1053 fsspri_t fsspri;
1054 1054 kthread_t *t;
1055 1055 int updated = 0;
1056 1056
1057 1057 mutex_enter(&fss_listlock[i]);
1058 1058 for (fssproc = fss_listhead[i].fss_next; fssproc != &fss_listhead[i];
1059 1059 fssproc = fssproc->fss_next) {
1060 1060 t = fssproc->fss_tp;
1061 1061 /*
1062 1062 * Lock the thread and verify the state.
1063 1063 */
1064 1064 thread_lock(t);
1065 1065 /*
1066 1066 * Skip the thread if it is no longer in the FSS class or
1067 1067 * is running with kernel mode priority.
1068 1068 */
1069 1069 if (t->t_cid != fss_cid)
1070 1070 goto next;
1071 1071 if ((fssproc->fss_flags & FSSKPRI) != 0)
1072 1072 goto next;
1073 1073
1074 1074 fssproj = FSSPROC2FSSPROJ(fssproc);
1075 1075 if (fssproj == NULL)
1076 1076 goto next;
1077 1077 if (fssproj->fssp_shares != 0) {
1078 1078 /*
1079 1079 * Decay fsspri value.
1080 1080 */
1081 1081 fsspri = fssproc->fss_fsspri;
1082 1082 fsspri = (fsspri * fss_nice_decay[fssproc->fss_nice]) /
1083 1083 FSS_DECAY_BASE;
1084 1084 fssproc->fss_fsspri = fsspri;
1085 1085 }
1086 1086
1087 1087 if (t->t_schedctl && schedctl_get_nopreempt(t))
1088 1088 goto next;
1089 1089 if (t->t_state != TS_RUN && t->t_state != TS_WAIT) {
1090 1090 /*
1091 1091 * Make next syscall/trap call fss_trapret
1092 1092 */
1093 1093 t->t_trapret = 1;
1094 1094 aston(t);
1095 1095 goto next;
1096 1096 }
1097 1097 fss_newpri(fssproc);
1098 1098 updated = 1;
1099 1099
1100 1100 /*
1101 1101 * Only dequeue the thread if it needs to be moved; otherwise
1102 1102 * it should just round-robin here.
1103 1103 */
1104 1104 if (t->t_pri != fssproc->fss_umdpri)
1105 1105 fss_change_priority(t, fssproc);
1106 1106 next:
1107 1107 thread_unlock(t);
1108 1108 }
1109 1109 mutex_exit(&fss_listlock[i]);
1110 1110 return (updated);
1111 1111 }
1112 1112
1113 1113 /*ARGSUSED*/
1114 1114 static int
1115 1115 fss_admin(caddr_t uaddr, cred_t *reqpcredp)
1116 1116 {
1117 1117 fssadmin_t fssadmin;
1118 1118
1119 1119 if (copyin(uaddr, &fssadmin, sizeof (fssadmin_t)))
1120 1120 return (EFAULT);
1121 1121
1122 1122 switch (fssadmin.fss_cmd) {
1123 1123 case FSS_SETADMIN:
1124 1124 if (secpolicy_dispadm(reqpcredp) != 0)
1125 1125 return (EPERM);
1126 1126 if (fssadmin.fss_quantum <= 0 || fssadmin.fss_quantum >= hz)
1127 1127 return (EINVAL);
1128 1128 fss_quantum = fssadmin.fss_quantum;
1129 1129 break;
1130 1130 case FSS_GETADMIN:
1131 1131 fssadmin.fss_quantum = fss_quantum;
1132 1132 if (copyout(&fssadmin, uaddr, sizeof (fssadmin_t)))
1133 1133 return (EFAULT);
1134 1134 break;
1135 1135 default:
1136 1136 return (EINVAL);
1137 1137 }
1138 1138 return (0);
1139 1139 }
1140 1140
1141 1141 static int
1142 1142 fss_getclinfo(void *infop)
1143 1143 {
1144 1144 fssinfo_t *fssinfo = (fssinfo_t *)infop;
1145 1145 fssinfo->fss_maxupri = fss_maxupri;
1146 1146 return (0);
1147 1147 }
1148 1148
1149 1149 static int
1150 1150 fss_parmsin(void *parmsp)
1151 1151 {
1152 1152 fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1153 1153
1154 1154 /*
1155 1155 * Check validity of parameters.
1156 1156 */
1157 1157 if ((fssparmsp->fss_uprilim > fss_maxupri ||
1158 1158 fssparmsp->fss_uprilim < -fss_maxupri) &&
1159 1159 fssparmsp->fss_uprilim != FSS_NOCHANGE)
1160 1160 return (EINVAL);
1161 1161
1162 1162 if ((fssparmsp->fss_upri > fss_maxupri ||
1163 1163 fssparmsp->fss_upri < -fss_maxupri) &&
1164 1164 fssparmsp->fss_upri != FSS_NOCHANGE)
1165 1165 return (EINVAL);
1166 1166
1167 1167 return (0);
1168 1168 }
1169 1169
1170 1170 /*ARGSUSED*/
1171 1171 static int
1172 1172 fss_parmsout(void *parmsp, pc_vaparms_t *vaparmsp)
1173 1173 {
1174 1174 return (0);
1175 1175 }
1176 1176
1177 1177 static int
1178 1178 fss_vaparmsin(void *parmsp, pc_vaparms_t *vaparmsp)
1179 1179 {
1180 1180 fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1181 1181 int priflag = 0;
1182 1182 int limflag = 0;
1183 1183 uint_t cnt;
1184 1184 pc_vaparm_t *vpp = &vaparmsp->pc_parms[0];
1185 1185
1186 1186 /*
1187 1187 * FSS_NOCHANGE (-32768) is outside of the range of values for
1188 1188 * fss_uprilim and fss_upri. If the structure fssparms_t is changed,
1189 1189 * FSS_NOCHANGE should be replaced by a flag word.
1190 1190 */
1191 1191 fssparmsp->fss_uprilim = FSS_NOCHANGE;
1192 1192 fssparmsp->fss_upri = FSS_NOCHANGE;
1193 1193
1194 1194 /*
1195 1195 * Get the varargs parameter and check validity of parameters.
1196 1196 */
1197 1197 if (vaparmsp->pc_vaparmscnt > PC_VAPARMCNT)
1198 1198 return (EINVAL);
1199 1199
1200 1200 for (cnt = 0; cnt < vaparmsp->pc_vaparmscnt; cnt++, vpp++) {
1201 1201 switch (vpp->pc_key) {
1202 1202 case FSS_KY_UPRILIM:
1203 1203 if (limflag++)
1204 1204 return (EINVAL);
1205 1205 fssparmsp->fss_uprilim = (pri_t)vpp->pc_parm;
1206 1206 if (fssparmsp->fss_uprilim > fss_maxupri ||
1207 1207 fssparmsp->fss_uprilim < -fss_maxupri)
1208 1208 return (EINVAL);
1209 1209 break;
1210 1210 case FSS_KY_UPRI:
1211 1211 if (priflag++)
1212 1212 return (EINVAL);
1213 1213 fssparmsp->fss_upri = (pri_t)vpp->pc_parm;
1214 1214 if (fssparmsp->fss_upri > fss_maxupri ||
1215 1215 fssparmsp->fss_upri < -fss_maxupri)
1216 1216 return (EINVAL);
1217 1217 break;
1218 1218 default:
1219 1219 return (EINVAL);
1220 1220 }
1221 1221 }
1222 1222
1223 1223 if (vaparmsp->pc_vaparmscnt == 0) {
1224 1224 /*
1225 1225 * Use default parameters.
1226 1226 */
1227 1227 fssparmsp->fss_upri = fssparmsp->fss_uprilim = 0;
1228 1228 }
1229 1229
1230 1230 return (0);
1231 1231 }
1232 1232
1233 1233 /*
1234 1234 * Copy all selected fair-sharing class parameters to the user. The parameters
1235 1235 * are specified by a key.
1236 1236 */
1237 1237 static int
1238 1238 fss_vaparmsout(void *parmsp, pc_vaparms_t *vaparmsp)
1239 1239 {
1240 1240 fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1241 1241 int priflag = 0;
1242 1242 int limflag = 0;
1243 1243 uint_t cnt;
1244 1244 pc_vaparm_t *vpp = &vaparmsp->pc_parms[0];
1245 1245
1246 1246 ASSERT(MUTEX_NOT_HELD(&curproc->p_lock));
1247 1247
1248 1248 if (vaparmsp->pc_vaparmscnt > PC_VAPARMCNT)
1249 1249 return (EINVAL);
1250 1250
1251 1251 for (cnt = 0; cnt < vaparmsp->pc_vaparmscnt; cnt++, vpp++) {
1252 1252 switch (vpp->pc_key) {
1253 1253 case FSS_KY_UPRILIM:
1254 1254 if (limflag++)
1255 1255 return (EINVAL);
1256 1256 if (copyout(&fssparmsp->fss_uprilim,
1257 1257 (caddr_t)(uintptr_t)vpp->pc_parm, sizeof (pri_t)))
1258 1258 return (EFAULT);
1259 1259 break;
1260 1260 case FSS_KY_UPRI:
1261 1261 if (priflag++)
1262 1262 return (EINVAL);
1263 1263 if (copyout(&fssparmsp->fss_upri,
1264 1264 (caddr_t)(uintptr_t)vpp->pc_parm, sizeof (pri_t)))
1265 1265 return (EFAULT);
1266 1266 break;
1267 1267 default:
1268 1268 return (EINVAL);
1269 1269 }
1270 1270 }
1271 1271
1272 1272 return (0);
1273 1273 }
1274 1274
1275 1275 /*
1276 1276 * Return the user mode scheduling priority range.
1277 1277 */
1278 1278 static int
1279 1279 fss_getclpri(pcpri_t *pcprip)
1280 1280 {
1281 1281 pcprip->pc_clpmax = fss_maxupri;
1282 1282 pcprip->pc_clpmin = -fss_maxupri;
1283 1283 return (0);
1284 1284 }
1285 1285
1286 1286 static int
1287 1287 fss_alloc(void **p, int flag)
1288 1288 {
1289 1289 void *bufp;
1290 1290
1291 1291 if ((bufp = kmem_zalloc(sizeof (fssproc_t), flag)) == NULL) {
1292 1292 return (ENOMEM);
1293 1293 } else {
1294 1294 *p = bufp;
1295 1295 return (0);
1296 1296 }
1297 1297 }
1298 1298
1299 1299 static void
1300 1300 fss_free(void *bufp)
1301 1301 {
1302 1302 if (bufp)
1303 1303 kmem_free(bufp, sizeof (fssproc_t));
1304 1304 }
1305 1305
1306 1306 /*
1307 1307 * Thread functions
1308 1308 */
1309 1309 static int
1310 1310 fss_enterclass(kthread_t *t, id_t cid, void *parmsp, cred_t *reqpcredp,
1311 1311 void *bufp)
1312 1312 {
1313 1313 fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1314 1314 fssproc_t *fssproc;
1315 1315 pri_t reqfssuprilim;
1316 1316 pri_t reqfssupri;
1317 1317 static uint32_t fssexists = 0;
1318 1318 fsspset_t *fsspset;
1319 1319 fssproj_t *fssproj;
1320 1320 fsszone_t *fsszone;
1321 1321 kproject_t *kpj;
1322 1322 zone_t *zone;
1323 1323 int fsszone_allocated = 0;
1324 1324
1325 1325 fssproc = (fssproc_t *)bufp;
1326 1326 ASSERT(fssproc != NULL);
1327 1327
1328 1328 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
1329 1329
1330 1330 /*
1331 1331 * Only root can move threads to FSS class.
1332 1332 */
1333 1333 if (reqpcredp != NULL && secpolicy_setpriority(reqpcredp) != 0)
1334 1334 return (EPERM);
1335 1335 /*
1336 1336 * Initialize the fssproc structure.
1337 1337 */
1338 1338 fssproc->fss_umdpri = fss_maxumdpri / 2;
1339 1339
1340 1340 if (fssparmsp == NULL) {
1341 1341 /*
1342 1342 * Use default values.
1343 1343 */
1344 1344 fssproc->fss_nice = NZERO;
1345 1345 fssproc->fss_uprilim = fssproc->fss_upri = 0;
1346 1346 } else {
1347 1347 /*
1348 1348 * Use supplied values.
1349 1349 */
1350 1350 if (fssparmsp->fss_uprilim == FSS_NOCHANGE) {
1351 1351 reqfssuprilim = 0;
1352 1352 } else {
1353 1353 if (fssparmsp->fss_uprilim > 0 &&
1354 1354 secpolicy_setpriority(reqpcredp) != 0)
1355 1355 return (EPERM);
1356 1356 reqfssuprilim = fssparmsp->fss_uprilim;
1357 1357 }
1358 1358 if (fssparmsp->fss_upri == FSS_NOCHANGE) {
1359 1359 reqfssupri = reqfssuprilim;
1360 1360 } else {
1361 1361 if (fssparmsp->fss_upri > 0 &&
1362 1362 secpolicy_setpriority(reqpcredp) != 0)
1363 1363 return (EPERM);
1364 1364 /*
1365 1365 * Set the user priority to the requested value or
1366 1366 * the upri limit, whichever is lower.
1367 1367 */
1368 1368 reqfssupri = fssparmsp->fss_upri;
1369 1369 if (reqfssupri > reqfssuprilim)
1370 1370 reqfssupri = reqfssuprilim;
1371 1371 }
1372 1372 fssproc->fss_uprilim = reqfssuprilim;
1373 1373 fssproc->fss_upri = reqfssupri;
1374 1374 fssproc->fss_nice = NZERO - (NZERO * reqfssupri) / fss_maxupri;
1375 1375 if (fssproc->fss_nice > FSS_NICE_MAX)
1376 1376 fssproc->fss_nice = FSS_NICE_MAX;
1377 1377 }
1378 1378
1379 1379 fssproc->fss_timeleft = fss_quantum;
1380 1380 fssproc->fss_tp = t;
1381 1381 cpucaps_sc_init(&fssproc->fss_caps);
1382 1382
1383 1383 /*
1384 1384 * Put a lock on our fsspset structure.
1385 1385 */
1386 1386 mutex_enter(&fsspsets_lock);
1387 1387 fsspset = fss_find_fsspset(t->t_cpupart);
1388 1388 mutex_enter(&fsspset->fssps_lock);
1389 1389 mutex_exit(&fsspsets_lock);
1390 1390
1391 1391 zone = ttoproc(t)->p_zone;
1392 1392 if ((fsszone = fss_find_fsszone(fsspset, zone)) == NULL) {
1393 1393 if ((fsszone = kmem_zalloc(sizeof (fsszone_t), KM_NOSLEEP))
1394 1394 == NULL) {
1395 1395 mutex_exit(&fsspset->fssps_lock);
1396 1396 return (ENOMEM);
1397 1397 } else {
1398 1398 fsszone_allocated = 1;
1399 1399 fss_insert_fsszone(fsspset, zone, fsszone);
1400 1400 }
1401 1401 }
1402 1402 kpj = ttoproj(t);
1403 1403 if ((fssproj = fss_find_fssproj(fsspset, kpj)) == NULL) {
1404 1404 if ((fssproj = kmem_zalloc(sizeof (fssproj_t), KM_NOSLEEP))
1405 1405 == NULL) {
1406 1406 if (fsszone_allocated) {
1407 1407 fss_remove_fsszone(fsspset, fsszone);
1408 1408 kmem_free(fsszone, sizeof (fsszone_t));
1409 1409 }
1410 1410 mutex_exit(&fsspset->fssps_lock);
1411 1411 return (ENOMEM);
1412 1412 } else {
1413 1413 fss_insert_fssproj(fsspset, kpj, fsszone, fssproj);
1414 1414 }
1415 1415 }
1416 1416 fssproj->fssp_threads++;
1417 1417 fssproc->fss_proj = fssproj;
1418 1418
1419 1419 /*
1420 1420 * Reset priority. Process goes to a "user mode" priority here
1421 1421 * regardless of whether or not it has slept since entering the kernel.
1422 1422 */
1423 1423 thread_lock(t);
1424 1424 t->t_clfuncs = &(sclass[cid].cl_funcs->thread);
1425 1425 t->t_cid = cid;
1426 1426 t->t_cldata = (void *)fssproc;
1427 1427 t->t_schedflag |= TS_RUNQMATCH;
1428 1428 fss_change_priority(t, fssproc);
1429 1429 if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
1430 1430 t->t_state == TS_WAIT)
1431 1431 fss_active(t);
1432 1432 thread_unlock(t);
1433 1433
1434 1434 mutex_exit(&fsspset->fssps_lock);
1435 1435
1436 1436 /*
1437 1437 * Link new structure into fssproc list.
↓ open down ↓ |
1437 lines elided |
↑ open up ↑ |
1438 1438 */
1439 1439 FSS_LIST_INSERT(fssproc);
1440 1440
1441 1441 /*
1442 1442 * If this is the first fair-sharing thread to occur since boot,
1443 1443 * we set up the initial call to fss_update() here. Use an atomic
1444 1444 * compare-and-swap since that's easier and faster than a mutex
1445 1445 * (but check with an ordinary load first since most of the time
1446 1446 * this will already be done).
1447 1447 */
1448 - if (fssexists == 0 && cas32(&fssexists, 0, 1) == 0)
1448 + if (fssexists == 0 && atomic_cas_32(&fssexists, 0, 1) == 0)
1449 1449 (void) timeout(fss_update, NULL, hz);
1450 1450
1451 1451 return (0);
1452 1452 }
1453 1453
1454 1454 /*
1455 1455 * Remove fssproc_t from the list.
1456 1456 */
1457 1457 static void
1458 1458 fss_exitclass(void *procp)
1459 1459 {
1460 1460 fssproc_t *fssproc = (fssproc_t *)procp;
1461 1461 fssproj_t *fssproj;
1462 1462 fsspset_t *fsspset;
1463 1463 fsszone_t *fsszone;
1464 1464 kthread_t *t = fssproc->fss_tp;
1465 1465
1466 1466 /*
1467 1467 * We should be either getting this thread off the deathrow or
1468 1468 * this thread has already moved to another scheduling class and
1469 1469 * we're being called with its old cldata buffer pointer. In both
1470 1470 * cases, the content of this buffer can not be changed while we're
1471 1471 * here.
1472 1472 */
1473 1473 mutex_enter(&fsspsets_lock);
1474 1474 thread_lock(t);
1475 1475 if (t->t_cid != fss_cid) {
1476 1476 /*
1477 1477 * We're being called as a result of the priocntl() system
1478 1478 * call -- someone is trying to move our thread to another
1479 1479 * scheduling class. We can't call fss_inactive() here
1480 1480 * because our thread's t_cldata pointer already points
1481 1481 * to another scheduling class specific data.
1482 1482 */
1483 1483 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
1484 1484
1485 1485 fssproj = FSSPROC2FSSPROJ(fssproc);
1486 1486 fsspset = FSSPROJ2FSSPSET(fssproj);
1487 1487 fsszone = fssproj->fssp_fsszone;
1488 1488
1489 1489 if (fssproc->fss_runnable) {
1490 1490 disp_lock_enter_high(&fsspset->fssps_displock);
1491 1491 if (--fssproj->fssp_runnable == 0) {
1492 1492 fsszone->fssz_shares -= fssproj->fssp_shares;
1493 1493 if (--fsszone->fssz_runnable == 0)
1494 1494 fsspset->fssps_shares -=
1495 1495 fsszone->fssz_rshares;
1496 1496 }
1497 1497 disp_lock_exit_high(&fsspset->fssps_displock);
1498 1498 }
1499 1499 thread_unlock(t);
1500 1500
1501 1501 mutex_enter(&fsspset->fssps_lock);
1502 1502 if (--fssproj->fssp_threads == 0) {
1503 1503 fss_remove_fssproj(fsspset, fssproj);
1504 1504 if (fsszone->fssz_nproj == 0)
1505 1505 kmem_free(fsszone, sizeof (fsszone_t));
1506 1506 kmem_free(fssproj, sizeof (fssproj_t));
1507 1507 }
1508 1508 mutex_exit(&fsspset->fssps_lock);
1509 1509
1510 1510 } else {
1511 1511 ASSERT(t->t_state == TS_FREE);
1512 1512 /*
1513 1513 * We're being called from thread_free() when our thread
1514 1514 * is removed from the deathrow. There is nothing we need
1515 1515 * do here since everything should've been done earlier
1516 1516 * in fss_exit().
1517 1517 */
1518 1518 thread_unlock(t);
1519 1519 }
1520 1520 mutex_exit(&fsspsets_lock);
1521 1521
1522 1522 FSS_LIST_DELETE(fssproc);
1523 1523 fss_free(fssproc);
1524 1524 }
1525 1525
1526 1526 /*ARGSUSED*/
1527 1527 static int
1528 1528 fss_canexit(kthread_t *t, cred_t *credp)
1529 1529 {
1530 1530 /*
1531 1531 * A thread is allowed to exit FSS only if we have sufficient
1532 1532 * privileges.
1533 1533 */
1534 1534 if (credp != NULL && secpolicy_setpriority(credp) != 0)
1535 1535 return (EPERM);
1536 1536 else
1537 1537 return (0);
1538 1538 }
1539 1539
1540 1540 /*
1541 1541 * Initialize fair-share class specific proc structure for a child.
1542 1542 */
1543 1543 static int
1544 1544 fss_fork(kthread_t *pt, kthread_t *ct, void *bufp)
1545 1545 {
1546 1546 fssproc_t *pfssproc; /* ptr to parent's fssproc structure */
1547 1547 fssproc_t *cfssproc; /* ptr to child's fssproc structure */
1548 1548 fssproj_t *fssproj;
1549 1549 fsspset_t *fsspset;
1550 1550
1551 1551 ASSERT(MUTEX_HELD(&ttoproc(pt)->p_lock));
1552 1552 ASSERT(ct->t_state == TS_STOPPED);
1553 1553
1554 1554 cfssproc = (fssproc_t *)bufp;
1555 1555 ASSERT(cfssproc != NULL);
1556 1556 bzero(cfssproc, sizeof (fssproc_t));
1557 1557
1558 1558 thread_lock(pt);
1559 1559 pfssproc = FSSPROC(pt);
1560 1560 fssproj = FSSPROC2FSSPROJ(pfssproc);
1561 1561 fsspset = FSSPROJ2FSSPSET(fssproj);
1562 1562 thread_unlock(pt);
1563 1563
1564 1564 mutex_enter(&fsspset->fssps_lock);
1565 1565 /*
1566 1566 * Initialize child's fssproc structure.
1567 1567 */
1568 1568 thread_lock(pt);
1569 1569 ASSERT(FSSPROJ(pt) == fssproj);
1570 1570 cfssproc->fss_proj = fssproj;
1571 1571 cfssproc->fss_timeleft = fss_quantum;
1572 1572 cfssproc->fss_umdpri = pfssproc->fss_umdpri;
1573 1573 cfssproc->fss_fsspri = 0;
1574 1574 cfssproc->fss_uprilim = pfssproc->fss_uprilim;
1575 1575 cfssproc->fss_upri = pfssproc->fss_upri;
1576 1576 cfssproc->fss_tp = ct;
1577 1577 cfssproc->fss_nice = pfssproc->fss_nice;
1578 1578 cpucaps_sc_init(&cfssproc->fss_caps);
1579 1579
1580 1580 cfssproc->fss_flags =
1581 1581 pfssproc->fss_flags & ~(FSSKPRI | FSSBACKQ | FSSRESTORE);
1582 1582 ct->t_cldata = (void *)cfssproc;
1583 1583 ct->t_schedflag |= TS_RUNQMATCH;
1584 1584 thread_unlock(pt);
1585 1585
1586 1586 fssproj->fssp_threads++;
1587 1587 mutex_exit(&fsspset->fssps_lock);
1588 1588
1589 1589 /*
1590 1590 * Link new structure into fssproc hash table.
1591 1591 */
1592 1592 FSS_LIST_INSERT(cfssproc);
1593 1593 return (0);
1594 1594 }
1595 1595
1596 1596 /*
1597 1597 * Child is placed at back of dispatcher queue and parent gives up processor
1598 1598 * so that the child runs first after the fork. This allows the child
1599 1599 * immediately execing to break the multiple use of copy on write pages with no
1600 1600 * disk home. The parent will get to steal them back rather than uselessly
1601 1601 * copying them.
1602 1602 */
1603 1603 static void
1604 1604 fss_forkret(kthread_t *t, kthread_t *ct)
1605 1605 {
1606 1606 proc_t *pp = ttoproc(t);
1607 1607 proc_t *cp = ttoproc(ct);
1608 1608 fssproc_t *fssproc;
1609 1609
1610 1610 ASSERT(t == curthread);
1611 1611 ASSERT(MUTEX_HELD(&pidlock));
1612 1612
1613 1613 /*
1614 1614 * Grab the child's p_lock before dropping pidlock to ensure the
1615 1615 * process does not disappear before we set it running.
1616 1616 */
1617 1617 mutex_enter(&cp->p_lock);
1618 1618 continuelwps(cp);
1619 1619 mutex_exit(&cp->p_lock);
1620 1620
1621 1621 mutex_enter(&pp->p_lock);
1622 1622 mutex_exit(&pidlock);
1623 1623 continuelwps(pp);
1624 1624
1625 1625 thread_lock(t);
1626 1626
1627 1627 fssproc = FSSPROC(t);
1628 1628 fss_newpri(fssproc);
1629 1629 fssproc->fss_timeleft = fss_quantum;
1630 1630 t->t_pri = fssproc->fss_umdpri;
1631 1631 ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri);
1632 1632 fssproc->fss_flags &= ~FSSKPRI;
1633 1633 THREAD_TRANSITION(t);
1634 1634
1635 1635 /*
1636 1636 * We don't want to call fss_setrun(t) here because it may call
1637 1637 * fss_active, which we don't need.
1638 1638 */
1639 1639 fssproc->fss_flags &= ~FSSBACKQ;
1640 1640
1641 1641 if (t->t_disp_time != ddi_get_lbolt())
1642 1642 setbackdq(t);
1643 1643 else
1644 1644 setfrontdq(t);
1645 1645
1646 1646 thread_unlock(t);
1647 1647 /*
1648 1648 * Safe to drop p_lock now since it is safe to change
1649 1649 * the scheduling class after this point.
1650 1650 */
1651 1651 mutex_exit(&pp->p_lock);
1652 1652
1653 1653 swtch();
1654 1654 }
1655 1655
1656 1656 /*
1657 1657 * Get the fair-sharing parameters of the thread pointed to by fssprocp into
1658 1658 * the buffer pointed by fssparmsp.
1659 1659 */
1660 1660 static void
1661 1661 fss_parmsget(kthread_t *t, void *parmsp)
1662 1662 {
1663 1663 fssproc_t *fssproc = FSSPROC(t);
1664 1664 fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1665 1665
1666 1666 fssparmsp->fss_uprilim = fssproc->fss_uprilim;
1667 1667 fssparmsp->fss_upri = fssproc->fss_upri;
1668 1668 }
1669 1669
1670 1670 /*ARGSUSED*/
1671 1671 static int
1672 1672 fss_parmsset(kthread_t *t, void *parmsp, id_t reqpcid, cred_t *reqpcredp)
1673 1673 {
1674 1674 char nice;
1675 1675 pri_t reqfssuprilim;
1676 1676 pri_t reqfssupri;
1677 1677 fssproc_t *fssproc = FSSPROC(t);
1678 1678 fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1679 1679
1680 1680 ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock));
1681 1681
1682 1682 if (fssparmsp->fss_uprilim == FSS_NOCHANGE)
1683 1683 reqfssuprilim = fssproc->fss_uprilim;
1684 1684 else
1685 1685 reqfssuprilim = fssparmsp->fss_uprilim;
1686 1686
1687 1687 if (fssparmsp->fss_upri == FSS_NOCHANGE)
1688 1688 reqfssupri = fssproc->fss_upri;
1689 1689 else
1690 1690 reqfssupri = fssparmsp->fss_upri;
1691 1691
1692 1692 /*
1693 1693 * Make sure the user priority doesn't exceed the upri limit.
1694 1694 */
1695 1695 if (reqfssupri > reqfssuprilim)
1696 1696 reqfssupri = reqfssuprilim;
1697 1697
1698 1698 /*
1699 1699 * Basic permissions enforced by generic kernel code for all classes
1700 1700 * require that a thread attempting to change the scheduling parameters
1701 1701 * of a target thread be privileged or have a real or effective UID
1702 1702 * matching that of the target thread. We are not called unless these
1703 1703 * basic permission checks have already passed. The fair-sharing class
1704 1704 * requires in addition that the calling thread be privileged if it
1705 1705 * is attempting to raise the upri limit above its current value.
1706 1706 * This may have been checked previously but if our caller passed us
1707 1707 * a non-NULL credential pointer we assume it hasn't and we check it
1708 1708 * here.
1709 1709 */
1710 1710 if ((reqpcredp != NULL) &&
1711 1711 (reqfssuprilim > fssproc->fss_uprilim) &&
1712 1712 secpolicy_raisepriority(reqpcredp) != 0)
1713 1713 return (EPERM);
1714 1714
1715 1715 /*
1716 1716 * Set fss_nice to the nice value corresponding to the user priority we
1717 1717 * are setting. Note that setting the nice field of the parameter
1718 1718 * struct won't affect upri or nice.
1719 1719 */
1720 1720 nice = NZERO - (reqfssupri * NZERO) / fss_maxupri;
1721 1721 if (nice > FSS_NICE_MAX)
1722 1722 nice = FSS_NICE_MAX;
1723 1723
1724 1724 thread_lock(t);
1725 1725
1726 1726 fssproc->fss_uprilim = reqfssuprilim;
1727 1727 fssproc->fss_upri = reqfssupri;
1728 1728 fssproc->fss_nice = nice;
1729 1729 fss_newpri(fssproc);
1730 1730
1731 1731 if ((fssproc->fss_flags & FSSKPRI) != 0) {
1732 1732 thread_unlock(t);
1733 1733 return (0);
1734 1734 }
1735 1735
1736 1736 fss_change_priority(t, fssproc);
1737 1737 thread_unlock(t);
1738 1738 return (0);
1739 1739
1740 1740 }
1741 1741
1742 1742 /*
1743 1743 * The thread is being stopped.
1744 1744 */
1745 1745 /*ARGSUSED*/
1746 1746 static void
1747 1747 fss_stop(kthread_t *t, int why, int what)
1748 1748 {
1749 1749 ASSERT(THREAD_LOCK_HELD(t));
1750 1750 ASSERT(t == curthread);
1751 1751
1752 1752 fss_inactive(t);
1753 1753 }
1754 1754
1755 1755 /*
1756 1756 * The current thread is exiting, do necessary adjustments to its project
1757 1757 */
1758 1758 static void
1759 1759 fss_exit(kthread_t *t)
1760 1760 {
1761 1761 fsspset_t *fsspset;
1762 1762 fssproj_t *fssproj;
1763 1763 fssproc_t *fssproc;
1764 1764 fsszone_t *fsszone;
1765 1765 int free = 0;
1766 1766
1767 1767 /*
1768 1768 * Thread t here is either a current thread (in which case we hold
1769 1769 * its process' p_lock), or a thread being destroyed by forklwp_fail(),
1770 1770 * in which case we hold pidlock and thread is no longer on the
1771 1771 * thread list.
1772 1772 */
1773 1773 ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock) || MUTEX_HELD(&pidlock));
1774 1774
1775 1775 fssproc = FSSPROC(t);
1776 1776 fssproj = FSSPROC2FSSPROJ(fssproc);
1777 1777 fsspset = FSSPROJ2FSSPSET(fssproj);
1778 1778 fsszone = fssproj->fssp_fsszone;
1779 1779
1780 1780 mutex_enter(&fsspsets_lock);
1781 1781 mutex_enter(&fsspset->fssps_lock);
1782 1782
1783 1783 thread_lock(t);
1784 1784 disp_lock_enter_high(&fsspset->fssps_displock);
1785 1785 if (t->t_state == TS_ONPROC || t->t_state == TS_RUN) {
1786 1786 if (--fssproj->fssp_runnable == 0) {
1787 1787 fsszone->fssz_shares -= fssproj->fssp_shares;
1788 1788 if (--fsszone->fssz_runnable == 0)
1789 1789 fsspset->fssps_shares -= fsszone->fssz_rshares;
1790 1790 }
1791 1791 ASSERT(fssproc->fss_runnable == 1);
1792 1792 fssproc->fss_runnable = 0;
1793 1793 }
1794 1794 if (--fssproj->fssp_threads == 0) {
1795 1795 fss_remove_fssproj(fsspset, fssproj);
1796 1796 free = 1;
1797 1797 }
1798 1798 disp_lock_exit_high(&fsspset->fssps_displock);
1799 1799 fssproc->fss_proj = NULL; /* mark this thread as already exited */
1800 1800 thread_unlock(t);
1801 1801
1802 1802 if (free) {
1803 1803 if (fsszone->fssz_nproj == 0)
1804 1804 kmem_free(fsszone, sizeof (fsszone_t));
1805 1805 kmem_free(fssproj, sizeof (fssproj_t));
1806 1806 }
1807 1807 mutex_exit(&fsspset->fssps_lock);
1808 1808 mutex_exit(&fsspsets_lock);
1809 1809
1810 1810 /*
1811 1811 * A thread could be exiting in between clock ticks, so we need to
1812 1812 * calculate how much CPU time it used since it was charged last time.
1813 1813 *
1814 1814 * CPU caps are not enforced on exiting processes - it is usually
1815 1815 * desirable to exit as soon as possible to free resources.
1816 1816 */
1817 1817 if (CPUCAPS_ON()) {
1818 1818 thread_lock(t);
1819 1819 fssproc = FSSPROC(t);
1820 1820 (void) cpucaps_charge(t, &fssproc->fss_caps,
1821 1821 CPUCAPS_CHARGE_ONLY);
1822 1822 thread_unlock(t);
1823 1823 }
1824 1824 }
1825 1825
1826 1826 static void
1827 1827 fss_nullsys()
1828 1828 {
1829 1829 }
1830 1830
1831 1831 /*
1832 1832 * fss_swapin() returns -1 if the thread is loaded or is not eligible to be
1833 1833 * swapped in. Otherwise, it returns the thread's effective priority based
1834 1834 * on swapout time and size of process (0 <= epri <= 0 SHRT_MAX).
1835 1835 */
1836 1836 /*ARGSUSED*/
1837 1837 static pri_t
1838 1838 fss_swapin(kthread_t *t, int flags)
1839 1839 {
1840 1840 fssproc_t *fssproc = FSSPROC(t);
1841 1841 long epri = -1;
1842 1842 proc_t *pp = ttoproc(t);
1843 1843
1844 1844 ASSERT(THREAD_LOCK_HELD(t));
1845 1845
1846 1846 if (t->t_state == TS_RUN && (t->t_schedflag & TS_LOAD) == 0) {
1847 1847 time_t swapout_time;
1848 1848
1849 1849 swapout_time = (ddi_get_lbolt() - t->t_stime) / hz;
1850 1850 if (INHERITED(t) || (fssproc->fss_flags & FSSKPRI)) {
1851 1851 epri = (long)DISP_PRIO(t) + swapout_time;
1852 1852 } else {
1853 1853 /*
1854 1854 * Threads which have been out for a long time,
1855 1855 * have high user mode priority and are associated
1856 1856 * with a small address space are more deserving.
1857 1857 */
1858 1858 epri = fssproc->fss_umdpri;
1859 1859 ASSERT(epri >= 0 && epri <= fss_maxumdpri);
1860 1860 epri += swapout_time - pp->p_swrss / nz(maxpgio)/2;
1861 1861 }
1862 1862 /*
1863 1863 * Scale epri so that SHRT_MAX / 2 represents zero priority.
1864 1864 */
1865 1865 epri += SHRT_MAX / 2;
1866 1866 if (epri < 0)
1867 1867 epri = 0;
1868 1868 else if (epri > SHRT_MAX)
1869 1869 epri = SHRT_MAX;
1870 1870 }
1871 1871 return ((pri_t)epri);
1872 1872 }
1873 1873
1874 1874 /*
1875 1875 * fss_swapout() returns -1 if the thread isn't loaded or is not eligible to
1876 1876 * be swapped out. Otherwise, it returns the thread's effective priority
1877 1877 * based on if the swapper is in softswap or hardswap mode.
1878 1878 */
1879 1879 static pri_t
1880 1880 fss_swapout(kthread_t *t, int flags)
1881 1881 {
1882 1882 fssproc_t *fssproc = FSSPROC(t);
1883 1883 long epri = -1;
1884 1884 proc_t *pp = ttoproc(t);
1885 1885 time_t swapin_time;
1886 1886
1887 1887 ASSERT(THREAD_LOCK_HELD(t));
1888 1888
1889 1889 if (INHERITED(t) ||
1890 1890 (fssproc->fss_flags & FSSKPRI) ||
1891 1891 (t->t_proc_flag & TP_LWPEXIT) ||
1892 1892 (t->t_state & (TS_ZOMB|TS_FREE|TS_STOPPED|TS_ONPROC|TS_WAIT)) ||
1893 1893 !(t->t_schedflag & TS_LOAD) ||
1894 1894 !(SWAP_OK(t)))
1895 1895 return (-1);
1896 1896
1897 1897 ASSERT(t->t_state & (TS_SLEEP | TS_RUN));
1898 1898
1899 1899 swapin_time = (ddi_get_lbolt() - t->t_stime) / hz;
1900 1900
1901 1901 if (flags == SOFTSWAP) {
1902 1902 if (t->t_state == TS_SLEEP && swapin_time > maxslp) {
1903 1903 epri = 0;
1904 1904 } else {
1905 1905 return ((pri_t)epri);
1906 1906 }
1907 1907 } else {
1908 1908 pri_t pri;
1909 1909
1910 1910 if ((t->t_state == TS_SLEEP && swapin_time > fss_minslp) ||
1911 1911 (t->t_state == TS_RUN && swapin_time > fss_minrun)) {
1912 1912 pri = fss_maxumdpri;
1913 1913 epri = swapin_time -
1914 1914 (rm_asrss(pp->p_as) / nz(maxpgio)/2) - (long)pri;
1915 1915 } else {
1916 1916 return ((pri_t)epri);
1917 1917 }
1918 1918 }
1919 1919
1920 1920 /*
1921 1921 * Scale epri so that SHRT_MAX / 2 represents zero priority.
1922 1922 */
1923 1923 epri += SHRT_MAX / 2;
1924 1924 if (epri < 0)
1925 1925 epri = 0;
1926 1926 else if (epri > SHRT_MAX)
1927 1927 epri = SHRT_MAX;
1928 1928
1929 1929 return ((pri_t)epri);
1930 1930 }
1931 1931
1932 1932 /*
1933 1933 * If thread is currently at a kernel mode priority (has slept) and is
1934 1934 * returning to the userland we assign it the appropriate user mode priority
1935 1935 * and time quantum here. If we're lowering the thread's priority below that
1936 1936 * of other runnable threads then we will set runrun via cpu_surrender() to
1937 1937 * cause preemption.
1938 1938 */
1939 1939 static void
1940 1940 fss_trapret(kthread_t *t)
1941 1941 {
1942 1942 fssproc_t *fssproc = FSSPROC(t);
1943 1943 cpu_t *cp = CPU;
1944 1944
1945 1945 ASSERT(THREAD_LOCK_HELD(t));
1946 1946 ASSERT(t == curthread);
1947 1947 ASSERT(cp->cpu_dispthread == t);
1948 1948 ASSERT(t->t_state == TS_ONPROC);
1949 1949
1950 1950 t->t_kpri_req = 0;
1951 1951 if (fssproc->fss_flags & FSSKPRI) {
1952 1952 /*
1953 1953 * If thread has blocked in the kernel
1954 1954 */
1955 1955 THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
1956 1956 cp->cpu_dispatch_pri = DISP_PRIO(t);
1957 1957 ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri);
1958 1958 fssproc->fss_flags &= ~FSSKPRI;
1959 1959
1960 1960 if (DISP_MUST_SURRENDER(t))
1961 1961 cpu_surrender(t);
1962 1962 }
1963 1963
1964 1964 /*
1965 1965 * Swapout lwp if the swapper is waiting for this thread to reach
1966 1966 * a safe point.
1967 1967 */
1968 1968 if (t->t_schedflag & TS_SWAPENQ) {
1969 1969 thread_unlock(t);
1970 1970 swapout_lwp(ttolwp(t));
1971 1971 thread_lock(t);
1972 1972 }
1973 1973 }
1974 1974
1975 1975 /*
1976 1976 * Arrange for thread to be placed in appropriate location on dispatcher queue.
1977 1977 * This is called with the current thread in TS_ONPROC and locked.
1978 1978 */
1979 1979 static void
1980 1980 fss_preempt(kthread_t *t)
1981 1981 {
1982 1982 fssproc_t *fssproc = FSSPROC(t);
1983 1983 klwp_t *lwp;
1984 1984 uint_t flags;
1985 1985
1986 1986 ASSERT(t == curthread);
1987 1987 ASSERT(THREAD_LOCK_HELD(curthread));
1988 1988 ASSERT(t->t_state == TS_ONPROC);
1989 1989
1990 1990 /*
1991 1991 * If preempted in the kernel, make sure the thread has a kernel
1992 1992 * priority if needed.
1993 1993 */
1994 1994 lwp = curthread->t_lwp;
1995 1995 if (!(fssproc->fss_flags & FSSKPRI) && lwp != NULL && t->t_kpri_req) {
1996 1996 fssproc->fss_flags |= FSSKPRI;
1997 1997 THREAD_CHANGE_PRI(t, minclsyspri);
1998 1998 ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri);
1999 1999 t->t_trapret = 1; /* so that fss_trapret will run */
2000 2000 aston(t);
2001 2001 }
2002 2002
2003 2003 /*
2004 2004 * This thread may be placed on wait queue by CPU Caps. In this case we
2005 2005 * do not need to do anything until it is removed from the wait queue.
2006 2006 * Do not enforce CPU caps on threads running at a kernel priority
2007 2007 */
2008 2008 if (CPUCAPS_ON()) {
2009 2009 (void) cpucaps_charge(t, &fssproc->fss_caps,
2010 2010 CPUCAPS_CHARGE_ENFORCE);
2011 2011
2012 2012 if (!(fssproc->fss_flags & FSSKPRI) && CPUCAPS_ENFORCE(t))
2013 2013 return;
2014 2014 }
2015 2015
2016 2016 /*
2017 2017 * If preempted in user-land mark the thread as swappable because it
2018 2018 * cannot be holding any kernel locks.
2019 2019 */
2020 2020 ASSERT(t->t_schedflag & TS_DONT_SWAP);
2021 2021 if (lwp != NULL && lwp->lwp_state == LWP_USER)
2022 2022 t->t_schedflag &= ~TS_DONT_SWAP;
2023 2023
2024 2024 /*
2025 2025 * Check to see if we're doing "preemption control" here. If
2026 2026 * we are, and if the user has requested that this thread not
2027 2027 * be preempted, and if preemptions haven't been put off for
2028 2028 * too long, let the preemption happen here but try to make
2029 2029 * sure the thread is rescheduled as soon as possible. We do
2030 2030 * this by putting it on the front of the highest priority run
2031 2031 * queue in the FSS class. If the preemption has been put off
2032 2032 * for too long, clear the "nopreempt" bit and let the thread
2033 2033 * be preempted.
2034 2034 */
2035 2035 if (t->t_schedctl && schedctl_get_nopreempt(t)) {
2036 2036 if (fssproc->fss_timeleft > -SC_MAX_TICKS) {
2037 2037 DTRACE_SCHED1(schedctl__nopreempt, kthread_t *, t);
2038 2038 if (!(fssproc->fss_flags & FSSKPRI)) {
2039 2039 /*
2040 2040 * If not already remembered, remember current
2041 2041 * priority for restoration in fss_yield().
2042 2042 */
2043 2043 if (!(fssproc->fss_flags & FSSRESTORE)) {
2044 2044 fssproc->fss_scpri = t->t_pri;
2045 2045 fssproc->fss_flags |= FSSRESTORE;
2046 2046 }
2047 2047 THREAD_CHANGE_PRI(t, fss_maxumdpri);
2048 2048 t->t_schedflag |= TS_DONT_SWAP;
2049 2049 }
2050 2050 schedctl_set_yield(t, 1);
2051 2051 setfrontdq(t);
2052 2052 return;
2053 2053 } else {
2054 2054 if (fssproc->fss_flags & FSSRESTORE) {
2055 2055 THREAD_CHANGE_PRI(t, fssproc->fss_scpri);
2056 2056 fssproc->fss_flags &= ~FSSRESTORE;
2057 2057 }
2058 2058 schedctl_set_nopreempt(t, 0);
2059 2059 DTRACE_SCHED1(schedctl__preempt, kthread_t *, t);
2060 2060 /*
2061 2061 * Fall through and be preempted below.
2062 2062 */
2063 2063 }
2064 2064 }
2065 2065
2066 2066 flags = fssproc->fss_flags & (FSSBACKQ | FSSKPRI);
2067 2067
2068 2068 if (flags == FSSBACKQ) {
2069 2069 fssproc->fss_timeleft = fss_quantum;
2070 2070 fssproc->fss_flags &= ~FSSBACKQ;
2071 2071 setbackdq(t);
2072 2072 } else if (flags == (FSSBACKQ | FSSKPRI)) {
2073 2073 fssproc->fss_flags &= ~FSSBACKQ;
2074 2074 setbackdq(t);
2075 2075 } else {
2076 2076 setfrontdq(t);
2077 2077 }
2078 2078 }
2079 2079
2080 2080 /*
2081 2081 * Called when a thread is waking up and is to be placed on the run queue.
2082 2082 */
2083 2083 static void
2084 2084 fss_setrun(kthread_t *t)
2085 2085 {
2086 2086 fssproc_t *fssproc = FSSPROC(t);
2087 2087
2088 2088 ASSERT(THREAD_LOCK_HELD(t)); /* t should be in transition */
2089 2089
2090 2090 if (t->t_state == TS_SLEEP || t->t_state == TS_STOPPED)
2091 2091 fss_active(t);
2092 2092
2093 2093 fssproc->fss_timeleft = fss_quantum;
2094 2094
2095 2095 fssproc->fss_flags &= ~FSSBACKQ;
2096 2096 /*
2097 2097 * If previously were running at the kernel priority then keep that
2098 2098 * priority and the fss_timeleft doesn't matter.
2099 2099 */
2100 2100 if ((fssproc->fss_flags & FSSKPRI) == 0)
2101 2101 THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
2102 2102
2103 2103 if (t->t_disp_time != ddi_get_lbolt())
2104 2104 setbackdq(t);
2105 2105 else
2106 2106 setfrontdq(t);
2107 2107 }
2108 2108
2109 2109 /*
2110 2110 * Prepare thread for sleep. We reset the thread priority so it will run at the
2111 2111 * kernel priority level when it wakes up.
2112 2112 */
2113 2113 static void
2114 2114 fss_sleep(kthread_t *t)
2115 2115 {
2116 2116 fssproc_t *fssproc = FSSPROC(t);
2117 2117
2118 2118 ASSERT(t == curthread);
2119 2119 ASSERT(THREAD_LOCK_HELD(t));
2120 2120
2121 2121 ASSERT(t->t_state == TS_ONPROC);
2122 2122
2123 2123 /*
2124 2124 * Account for time spent on CPU before going to sleep.
2125 2125 */
2126 2126 (void) CPUCAPS_CHARGE(t, &fssproc->fss_caps, CPUCAPS_CHARGE_ENFORCE);
2127 2127
2128 2128 fss_inactive(t);
2129 2129
2130 2130 /*
2131 2131 * Assign a system priority to the thread and arrange for it to be
2132 2132 * retained when the thread is next placed on the run queue (i.e.,
2133 2133 * when it wakes up) instead of being given a new pri. Also arrange
2134 2134 * for trapret processing as the thread leaves the system call so it
2135 2135 * will drop back to normal priority range.
2136 2136 */
2137 2137 if (t->t_kpri_req) {
2138 2138 THREAD_CHANGE_PRI(t, minclsyspri);
2139 2139 fssproc->fss_flags |= FSSKPRI;
2140 2140 t->t_trapret = 1; /* so that fss_trapret will run */
2141 2141 aston(t);
2142 2142 } else if (fssproc->fss_flags & FSSKPRI) {
2143 2143 /*
2144 2144 * The thread has done a THREAD_KPRI_REQUEST(), slept, then
2145 2145 * done THREAD_KPRI_RELEASE() (so no t_kpri_req is 0 again),
2146 2146 * then slept again all without finishing the current system
2147 2147 * call so trapret won't have cleared FSSKPRI
2148 2148 */
2149 2149 fssproc->fss_flags &= ~FSSKPRI;
2150 2150 THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
2151 2151 if (DISP_MUST_SURRENDER(curthread))
2152 2152 cpu_surrender(t);
2153 2153 }
2154 2154 t->t_stime = ddi_get_lbolt(); /* time stamp for the swapper */
2155 2155 }
2156 2156
2157 2157 /*
2158 2158 * A tick interrupt has ocurrend on a running thread. Check to see if our
2159 2159 * time slice has expired. We must also clear the TS_DONT_SWAP flag in
2160 2160 * t_schedflag if the thread is eligible to be swapped out.
2161 2161 */
2162 2162 static void
2163 2163 fss_tick(kthread_t *t)
2164 2164 {
2165 2165 fssproc_t *fssproc;
2166 2166 fssproj_t *fssproj;
2167 2167 klwp_t *lwp;
2168 2168 boolean_t call_cpu_surrender = B_FALSE;
2169 2169 boolean_t cpucaps_enforce = B_FALSE;
2170 2170
2171 2171 ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock));
2172 2172
2173 2173 /*
2174 2174 * It's safe to access fsspset and fssproj structures because we're
2175 2175 * holding our p_lock here.
2176 2176 */
2177 2177 thread_lock(t);
2178 2178 fssproc = FSSPROC(t);
2179 2179 fssproj = FSSPROC2FSSPROJ(fssproc);
2180 2180 if (fssproj != NULL) {
2181 2181 fsspset_t *fsspset = FSSPROJ2FSSPSET(fssproj);
2182 2182 disp_lock_enter_high(&fsspset->fssps_displock);
2183 2183 fssproj->fssp_ticks += fss_nice_tick[fssproc->fss_nice];
2184 2184 fssproc->fss_ticks++;
2185 2185 disp_lock_exit_high(&fsspset->fssps_displock);
2186 2186 }
2187 2187
2188 2188 /*
2189 2189 * Keep track of thread's project CPU usage. Note that projects
2190 2190 * get charged even when threads are running in the kernel.
2191 2191 * Do not surrender CPU if running in the SYS class.
2192 2192 */
2193 2193 if (CPUCAPS_ON()) {
2194 2194 cpucaps_enforce = cpucaps_charge(t,
2195 2195 &fssproc->fss_caps, CPUCAPS_CHARGE_ENFORCE) &&
2196 2196 !(fssproc->fss_flags & FSSKPRI);
2197 2197 }
2198 2198
2199 2199 /*
2200 2200 * A thread's execution time for threads running in the SYS class
2201 2201 * is not tracked.
2202 2202 */
2203 2203 if ((fssproc->fss_flags & FSSKPRI) == 0) {
2204 2204 /*
2205 2205 * If thread is not in kernel mode, decrement its fss_timeleft
2206 2206 */
2207 2207 if (--fssproc->fss_timeleft <= 0) {
2208 2208 pri_t new_pri;
2209 2209
2210 2210 /*
2211 2211 * If we're doing preemption control and trying to
2212 2212 * avoid preempting this thread, just note that the
2213 2213 * thread should yield soon and let it keep running
2214 2214 * (unless it's been a while).
2215 2215 */
2216 2216 if (t->t_schedctl && schedctl_get_nopreempt(t)) {
2217 2217 if (fssproc->fss_timeleft > -SC_MAX_TICKS) {
2218 2218 DTRACE_SCHED1(schedctl__nopreempt,
2219 2219 kthread_t *, t);
2220 2220 schedctl_set_yield(t, 1);
2221 2221 thread_unlock_nopreempt(t);
2222 2222 return;
2223 2223 }
2224 2224 }
2225 2225 fssproc->fss_flags &= ~FSSRESTORE;
2226 2226
2227 2227 fss_newpri(fssproc);
2228 2228 new_pri = fssproc->fss_umdpri;
2229 2229 ASSERT(new_pri >= 0 && new_pri <= fss_maxglobpri);
2230 2230
2231 2231 /*
2232 2232 * When the priority of a thread is changed, it may
2233 2233 * be necessary to adjust its position on a sleep queue
2234 2234 * or dispatch queue. The function thread_change_pri
2235 2235 * accomplishes this.
2236 2236 */
2237 2237 if (thread_change_pri(t, new_pri, 0)) {
2238 2238 if ((t->t_schedflag & TS_LOAD) &&
2239 2239 (lwp = t->t_lwp) &&
2240 2240 lwp->lwp_state == LWP_USER)
2241 2241 t->t_schedflag &= ~TS_DONT_SWAP;
2242 2242 fssproc->fss_timeleft = fss_quantum;
2243 2243 } else {
2244 2244 call_cpu_surrender = B_TRUE;
2245 2245 }
2246 2246 } else if (t->t_state == TS_ONPROC &&
2247 2247 t->t_pri < t->t_disp_queue->disp_maxrunpri) {
2248 2248 /*
2249 2249 * If there is a higher-priority thread which is
2250 2250 * waiting for a processor, then thread surrenders
2251 2251 * the processor.
2252 2252 */
2253 2253 call_cpu_surrender = B_TRUE;
2254 2254 }
2255 2255 }
2256 2256
2257 2257 if (cpucaps_enforce && 2 * fssproc->fss_timeleft > fss_quantum) {
2258 2258 /*
2259 2259 * The thread used more than half of its quantum, so assume that
2260 2260 * it used the whole quantum.
2261 2261 *
2262 2262 * Update thread's priority just before putting it on the wait
2263 2263 * queue so that it gets charged for the CPU time from its
2264 2264 * quantum even before that quantum expires.
2265 2265 */
2266 2266 fss_newpri(fssproc);
2267 2267 if (t->t_pri != fssproc->fss_umdpri)
2268 2268 fss_change_priority(t, fssproc);
2269 2269
2270 2270 /*
2271 2271 * We need to call cpu_surrender for this thread due to cpucaps
2272 2272 * enforcement, but fss_change_priority may have already done
2273 2273 * so. In this case FSSBACKQ is set and there is no need to call
2274 2274 * cpu-surrender again.
2275 2275 */
2276 2276 if (!(fssproc->fss_flags & FSSBACKQ))
2277 2277 call_cpu_surrender = B_TRUE;
2278 2278 }
2279 2279
2280 2280 if (call_cpu_surrender) {
2281 2281 fssproc->fss_flags |= FSSBACKQ;
2282 2282 cpu_surrender(t);
2283 2283 }
2284 2284
2285 2285 thread_unlock_nopreempt(t); /* clock thread can't be preempted */
2286 2286 }
2287 2287
2288 2288 /*
2289 2289 * Processes waking up go to the back of their queue. We don't need to assign
2290 2290 * a time quantum here because thread is still at a kernel mode priority and
2291 2291 * the time slicing is not done for threads running in the kernel after
2292 2292 * sleeping. The proper time quantum will be assigned by fss_trapret before the
2293 2293 * thread returns to user mode.
2294 2294 */
2295 2295 static void
2296 2296 fss_wakeup(kthread_t *t)
2297 2297 {
2298 2298 fssproc_t *fssproc;
2299 2299
2300 2300 ASSERT(THREAD_LOCK_HELD(t));
2301 2301 ASSERT(t->t_state == TS_SLEEP);
2302 2302
2303 2303 fss_active(t);
2304 2304
2305 2305 t->t_stime = ddi_get_lbolt(); /* time stamp for the swapper */
2306 2306 fssproc = FSSPROC(t);
2307 2307 fssproc->fss_flags &= ~FSSBACKQ;
2308 2308
2309 2309 if (fssproc->fss_flags & FSSKPRI) {
2310 2310 /*
2311 2311 * If we already have a kernel priority assigned, then we
2312 2312 * just use it.
2313 2313 */
2314 2314 setbackdq(t);
2315 2315 } else if (t->t_kpri_req) {
2316 2316 /*
2317 2317 * Give thread a priority boost if we were asked.
2318 2318 */
2319 2319 fssproc->fss_flags |= FSSKPRI;
2320 2320 THREAD_CHANGE_PRI(t, minclsyspri);
2321 2321 setbackdq(t);
2322 2322 t->t_trapret = 1; /* so that fss_trapret will run */
2323 2323 aston(t);
2324 2324 } else {
2325 2325 /*
2326 2326 * Otherwise, we recalculate the priority.
2327 2327 */
2328 2328 if (t->t_disp_time == ddi_get_lbolt()) {
2329 2329 setfrontdq(t);
2330 2330 } else {
2331 2331 fssproc->fss_timeleft = fss_quantum;
2332 2332 THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
2333 2333 setbackdq(t);
2334 2334 }
2335 2335 }
2336 2336 }
2337 2337
2338 2338 /*
2339 2339 * fss_donice() is called when a nice(1) command is issued on the thread to
2340 2340 * alter the priority. The nice(1) command exists in Solaris for compatibility.
2341 2341 * Thread priority adjustments should be done via priocntl(1).
2342 2342 */
2343 2343 static int
2344 2344 fss_donice(kthread_t *t, cred_t *cr, int incr, int *retvalp)
2345 2345 {
2346 2346 int newnice;
2347 2347 fssproc_t *fssproc = FSSPROC(t);
2348 2348 fssparms_t fssparms;
2349 2349
2350 2350 /*
2351 2351 * If there is no change to priority, just return current setting.
2352 2352 */
2353 2353 if (incr == 0) {
2354 2354 if (retvalp)
2355 2355 *retvalp = fssproc->fss_nice - NZERO;
2356 2356 return (0);
2357 2357 }
2358 2358
2359 2359 if ((incr < 0 || incr > 2 * NZERO) && secpolicy_raisepriority(cr) != 0)
2360 2360 return (EPERM);
2361 2361
2362 2362 /*
2363 2363 * Specifying a nice increment greater than the upper limit of
2364 2364 * FSS_NICE_MAX (== 2 * NZERO - 1) will result in the thread's nice
2365 2365 * value being set to the upper limit. We check for this before
2366 2366 * computing the new value because otherwise we could get overflow
2367 2367 * if a privileged user specified some ridiculous increment.
2368 2368 */
2369 2369 if (incr > FSS_NICE_MAX)
2370 2370 incr = FSS_NICE_MAX;
2371 2371
2372 2372 newnice = fssproc->fss_nice + incr;
2373 2373 if (newnice > FSS_NICE_MAX)
2374 2374 newnice = FSS_NICE_MAX;
2375 2375 else if (newnice < FSS_NICE_MIN)
2376 2376 newnice = FSS_NICE_MIN;
2377 2377
2378 2378 fssparms.fss_uprilim = fssparms.fss_upri =
2379 2379 -((newnice - NZERO) * fss_maxupri) / NZERO;
2380 2380
2381 2381 /*
2382 2382 * Reset the uprilim and upri values of the thread.
2383 2383 */
2384 2384 (void) fss_parmsset(t, (void *)&fssparms, (id_t)0, (cred_t *)NULL);
2385 2385
2386 2386 /*
2387 2387 * Although fss_parmsset already reset fss_nice it may not have been
2388 2388 * set to precisely the value calculated above because fss_parmsset
2389 2389 * determines the nice value from the user priority and we may have
2390 2390 * truncated during the integer conversion from nice value to user
2391 2391 * priority and back. We reset fss_nice to the value we calculated
2392 2392 * above.
2393 2393 */
2394 2394 fssproc->fss_nice = (char)newnice;
2395 2395
2396 2396 if (retvalp)
2397 2397 *retvalp = newnice - NZERO;
2398 2398 return (0);
2399 2399 }
2400 2400
2401 2401 /*
2402 2402 * Increment the priority of the specified thread by incr and
2403 2403 * return the new value in *retvalp.
2404 2404 */
2405 2405 static int
2406 2406 fss_doprio(kthread_t *t, cred_t *cr, int incr, int *retvalp)
2407 2407 {
2408 2408 int newpri;
2409 2409 fssproc_t *fssproc = FSSPROC(t);
2410 2410 fssparms_t fssparms;
2411 2411
2412 2412 /*
2413 2413 * If there is no change to priority, just return current setting.
2414 2414 */
2415 2415 if (incr == 0) {
2416 2416 *retvalp = fssproc->fss_upri;
2417 2417 return (0);
2418 2418 }
2419 2419
2420 2420 newpri = fssproc->fss_upri + incr;
2421 2421 if (newpri > fss_maxupri || newpri < -fss_maxupri)
2422 2422 return (EINVAL);
2423 2423
2424 2424 *retvalp = newpri;
2425 2425 fssparms.fss_uprilim = fssparms.fss_upri = newpri;
2426 2426
2427 2427 /*
2428 2428 * Reset the uprilim and upri values of the thread.
2429 2429 */
2430 2430 return (fss_parmsset(t, &fssparms, (id_t)0, cr));
2431 2431 }
2432 2432
2433 2433 /*
2434 2434 * Return the global scheduling priority that would be assigned to a thread
2435 2435 * entering the fair-sharing class with the fss_upri.
2436 2436 */
2437 2437 /*ARGSUSED*/
2438 2438 static pri_t
2439 2439 fss_globpri(kthread_t *t)
2440 2440 {
2441 2441 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
2442 2442
2443 2443 return (fss_maxumdpri / 2);
2444 2444 }
2445 2445
2446 2446 /*
2447 2447 * Called from the yield(2) system call when a thread is yielding (surrendering)
2448 2448 * the processor. The kernel thread is placed at the back of a dispatch queue.
2449 2449 */
2450 2450 static void
2451 2451 fss_yield(kthread_t *t)
2452 2452 {
2453 2453 fssproc_t *fssproc = FSSPROC(t);
2454 2454
2455 2455 ASSERT(t == curthread);
2456 2456 ASSERT(THREAD_LOCK_HELD(t));
2457 2457
2458 2458 /*
2459 2459 * Collect CPU usage spent before yielding
2460 2460 */
2461 2461 (void) CPUCAPS_CHARGE(t, &fssproc->fss_caps, CPUCAPS_CHARGE_ENFORCE);
2462 2462
2463 2463 /*
2464 2464 * Clear the preemption control "yield" bit since the user is
2465 2465 * doing a yield.
2466 2466 */
2467 2467 if (t->t_schedctl)
2468 2468 schedctl_set_yield(t, 0);
2469 2469 /*
2470 2470 * If fss_preempt() artifically increased the thread's priority
2471 2471 * to avoid preemption, restore the original priority now.
2472 2472 */
2473 2473 if (fssproc->fss_flags & FSSRESTORE) {
2474 2474 THREAD_CHANGE_PRI(t, fssproc->fss_scpri);
2475 2475 fssproc->fss_flags &= ~FSSRESTORE;
2476 2476 }
2477 2477 if (fssproc->fss_timeleft < 0) {
2478 2478 /*
2479 2479 * Time slice was artificially extended to avoid preemption,
2480 2480 * so pretend we're preempting it now.
2481 2481 */
2482 2482 DTRACE_SCHED1(schedctl__yield, int, -fssproc->fss_timeleft);
2483 2483 fssproc->fss_timeleft = fss_quantum;
2484 2484 }
2485 2485 fssproc->fss_flags &= ~FSSBACKQ;
2486 2486 setbackdq(t);
2487 2487 }
2488 2488
2489 2489 void
2490 2490 fss_changeproj(kthread_t *t, void *kp, void *zp, fssbuf_t *projbuf,
2491 2491 fssbuf_t *zonebuf)
2492 2492 {
2493 2493 kproject_t *kpj_new = kp;
2494 2494 zone_t *zone = zp;
2495 2495 fssproj_t *fssproj_old, *fssproj_new;
2496 2496 fsspset_t *fsspset;
2497 2497 kproject_t *kpj_old;
2498 2498 fssproc_t *fssproc;
2499 2499 fsszone_t *fsszone_old, *fsszone_new;
2500 2500 int free = 0;
2501 2501 int id;
2502 2502
2503 2503 ASSERT(MUTEX_HELD(&cpu_lock));
2504 2504 ASSERT(MUTEX_HELD(&pidlock));
2505 2505 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
2506 2506
2507 2507 if (t->t_cid != fss_cid)
2508 2508 return;
2509 2509
2510 2510 fssproc = FSSPROC(t);
2511 2511 mutex_enter(&fsspsets_lock);
2512 2512 fssproj_old = FSSPROC2FSSPROJ(fssproc);
2513 2513 if (fssproj_old == NULL) {
2514 2514 mutex_exit(&fsspsets_lock);
2515 2515 return;
2516 2516 }
2517 2517
2518 2518 fsspset = FSSPROJ2FSSPSET(fssproj_old);
2519 2519 mutex_enter(&fsspset->fssps_lock);
2520 2520 kpj_old = FSSPROJ2KPROJ(fssproj_old);
2521 2521 fsszone_old = fssproj_old->fssp_fsszone;
2522 2522
2523 2523 ASSERT(t->t_cpupart == fsspset->fssps_cpupart);
2524 2524
2525 2525 if (kpj_old == kpj_new) {
2526 2526 mutex_exit(&fsspset->fssps_lock);
2527 2527 mutex_exit(&fsspsets_lock);
2528 2528 return;
2529 2529 }
2530 2530
2531 2531 if ((fsszone_new = fss_find_fsszone(fsspset, zone)) == NULL) {
2532 2532 /*
2533 2533 * If the zone for the new project is not currently active on
2534 2534 * the cpu partition we're on, get one of the pre-allocated
2535 2535 * buffers and link it in our per-pset zone list. Such buffers
2536 2536 * should already exist.
2537 2537 */
2538 2538 for (id = 0; id < zonebuf->fssb_size; id++) {
2539 2539 if ((fsszone_new = zonebuf->fssb_list[id]) != NULL) {
2540 2540 fss_insert_fsszone(fsspset, zone, fsszone_new);
2541 2541 zonebuf->fssb_list[id] = NULL;
2542 2542 break;
2543 2543 }
2544 2544 }
2545 2545 }
2546 2546 ASSERT(fsszone_new != NULL);
2547 2547 if ((fssproj_new = fss_find_fssproj(fsspset, kpj_new)) == NULL) {
2548 2548 /*
2549 2549 * If our new project is not currently running
2550 2550 * on the cpu partition we're on, get one of the
2551 2551 * pre-allocated buffers and link it in our new cpu
2552 2552 * partition doubly linked list. Such buffers should already
2553 2553 * exist.
2554 2554 */
2555 2555 for (id = 0; id < projbuf->fssb_size; id++) {
2556 2556 if ((fssproj_new = projbuf->fssb_list[id]) != NULL) {
2557 2557 fss_insert_fssproj(fsspset, kpj_new,
2558 2558 fsszone_new, fssproj_new);
2559 2559 projbuf->fssb_list[id] = NULL;
2560 2560 break;
2561 2561 }
2562 2562 }
2563 2563 }
2564 2564 ASSERT(fssproj_new != NULL);
2565 2565
2566 2566 thread_lock(t);
2567 2567 if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2568 2568 t->t_state == TS_WAIT)
2569 2569 fss_inactive(t);
2570 2570 ASSERT(fssproj_old->fssp_threads > 0);
2571 2571 if (--fssproj_old->fssp_threads == 0) {
2572 2572 fss_remove_fssproj(fsspset, fssproj_old);
2573 2573 free = 1;
2574 2574 }
2575 2575 fssproc->fss_proj = fssproj_new;
2576 2576 fssproc->fss_fsspri = 0;
2577 2577 fssproj_new->fssp_threads++;
2578 2578 if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2579 2579 t->t_state == TS_WAIT)
2580 2580 fss_active(t);
2581 2581 thread_unlock(t);
2582 2582 if (free) {
2583 2583 if (fsszone_old->fssz_nproj == 0)
2584 2584 kmem_free(fsszone_old, sizeof (fsszone_t));
2585 2585 kmem_free(fssproj_old, sizeof (fssproj_t));
2586 2586 }
2587 2587
2588 2588 mutex_exit(&fsspset->fssps_lock);
2589 2589 mutex_exit(&fsspsets_lock);
2590 2590 }
2591 2591
2592 2592 void
2593 2593 fss_changepset(kthread_t *t, void *newcp, fssbuf_t *projbuf,
2594 2594 fssbuf_t *zonebuf)
2595 2595 {
2596 2596 fsspset_t *fsspset_old, *fsspset_new;
2597 2597 fssproj_t *fssproj_old, *fssproj_new;
2598 2598 fsszone_t *fsszone_old, *fsszone_new;
2599 2599 fssproc_t *fssproc;
2600 2600 kproject_t *kpj;
2601 2601 zone_t *zone;
2602 2602 int id;
2603 2603
2604 2604 ASSERT(MUTEX_HELD(&cpu_lock));
2605 2605 ASSERT(MUTEX_HELD(&pidlock));
2606 2606 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
2607 2607
2608 2608 if (t->t_cid != fss_cid)
2609 2609 return;
2610 2610
2611 2611 fssproc = FSSPROC(t);
2612 2612 zone = ttoproc(t)->p_zone;
2613 2613 mutex_enter(&fsspsets_lock);
2614 2614 fssproj_old = FSSPROC2FSSPROJ(fssproc);
2615 2615 if (fssproj_old == NULL) {
2616 2616 mutex_exit(&fsspsets_lock);
2617 2617 return;
2618 2618 }
2619 2619 fsszone_old = fssproj_old->fssp_fsszone;
2620 2620 fsspset_old = FSSPROJ2FSSPSET(fssproj_old);
2621 2621 kpj = FSSPROJ2KPROJ(fssproj_old);
2622 2622
2623 2623 if (fsspset_old->fssps_cpupart == newcp) {
2624 2624 mutex_exit(&fsspsets_lock);
2625 2625 return;
2626 2626 }
2627 2627
2628 2628 ASSERT(ttoproj(t) == kpj);
2629 2629
2630 2630 fsspset_new = fss_find_fsspset(newcp);
2631 2631
2632 2632 mutex_enter(&fsspset_new->fssps_lock);
2633 2633 if ((fsszone_new = fss_find_fsszone(fsspset_new, zone)) == NULL) {
2634 2634 for (id = 0; id < zonebuf->fssb_size; id++) {
2635 2635 if ((fsszone_new = zonebuf->fssb_list[id]) != NULL) {
2636 2636 fss_insert_fsszone(fsspset_new, zone,
2637 2637 fsszone_new);
2638 2638 zonebuf->fssb_list[id] = NULL;
2639 2639 break;
2640 2640 }
2641 2641 }
2642 2642 }
2643 2643 ASSERT(fsszone_new != NULL);
2644 2644 if ((fssproj_new = fss_find_fssproj(fsspset_new, kpj)) == NULL) {
2645 2645 for (id = 0; id < projbuf->fssb_size; id++) {
2646 2646 if ((fssproj_new = projbuf->fssb_list[id]) != NULL) {
2647 2647 fss_insert_fssproj(fsspset_new, kpj,
2648 2648 fsszone_new, fssproj_new);
2649 2649 projbuf->fssb_list[id] = NULL;
2650 2650 break;
2651 2651 }
2652 2652 }
2653 2653 }
2654 2654 ASSERT(fssproj_new != NULL);
2655 2655
2656 2656 fssproj_new->fssp_threads++;
2657 2657 thread_lock(t);
2658 2658 if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2659 2659 t->t_state == TS_WAIT)
2660 2660 fss_inactive(t);
2661 2661 fssproc->fss_proj = fssproj_new;
2662 2662 fssproc->fss_fsspri = 0;
2663 2663 if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2664 2664 t->t_state == TS_WAIT)
2665 2665 fss_active(t);
2666 2666 thread_unlock(t);
2667 2667 mutex_exit(&fsspset_new->fssps_lock);
2668 2668
2669 2669 mutex_enter(&fsspset_old->fssps_lock);
2670 2670 if (--fssproj_old->fssp_threads == 0) {
2671 2671 fss_remove_fssproj(fsspset_old, fssproj_old);
2672 2672 if (fsszone_old->fssz_nproj == 0)
2673 2673 kmem_free(fsszone_old, sizeof (fsszone_t));
2674 2674 kmem_free(fssproj_old, sizeof (fssproj_t));
2675 2675 }
2676 2676 mutex_exit(&fsspset_old->fssps_lock);
2677 2677
2678 2678 mutex_exit(&fsspsets_lock);
2679 2679 }
↓ open down ↓ |
1221 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX