Print this page
patch setfrontbackdq

*** 1167,1192 **** * some affinity for t->t_cpu. */ #define THREAD_HAS_CACHE_WARMTH(thread) \ ((thread == curthread) || \ ((ddi_get_lbolt() - thread->t_disp_time) <= rechoose_interval)) /* ! * Put the specified thread on the back of the dispatcher ! * queue corresponding to its current priority. * ! * Called with the thread in transition, onproc or stopped state ! * and locked (transition implies locked) and at high spl. ! * Returns with the thread in TS_RUN state and still locked. */ ! void ! setbackdq(kthread_t *tp) { dispq_t *dq; disp_t *dp; cpu_t *cp; pri_t tpri; ! int bound; boolean_t self; ASSERT(THREAD_LOCK_HELD(tp)); ASSERT((tp->t_schedflag & TS_ALLSTART) == 0); ASSERT(!thread_on_queue(tp)); /* make sure tp isn't on a runq */ --- 1167,1193 ---- * some affinity for t->t_cpu. */ #define THREAD_HAS_CACHE_WARMTH(thread) \ ((thread == curthread) || \ ((ddi_get_lbolt() - thread->t_disp_time) <= rechoose_interval)) + /* ! * Put the specified thread on the front/back of the dispatcher queue ! * corresponding to its current priority. * ! * Called with the thread in transition, onproc or stopped state and locked ! * (transition implies locked) and at high spl. Returns with the thread in ! * TS_RUN state and still locked. */ ! static void ! setfrontbackdq(kthread_t *tp, boolean_t front) { dispq_t *dq; disp_t *dp; cpu_t *cp; pri_t tpri; ! boolean_t bound; boolean_t self; ASSERT(THREAD_LOCK_HELD(tp)); ASSERT((tp->t_schedflag & TS_ALLSTART) == 0); ASSERT(!thread_on_queue(tp)); /* make sure tp isn't on a runq */
*** 1199,1242 **** disp_swapped_setrun(tp); return; } self = (tp == curthread); ! ! if (tp->t_bound_cpu || tp->t_weakbound_cpu) ! bound = 1; ! else ! bound = 0; tpri = DISP_PRIO(tp); if (ncpus == 1) cp = tp->t_cpu; else if (!bound) { if (tpri >= kpqpri) { ! setkpdq(tp, SETKP_BACK); return; } /* * We'll generally let this thread continue to run where * it last ran...but will consider migration if: * - We thread probably doesn't have much cache warmth. * - The CPU where it last ran is the target of an offline * request. * - The thread last ran outside it's home lgroup. */ ! if ((!THREAD_HAS_CACHE_WARMTH(tp)) || ! (tp->t_cpu == cpu_inmotion)) { ! cp = disp_lowpri_cpu(tp->t_cpu, tp->t_lpl, tpri, NULL); ! } else if (!LGRP_CONTAINS_CPU(tp->t_lpl->lpl_lgrp, tp->t_cpu)) { ! cp = disp_lowpri_cpu(tp->t_cpu, tp->t_lpl, tpri, ! self ? tp->t_cpu : NULL); ! } else { ! cp = tp->t_cpu; } if (tp->t_cpupart == cp->cpu_part) { int qlen; /* * Perform any CMT load balancing */ --- 1200,1263 ---- disp_swapped_setrun(tp); return; } self = (tp == curthread); ! bound = (tp->t_bound_cpu || tp->t_weakbound_cpu); tpri = DISP_PRIO(tp); if (ncpus == 1) cp = tp->t_cpu; else if (!bound) { if (tpri >= kpqpri) { ! setkpdq(tp, front ? SETKP_FRONT : SETKP_BACK); return; } + cp = tp->t_cpu; + + if (!front) { /* * We'll generally let this thread continue to run where * it last ran...but will consider migration if: * - We thread probably doesn't have much cache warmth. * - The CPU where it last ran is the target of an offline * request. * - The thread last ran outside it's home lgroup. */ ! if ((!THREAD_HAS_CACHE_WARMTH(tp)) || (cp == cpu_inmotion)) { ! cp = disp_lowpri_cpu(cp, tp->t_lpl, tpri, NULL); ! } else if (!LGRP_CONTAINS_CPU(tp->t_lpl->lpl_lgrp, cp)) { ! cp = disp_lowpri_cpu(cp, tp->t_lpl, tpri, ! self ? cp : NULL); ! } ! } if (tp->t_cpupart == cp->cpu_part) { + if (front) { + /* + * We'll generally let this thread continue to run + * where it last ran, but will consider migration if: + * - The thread last ran outside it's home lgroup. + * - The CPU where it last ran is the target of an + * offline request (a thread_nomigrate() on the in + * motion CPU relies on this when forcing a preempt). + * - The thread isn't the highest priority thread where + * it last ran, and it is considered not likely to + * have significant cache warmth. + */ + if ((!LGRP_CONTAINS_CPU(tp->t_lpl->lpl_lgrp, cp)) || + (cp == cpu_inmotion)) { + cp = disp_lowpri_cpu(cp, tp->t_lpl, tpri, + self ? cp : NULL); + } else if ((tpri < cp->cpu_disp->disp_maxrunpri) && + (!THREAD_HAS_CACHE_WARMTH(tp))) { + cp = disp_lowpri_cpu(cp, tp->t_lpl, tpri, + NULL); + } + } else { int qlen; /* * Perform any CMT load balancing */
*** 1263,1279 **** --- 1284,1302 ---- kthread_t *, tp, cpu_t *, cp, cpu_t *, newcp); cp = newcp; } } + } } else { /* * Migrate to a cpu in the new partition. */ cp = disp_lowpri_cpu(tp->t_cpupart->cp_cpulist, tp->t_lpl, tp->t_pri, NULL); } + ASSERT((cp->cpu_flags & CPU_QUIESCED) == 0); } else { /* * It is possible that t_weakbound_cpu != t_bound_cpu (for * a short time until weak binding that existed when the
*** 1281,1290 **** --- 1304,1314 ---- * favour weak binding over strong. */ cp = tp->t_weakbound_cpu ? tp->t_weakbound_cpu : tp->t_bound_cpu; } + /* * A thread that is ONPROC may be temporarily placed on the run queue * but then chosen to run again by disp. If the thread we're placing on * the queue is in TS_ONPROC state, don't set its t_waitrq until a * replacement process is actually scheduled in swtch(). In this
*** 1302,1314 **** } dp = cp->cpu_disp; disp_lock_enter_high(&dp->disp_lock); ! DTRACE_SCHED3(enqueue, kthread_t *, tp, disp_t *, dp, int, 0); TRACE_3(TR_FAC_DISP, TR_BACKQ, "setbackdq:pri %d cpu %p tid %p", tpri, cp, tp); #ifndef NPROBE /* Kernel probe */ if (tnf_tracing_active) tnf_thread_queue(tp, cp, tpri); --- 1326,1343 ---- } dp = cp->cpu_disp; disp_lock_enter_high(&dp->disp_lock); ! DTRACE_SCHED3(enqueue, kthread_t *, tp, disp_t *, dp, int, front); ! if (front) { ! TRACE_2(TR_FAC_DISP, TR_FRONTQ, "frontq:pri %d tid %p", tpri, ! tp); ! } else { TRACE_3(TR_FAC_DISP, TR_BACKQ, "setbackdq:pri %d cpu %p tid %p", tpri, cp, tp); + } #ifndef NPROBE /* Kernel probe */ if (tnf_tracing_active) tnf_thread_queue(tp, cp, tpri);
*** 1325,1337 **** --- 1354,1372 ---- if (!bound) dp->disp_steal = 0; membar_enter(); if (dq->dq_sruncnt++ != 0) { + if (front) { + ASSERT(dq->dq_last != NULL); + tp->t_link = dq->dq_first; + dq->dq_first = tp; + } else { ASSERT(dq->dq_first != NULL); dq->dq_last->t_link = tp; dq->dq_last = tp; + } } else { ASSERT(dq->dq_first == NULL); ASSERT(dq->dq_last == NULL); dq->dq_first = dq->dq_last = tp; BT_SET(dp->disp_qactmap, tpri);
*** 1354,1524 **** */ cp->cpu_disp_flags |= CPU_DISP_DONTSTEAL; } dp->disp_max_unbound_pri = tpri; } (*disp_enq_thread)(cp, bound); } /* ! * Put the specified thread on the front of the dispatcher * queue corresponding to its current priority. * * Called with the thread in transition, onproc or stopped state * and locked (transition implies locked) and at high spl. * Returns with the thread in TS_RUN state and still locked. */ void ! setfrontdq(kthread_t *tp) { ! disp_t *dp; ! dispq_t *dq; ! cpu_t *cp; ! pri_t tpri; ! int bound; ! ! ASSERT(THREAD_LOCK_HELD(tp)); ! ASSERT((tp->t_schedflag & TS_ALLSTART) == 0); ! ASSERT(!thread_on_queue(tp)); /* make sure tp isn't on a runq */ ! ! /* ! * If thread is "swapped" or on the swap queue don't ! * queue it, but wake sched. ! */ ! if ((tp->t_schedflag & (TS_LOAD | TS_ON_SWAPQ)) != TS_LOAD) { ! disp_swapped_setrun(tp); ! return; ! } ! ! if (tp->t_bound_cpu || tp->t_weakbound_cpu) ! bound = 1; ! else ! bound = 0; ! ! tpri = DISP_PRIO(tp); ! if (ncpus == 1) ! cp = tp->t_cpu; ! else if (!bound) { ! if (tpri >= kpqpri) { ! setkpdq(tp, SETKP_FRONT); ! return; ! } ! cp = tp->t_cpu; ! if (tp->t_cpupart == cp->cpu_part) { ! /* ! * We'll generally let this thread continue to run ! * where it last ran, but will consider migration if: ! * - The thread last ran outside it's home lgroup. ! * - The CPU where it last ran is the target of an ! * offline request (a thread_nomigrate() on the in ! * motion CPU relies on this when forcing a preempt). ! * - The thread isn't the highest priority thread where ! * it last ran, and it is considered not likely to ! * have significant cache warmth. ! */ ! if ((!LGRP_CONTAINS_CPU(tp->t_lpl->lpl_lgrp, cp)) || ! (cp == cpu_inmotion)) { ! cp = disp_lowpri_cpu(tp->t_cpu, tp->t_lpl, tpri, ! (tp == curthread) ? cp : NULL); ! } else if ((tpri < cp->cpu_disp->disp_maxrunpri) && ! (!THREAD_HAS_CACHE_WARMTH(tp))) { ! cp = disp_lowpri_cpu(tp->t_cpu, tp->t_lpl, tpri, ! NULL); ! } ! } else { ! /* ! * Migrate to a cpu in the new partition. ! */ ! cp = disp_lowpri_cpu(tp->t_cpupart->cp_cpulist, ! tp->t_lpl, tp->t_pri, NULL); ! } ! ASSERT((cp->cpu_flags & CPU_QUIESCED) == 0); ! } else { ! /* ! * It is possible that t_weakbound_cpu != t_bound_cpu (for ! * a short time until weak binding that existed when the ! * strong binding was established has dropped) so we must ! * favour weak binding over strong. ! */ ! cp = tp->t_weakbound_cpu ? ! tp->t_weakbound_cpu : tp->t_bound_cpu; ! } ! ! /* ! * A thread that is ONPROC may be temporarily placed on the run queue ! * but then chosen to run again by disp. If the thread we're placing on ! * the queue is in TS_ONPROC state, don't set its t_waitrq until a ! * replacement process is actually scheduled in swtch(). In this ! * situation, curthread is the only thread that could be in the ONPROC ! * state. ! */ ! if ((tp != curthread) && (tp->t_waitrq == 0)) { ! hrtime_t curtime; ! ! curtime = gethrtime_unscaled(); ! (void) cpu_update_pct(tp, curtime); ! tp->t_waitrq = curtime; ! } else { ! (void) cpu_update_pct(tp, gethrtime_unscaled()); ! } ! ! dp = cp->cpu_disp; ! disp_lock_enter_high(&dp->disp_lock); ! ! TRACE_2(TR_FAC_DISP, TR_FRONTQ, "frontq:pri %d tid %p", tpri, tp); ! DTRACE_SCHED3(enqueue, kthread_t *, tp, disp_t *, dp, int, 1); ! ! #ifndef NPROBE ! /* Kernel probe */ ! if (tnf_tracing_active) ! tnf_thread_queue(tp, cp, tpri); ! #endif /* NPROBE */ ! ! ASSERT(tpri >= 0 && tpri < dp->disp_npri); ! ! THREAD_RUN(tp, &dp->disp_lock); /* set TS_RUN state and lock */ ! tp->t_disp_queue = dp; ! ! dq = &dp->disp_q[tpri]; ! dp->disp_nrunnable++; ! if (!bound) ! dp->disp_steal = 0; ! membar_enter(); ! ! if (dq->dq_sruncnt++ != 0) { ! ASSERT(dq->dq_last != NULL); ! tp->t_link = dq->dq_first; ! dq->dq_first = tp; ! } else { ! ASSERT(dq->dq_last == NULL); ! ASSERT(dq->dq_first == NULL); ! tp->t_link = NULL; ! dq->dq_first = dq->dq_last = tp; ! BT_SET(dp->disp_qactmap, tpri); ! if (tpri > dp->disp_maxrunpri) { ! dp->disp_maxrunpri = tpri; ! membar_enter(); ! cpu_resched(cp, tpri); ! } ! } ! if (!bound && tpri > dp->disp_max_unbound_pri) { ! if (tp == curthread && dp->disp_max_unbound_pri == -1 && ! cp == CPU) { ! /* ! * If there are no other unbound threads on the ! * run queue, don't allow other CPUs to steal ! * this thread while we are in the middle of a ! * context switch. We may just switch to it ! * again right away. CPU_DISP_DONTSTEAL is cleared ! * in swtch and swtch_to. */ ! cp->cpu_disp_flags |= CPU_DISP_DONTSTEAL; ! } ! dp->disp_max_unbound_pri = tpri; ! } ! (*disp_enq_thread)(cp, bound); } /* * Put a high-priority unbound thread on the kp queue */ --- 1389,1428 ---- */ cp->cpu_disp_flags |= CPU_DISP_DONTSTEAL; } dp->disp_max_unbound_pri = tpri; } + (*disp_enq_thread)(cp, bound); } /* ! * Put the specified thread on the back of the dispatcher * queue corresponding to its current priority. * * Called with the thread in transition, onproc or stopped state * and locked (transition implies locked) and at high spl. * Returns with the thread in TS_RUN state and still locked. */ void ! setbackdq(kthread_t *tp) { ! setfrontbackdq(tp, B_FALSE); ! } ! /* ! * Put the specified thread on the front of the dispatcher ! * queue corresponding to its current priority. ! * ! * Called with the thread in transition, onproc or stopped state ! * and locked (transition implies locked) and at high spl. ! * Returns with the thread in TS_RUN state and still locked. */ ! void ! setfrontdq(kthread_t *tp) ! { ! setfrontbackdq(tp, B_TRUE); } /* * Put a high-priority unbound thread on the kp queue */