781 new_mp->b_wptr = mp->b_wptr;
782 new_mp->b_datap = dbp;
783 new_mp->b_queue = NULL;
784 MBLK_BAND_FLAG_WORD(new_mp) = MBLK_BAND_FLAG_WORD(mp);
785
786 STR_FTEVENT_MBLK(mp, caller(), FTEV_DUPB, dbp->db_ref);
787
788 dbp->db_free = dblk_decref;
789 do {
790 ASSERT(dbp->db_ref > 0);
791 oldrtfu = DBLK_RTFU_WORD(dbp);
792 newrtfu = oldrtfu + (1 << DBLK_RTFU_SHIFT(db_ref));
793 /*
794 * If db_ref is maxed out we can't dup this message anymore.
795 */
796 if ((oldrtfu & DBLK_RTFU_REF_MASK) == DBLK_RTFU_REF_MASK) {
797 kmem_cache_free(mblk_cache, new_mp);
798 new_mp = NULL;
799 goto out;
800 }
801 } while (cas32(&DBLK_RTFU_WORD(dbp), oldrtfu, newrtfu) != oldrtfu);
802
803 out:
804 FTRACE_1("dupb(): new_mp=0x%lx", (uintptr_t)new_mp);
805 return (new_mp);
806 }
807
808 static void
809 dblk_lastfree_desb(mblk_t *mp, dblk_t *dbp)
810 {
811 frtn_t *frp = dbp->db_frtnp;
812
813 ASSERT(dbp->db_mblk == mp);
814 frp->free_func(frp->free_arg);
815 if (dbp->db_fthdr != NULL)
816 str_ftfree(dbp);
817
818 /* set credp and projid to be 'unspecified' before returning to cache */
819 if (dbp->db_credp != NULL) {
820 crfree(dbp->db_credp);
821 dbp->db_credp = NULL;
4233 * tail. If this succeeds, this is the new
4234 * "next" block. Otherwise, another thread
4235 * got here first, so free the block and start
4236 * again.
4237 */
4238 nbp = kmem_cache_alloc(ftblk_cache, KM_NOSLEEP);
4239 if (nbp == NULL) {
4240 /* no mem, so punt */
4241 str_ftnever++;
4242 /* free up all flow data? */
4243 return;
4244 }
4245 nbp->nxt = NULL;
4246 nbp->ix = 1;
4247 /*
4248 * Just in case there is another thread about
4249 * to get the next index, we need to make sure
4250 * the value is there for it.
4251 */
4252 membar_producer();
4253 if (casptr(&hp->tail, bp, nbp) == bp) {
4254 /* CAS was successful */
4255 bp->nxt = nbp;
4256 membar_producer();
4257 bp = nbp;
4258 ix = 0;
4259 goto cas_good;
4260 } else {
4261 kmem_cache_free(ftblk_cache, nbp);
4262 bp = hp->tail;
4263 continue;
4264 }
4265 }
4266 nix = ix + 1;
4267 if (cas32((uint32_t *)&bp->ix, ix, nix) == ix) {
4268 cas_good:
4269 if (curthread != hp->thread) {
4270 hp->thread = curthread;
4271 evnt |= FTEV_CS;
4272 }
4273 if (CPU->cpu_seqid != hp->cpu_seqid) {
4274 hp->cpu_seqid = CPU->cpu_seqid;
4275 evnt |= FTEV_PS;
4276 }
4277 ep = &bp->ev[ix];
4278 break;
4279 }
4280 }
4281
4282 if (evnt & FTEV_QMASK) {
4283 queue_t *qp = p;
4284
4285 if (!(qp->q_flag & QREADR))
4286 evnt |= FTEV_ISWR;
4287
|
781 new_mp->b_wptr = mp->b_wptr;
782 new_mp->b_datap = dbp;
783 new_mp->b_queue = NULL;
784 MBLK_BAND_FLAG_WORD(new_mp) = MBLK_BAND_FLAG_WORD(mp);
785
786 STR_FTEVENT_MBLK(mp, caller(), FTEV_DUPB, dbp->db_ref);
787
788 dbp->db_free = dblk_decref;
789 do {
790 ASSERT(dbp->db_ref > 0);
791 oldrtfu = DBLK_RTFU_WORD(dbp);
792 newrtfu = oldrtfu + (1 << DBLK_RTFU_SHIFT(db_ref));
793 /*
794 * If db_ref is maxed out we can't dup this message anymore.
795 */
796 if ((oldrtfu & DBLK_RTFU_REF_MASK) == DBLK_RTFU_REF_MASK) {
797 kmem_cache_free(mblk_cache, new_mp);
798 new_mp = NULL;
799 goto out;
800 }
801 } while (atomic_cas_32(&DBLK_RTFU_WORD(dbp), oldrtfu, newrtfu) !=
802 oldrtfu);
803
804 out:
805 FTRACE_1("dupb(): new_mp=0x%lx", (uintptr_t)new_mp);
806 return (new_mp);
807 }
808
809 static void
810 dblk_lastfree_desb(mblk_t *mp, dblk_t *dbp)
811 {
812 frtn_t *frp = dbp->db_frtnp;
813
814 ASSERT(dbp->db_mblk == mp);
815 frp->free_func(frp->free_arg);
816 if (dbp->db_fthdr != NULL)
817 str_ftfree(dbp);
818
819 /* set credp and projid to be 'unspecified' before returning to cache */
820 if (dbp->db_credp != NULL) {
821 crfree(dbp->db_credp);
822 dbp->db_credp = NULL;
4234 * tail. If this succeeds, this is the new
4235 * "next" block. Otherwise, another thread
4236 * got here first, so free the block and start
4237 * again.
4238 */
4239 nbp = kmem_cache_alloc(ftblk_cache, KM_NOSLEEP);
4240 if (nbp == NULL) {
4241 /* no mem, so punt */
4242 str_ftnever++;
4243 /* free up all flow data? */
4244 return;
4245 }
4246 nbp->nxt = NULL;
4247 nbp->ix = 1;
4248 /*
4249 * Just in case there is another thread about
4250 * to get the next index, we need to make sure
4251 * the value is there for it.
4252 */
4253 membar_producer();
4254 if (atomic_cas_ptr(&hp->tail, bp, nbp) == bp) {
4255 /* CAS was successful */
4256 bp->nxt = nbp;
4257 membar_producer();
4258 bp = nbp;
4259 ix = 0;
4260 goto cas_good;
4261 } else {
4262 kmem_cache_free(ftblk_cache, nbp);
4263 bp = hp->tail;
4264 continue;
4265 }
4266 }
4267 nix = ix + 1;
4268 if (atomic_cas_32((uint32_t *)&bp->ix, ix, nix) == ix) {
4269 cas_good:
4270 if (curthread != hp->thread) {
4271 hp->thread = curthread;
4272 evnt |= FTEV_CS;
4273 }
4274 if (CPU->cpu_seqid != hp->cpu_seqid) {
4275 hp->cpu_seqid = CPU->cpu_seqid;
4276 evnt |= FTEV_PS;
4277 }
4278 ep = &bp->ev[ix];
4279 break;
4280 }
4281 }
4282
4283 if (evnt & FTEV_QMASK) {
4284 queue_t *qp = p;
4285
4286 if (!(qp->q_flag & QREADR))
4287 evnt |= FTEV_ISWR;
4288
|