Print this page
patch as-lock-macro-simplification
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/os/move.c
+++ new/usr/src/uts/common/os/move.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 23 * Use is subject to license terms.
24 24 */
25 25
26 26 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
27 27 /* All Rights Reserved */
28 28
29 29 /*
30 30 * University Copyright- Copyright (c) 1982, 1986, 1988
31 31 * The Regents of the University of California
32 32 * All Rights Reserved
33 33 *
34 34 * University Acknowledgment- Portions of this document are derived from
35 35 * software developed by the University of California, Berkeley, and its
36 36 * contributors.
37 37 */
38 38
39 39 #include <sys/types.h>
40 40 #include <sys/sysmacros.h>
41 41 #include <sys/param.h>
42 42 #include <sys/systm.h>
43 43 #include <sys/uio.h>
44 44 #include <sys/errno.h>
45 45 #include <sys/vmsystm.h>
46 46 #include <sys/cmn_err.h>
47 47 #include <vm/as.h>
48 48 #include <vm/page.h>
49 49
50 50 #include <sys/dcopy.h>
51 51
52 52 int64_t uioa_maxpoll = -1; /* <0 = noblock, 0 = block, >0 = block after */
53 53 #define UIO_DCOPY_CHANNEL 0
54 54 #define UIO_DCOPY_CMD 1
55 55
56 56 /*
57 57 * Move "n" bytes at byte address "p"; "rw" indicates the direction
58 58 * of the move, and the I/O parameters are provided in "uio", which is
59 59 * update to reflect the data which was moved. Returns 0 on success or
60 60 * a non-zero errno on failure.
61 61 */
62 62 int
63 63 uiomove(void *p, size_t n, enum uio_rw rw, struct uio *uio)
64 64 {
65 65 struct iovec *iov;
66 66 ulong_t cnt;
67 67 int error;
68 68
69 69 while (n && uio->uio_resid) {
70 70 iov = uio->uio_iov;
71 71 cnt = MIN(iov->iov_len, n);
72 72 if (cnt == 0l) {
73 73 uio->uio_iov++;
74 74 uio->uio_iovcnt--;
75 75 continue;
76 76 }
77 77 switch (uio->uio_segflg) {
78 78
79 79 case UIO_USERSPACE:
80 80 case UIO_USERISPACE:
81 81 if (rw == UIO_READ) {
82 82 error = xcopyout_nta(p, iov->iov_base, cnt,
83 83 (uio->uio_extflg & UIO_COPY_CACHED));
84 84 } else {
85 85 error = xcopyin_nta(iov->iov_base, p, cnt,
86 86 (uio->uio_extflg & UIO_COPY_CACHED));
87 87 }
88 88
89 89 if (error)
90 90 return (error);
91 91 break;
92 92
93 93 case UIO_SYSSPACE:
94 94 if (rw == UIO_READ)
95 95 error = kcopy_nta(p, iov->iov_base, cnt,
96 96 (uio->uio_extflg & UIO_COPY_CACHED));
97 97 else
98 98 error = kcopy_nta(iov->iov_base, p, cnt,
99 99 (uio->uio_extflg & UIO_COPY_CACHED));
100 100 if (error)
101 101 return (error);
102 102 break;
103 103 }
104 104 iov->iov_base += cnt;
105 105 iov->iov_len -= cnt;
106 106 uio->uio_resid -= cnt;
107 107 uio->uio_loffset += cnt;
108 108 p = (caddr_t)p + cnt;
109 109 n -= cnt;
110 110 }
111 111 return (0);
112 112 }
113 113
114 114 /*
115 115 * Fault in the pages of the first n bytes specified by the uio structure.
116 116 * 1 byte in each page is touched and the uio struct is unmodified. Any
117 117 * error will terminate the process as this is only a best attempt to get
118 118 * the pages resident.
119 119 */
120 120 void
121 121 uio_prefaultpages(ssize_t n, struct uio *uio)
122 122 {
123 123 struct iovec *iov;
124 124 ulong_t cnt, incr;
125 125 caddr_t p;
126 126 uint8_t tmp;
127 127 int iovcnt;
128 128
129 129 iov = uio->uio_iov;
130 130 iovcnt = uio->uio_iovcnt;
131 131
132 132 while ((n > 0) && (iovcnt > 0)) {
133 133 cnt = MIN(iov->iov_len, n);
134 134 if (cnt == 0) {
135 135 /* empty iov entry */
136 136 iov++;
137 137 iovcnt--;
138 138 continue;
139 139 }
140 140 n -= cnt;
141 141 /*
142 142 * touch each page in this segment.
143 143 */
144 144 p = iov->iov_base;
145 145 while (cnt) {
146 146 switch (uio->uio_segflg) {
147 147 case UIO_USERSPACE:
148 148 case UIO_USERISPACE:
149 149 if (fuword8(p, &tmp))
150 150 return;
151 151 break;
152 152 case UIO_SYSSPACE:
153 153 if (kcopy(p, &tmp, 1))
154 154 return;
155 155 break;
156 156 }
157 157 incr = MIN(cnt, PAGESIZE);
158 158 p += incr;
159 159 cnt -= incr;
160 160 }
161 161 /*
162 162 * touch the last byte in case it straddles a page.
163 163 */
164 164 p--;
165 165 switch (uio->uio_segflg) {
166 166 case UIO_USERSPACE:
167 167 case UIO_USERISPACE:
168 168 if (fuword8(p, &tmp))
169 169 return;
170 170 break;
171 171 case UIO_SYSSPACE:
172 172 if (kcopy(p, &tmp, 1))
173 173 return;
174 174 break;
175 175 }
176 176 iov++;
177 177 iovcnt--;
178 178 }
179 179 }
180 180
181 181 /*
182 182 * same as uiomove() but doesn't modify uio structure.
183 183 * return in cbytes how many bytes were copied.
184 184 */
185 185 int
186 186 uiocopy(void *p, size_t n, enum uio_rw rw, struct uio *uio, size_t *cbytes)
187 187 {
188 188 struct iovec *iov;
189 189 ulong_t cnt;
190 190 int error;
191 191 int iovcnt;
192 192
193 193 iovcnt = uio->uio_iovcnt;
194 194 *cbytes = 0;
195 195
196 196 for (iov = uio->uio_iov; n && iovcnt; iov++, iovcnt--) {
197 197 cnt = MIN(iov->iov_len, n);
198 198 if (cnt == 0)
199 199 continue;
200 200
201 201 switch (uio->uio_segflg) {
202 202
203 203 case UIO_USERSPACE:
204 204 case UIO_USERISPACE:
205 205 if (rw == UIO_READ) {
206 206 error = xcopyout_nta(p, iov->iov_base, cnt,
207 207 (uio->uio_extflg & UIO_COPY_CACHED));
208 208 } else {
209 209 error = xcopyin_nta(iov->iov_base, p, cnt,
210 210 (uio->uio_extflg & UIO_COPY_CACHED));
211 211 }
212 212
213 213 if (error)
214 214 return (error);
215 215 break;
216 216
217 217 case UIO_SYSSPACE:
218 218 if (rw == UIO_READ)
219 219 error = kcopy_nta(p, iov->iov_base, cnt,
220 220 (uio->uio_extflg & UIO_COPY_CACHED));
221 221 else
222 222 error = kcopy_nta(iov->iov_base, p, cnt,
223 223 (uio->uio_extflg & UIO_COPY_CACHED));
224 224 if (error)
225 225 return (error);
226 226 break;
227 227 }
228 228 p = (caddr_t)p + cnt;
229 229 n -= cnt;
230 230 *cbytes += cnt;
231 231 }
232 232 return (0);
233 233 }
234 234
235 235 /*
236 236 * transfer a character value into the address space
237 237 * delineated by a uio and update fields within the
238 238 * uio for next character. Return 0 for success, EFAULT
239 239 * for error.
240 240 */
241 241 int
242 242 ureadc(int val, struct uio *uiop)
243 243 {
244 244 struct iovec *iovp;
245 245 unsigned char c;
246 246
247 247 /*
248 248 * first determine if uio is valid. uiop should be
249 249 * non-NULL and the resid count > 0.
250 250 */
251 251 if (!(uiop && uiop->uio_resid > 0))
252 252 return (EFAULT);
253 253
254 254 /*
255 255 * scan through iovecs until one is found that is non-empty.
256 256 * Return EFAULT if none found.
257 257 */
258 258 while (uiop->uio_iovcnt > 0) {
259 259 iovp = uiop->uio_iov;
260 260 if (iovp->iov_len <= 0) {
261 261 uiop->uio_iovcnt--;
262 262 uiop->uio_iov++;
263 263 } else
264 264 break;
265 265 }
266 266
267 267 if (uiop->uio_iovcnt <= 0)
268 268 return (EFAULT);
269 269
270 270 /*
271 271 * Transfer character to uio space.
272 272 */
273 273
274 274 c = (unsigned char) (val & 0xFF);
275 275
276 276 switch (uiop->uio_segflg) {
277 277
278 278 case UIO_USERISPACE:
279 279 case UIO_USERSPACE:
280 280 if (copyout(&c, iovp->iov_base, sizeof (unsigned char)))
281 281 return (EFAULT);
282 282 break;
283 283
284 284 case UIO_SYSSPACE: /* can do direct copy since kernel-kernel */
285 285 *iovp->iov_base = c;
286 286 break;
287 287
288 288 default:
289 289 return (EFAULT); /* invalid segflg value */
290 290 }
291 291
292 292 /*
293 293 * bump up/down iovec and uio members to reflect transfer.
294 294 */
295 295 iovp->iov_base++;
296 296 iovp->iov_len--;
297 297 uiop->uio_resid--;
298 298 uiop->uio_loffset++;
299 299 return (0); /* success */
300 300 }
301 301
302 302 /*
303 303 * return a character value from the address space
304 304 * delineated by a uio and update fields within the
305 305 * uio for next character. Return the character for success,
306 306 * -1 for error.
307 307 */
308 308 int
309 309 uwritec(struct uio *uiop)
310 310 {
311 311 struct iovec *iovp;
312 312 unsigned char c;
313 313
314 314 /*
315 315 * verify we were passed a valid uio structure.
316 316 * (1) non-NULL uiop, (2) positive resid count
317 317 * (3) there is an iovec with positive length
318 318 */
319 319
320 320 if (!(uiop && uiop->uio_resid > 0))
321 321 return (-1);
322 322
323 323 while (uiop->uio_iovcnt > 0) {
324 324 iovp = uiop->uio_iov;
325 325 if (iovp->iov_len <= 0) {
326 326 uiop->uio_iovcnt--;
327 327 uiop->uio_iov++;
328 328 } else
329 329 break;
330 330 }
331 331
332 332 if (uiop->uio_iovcnt <= 0)
333 333 return (-1);
334 334
335 335 /*
336 336 * Get the character from the uio address space.
337 337 */
338 338 switch (uiop->uio_segflg) {
339 339
340 340 case UIO_USERISPACE:
341 341 case UIO_USERSPACE:
342 342 if (copyin(iovp->iov_base, &c, sizeof (unsigned char)))
343 343 return (-1);
344 344 break;
345 345
346 346 case UIO_SYSSPACE:
347 347 c = *iovp->iov_base;
348 348 break;
349 349
350 350 default:
351 351 return (-1); /* invalid segflg */
352 352 }
353 353
354 354 /*
355 355 * Adjust fields of iovec and uio appropriately.
356 356 */
357 357 iovp->iov_base++;
358 358 iovp->iov_len--;
359 359 uiop->uio_resid--;
360 360 uiop->uio_loffset++;
361 361 return ((int)c & 0xFF); /* success */
362 362 }
363 363
364 364 /*
365 365 * Drop the next n chars out of *uiop.
366 366 */
367 367 void
368 368 uioskip(uio_t *uiop, size_t n)
369 369 {
370 370 if (n > uiop->uio_resid)
371 371 return;
372 372 while (n != 0) {
373 373 register iovec_t *iovp = uiop->uio_iov;
374 374 register size_t niovb = MIN(iovp->iov_len, n);
375 375
376 376 if (niovb == 0) {
377 377 uiop->uio_iov++;
378 378 uiop->uio_iovcnt--;
379 379 continue;
380 380 }
381 381 iovp->iov_base += niovb;
382 382 uiop->uio_loffset += niovb;
383 383 iovp->iov_len -= niovb;
384 384 uiop->uio_resid -= niovb;
385 385 n -= niovb;
386 386 }
387 387 }
388 388
389 389 /*
390 390 * Dup the suio into the duio and diovec of size diov_cnt. If diov
391 391 * is too small to dup suio then an error will be returned, else 0.
392 392 */
393 393 int
394 394 uiodup(uio_t *suio, uio_t *duio, iovec_t *diov, int diov_cnt)
395 395 {
396 396 int ix;
397 397 iovec_t *siov = suio->uio_iov;
398 398
399 399 *duio = *suio;
400 400 for (ix = 0; ix < suio->uio_iovcnt; ix++) {
401 401 diov[ix] = siov[ix];
402 402 if (ix >= diov_cnt)
403 403 return (1);
404 404 }
405 405 duio->uio_iov = diov;
406 406 return (0);
407 407 }
408 408
409 409 /*
410 410 * Shadow state for checking if a platform has hardware asynchronous
411 411 * copy capability and minimum copy size, e.g. Intel's I/OAT dma engine,
412 412 *
413 413 * Dcopy does a call-back to uioa_dcopy_enable() when a dma device calls
414 414 * into dcopy to register and uioa_dcopy_disable() when the device calls
415 415 * into dcopy to unregister.
416 416 */
417 417 uioasync_t uioasync = {B_FALSE, 1024};
418 418
419 419 void
420 420 uioa_dcopy_enable()
421 421 {
422 422 uioasync.enabled = B_TRUE;
423 423 }
424 424
425 425 void
426 426 uioa_dcopy_disable()
427 427 {
428 428 uioasync.enabled = B_FALSE;
429 429 }
430 430
431 431 /*
432 432 * Schedule an asynchronous move of "n" bytes at byte address "p",
433 433 * "rw" indicates the direction of the move, I/O parameters and
434 434 * async state are provided in "uioa" which is update to reflect
435 435 * the data which is to be moved.
436 436 *
437 437 * Returns 0 on success or a non-zero errno on failure.
438 438 *
439 439 * Note, while the uioasync APIs are general purpose in design
440 440 * the current implementation is Intel I/OAT specific.
441 441 */
442 442 int
443 443 uioamove(void *p, size_t n, enum uio_rw rw, uioa_t *uioa)
444 444 {
445 445 int soff, doff;
446 446 uint64_t pa;
447 447 int cnt;
448 448 iovec_t *iov;
449 449 dcopy_handle_t channel;
450 450 dcopy_cmd_t cmd;
451 451 int ret = 0;
452 452 int dcopy_flags;
453 453
454 454 if (!(uioa->uioa_state & UIOA_ENABLED)) {
455 455 /* The uioa_t isn't enabled */
456 456 return (ENXIO);
457 457 }
458 458
459 459 if (uioa->uio_segflg != UIO_USERSPACE || rw != UIO_READ) {
460 460 /* Only support to user-land from kernel */
461 461 return (ENOTSUP);
462 462 }
463 463
464 464
465 465 channel = uioa->uioa_hwst[UIO_DCOPY_CHANNEL];
466 466 cmd = uioa->uioa_hwst[UIO_DCOPY_CMD];
467 467 dcopy_flags = DCOPY_NOSLEEP;
468 468
469 469 /*
470 470 * While source bytes and destination bytes.
471 471 */
472 472 while (n > 0 && uioa->uio_resid > 0) {
473 473 iov = uioa->uio_iov;
474 474 if (iov->iov_len == 0l) {
475 475 uioa->uio_iov++;
476 476 uioa->uio_iovcnt--;
477 477 uioa->uioa_lcur++;
478 478 uioa->uioa_lppp = uioa->uioa_lcur->uioa_ppp;
479 479 continue;
480 480 }
481 481 /*
482 482 * While source bytes schedule an async
483 483 * dma for destination page by page.
484 484 */
485 485 while (n > 0) {
486 486 /* Addr offset in page src/dst */
487 487 soff = (uintptr_t)p & PAGEOFFSET;
488 488 doff = (uintptr_t)iov->iov_base & PAGEOFFSET;
489 489 /* Min copy count src and dst and page sized */
490 490 cnt = MIN(n, iov->iov_len);
491 491 cnt = MIN(cnt, PAGESIZE - soff);
492 492 cnt = MIN(cnt, PAGESIZE - doff);
493 493 /* XXX if next page(s) contiguous could use multipage */
494 494
495 495 /*
496 496 * if we have an old command, we want to link all
497 497 * other commands to the next command we alloced so
498 498 * we only need to track the last command but can
499 499 * still free them all.
500 500 */
501 501 if (cmd != NULL) {
502 502 dcopy_flags |= DCOPY_ALLOC_LINK;
503 503 }
504 504 ret = dcopy_cmd_alloc(channel, dcopy_flags, &cmd);
505 505 if (ret != DCOPY_SUCCESS) {
506 506 /* Error of some sort */
507 507 return (EIO);
508 508 }
509 509 uioa->uioa_hwst[UIO_DCOPY_CMD] = cmd;
510 510
511 511 ASSERT(cmd->dp_version == DCOPY_CMD_V0);
512 512 if (uioa_maxpoll >= 0) {
513 513 /* Blocking (>0 may be) used in uioafini() */
514 514 cmd->dp_flags = DCOPY_CMD_INTR;
515 515 } else {
516 516 /* Non blocking uioafini() so no intr */
517 517 cmd->dp_flags = DCOPY_CMD_NOFLAGS;
518 518 }
519 519 cmd->dp_cmd = DCOPY_CMD_COPY;
520 520 pa = ptob((uint64_t)hat_getpfnum(kas.a_hat, p));
521 521 cmd->dp.copy.cc_source = pa + soff;
522 522 if (uioa->uioa_lcur->uioa_pfncnt == 0) {
523 523 /* Have a (page_t **) */
524 524 pa = ptob((uint64_t)(
525 525 *(page_t **)uioa->uioa_lppp)->p_pagenum);
526 526 } else {
527 527 /* Have a (pfn_t *) */
528 528 pa = ptob((uint64_t)(
529 529 *(pfn_t *)uioa->uioa_lppp));
530 530 }
531 531 cmd->dp.copy.cc_dest = pa + doff;
532 532 cmd->dp.copy.cc_size = cnt;
533 533 ret = dcopy_cmd_post(cmd);
534 534 if (ret != DCOPY_SUCCESS) {
535 535 /* Error of some sort */
536 536 return (EIO);
537 537 }
538 538 ret = 0;
539 539
540 540 /* If UIOA_POLL not set, set it */
541 541 if (!(uioa->uioa_state & UIOA_POLL))
542 542 uioa->uioa_state |= UIOA_POLL;
543 543
544 544 /* Update iov, uio, and local pointers/counters */
545 545 iov->iov_base += cnt;
546 546 iov->iov_len -= cnt;
547 547 uioa->uio_resid -= cnt;
548 548 uioa->uioa_mbytes += cnt;
549 549 uioa->uio_loffset += cnt;
550 550 p = (caddr_t)p + cnt;
551 551 n -= cnt;
552 552
553 553 /* End of iovec? */
554 554 if (iov->iov_len == 0) {
555 555 /* Yup, next iovec */
556 556 break;
557 557 }
558 558
559 559 /* Next dst addr page? */
560 560 if (doff + cnt == PAGESIZE) {
561 561 /* Yup, next page_t */
562 562 uioa->uioa_lppp++;
563 563 }
564 564 }
565 565 }
566 566
567 567 return (ret);
568 568 }
569 569
570 570 /*
571 571 * Initialize a uioa_t for a given uio_t for the current user context,
572 572 * copy the common uio_t to the uioa_t, walk the shared iovec_t and
573 573 * lock down the user-land page(s) containing iovec_t data, then mapin
574 574 * user-land pages using segkpm.
575 575 */
576 576 int
577 577 uioainit(uio_t *uiop, uioa_t *uioap)
578 578 {
579 579 caddr_t addr;
580 580 page_t **pages;
581 581 int off;
582 582 int len;
583 583 proc_t *procp = ttoproc(curthread);
584 584 struct as *as = procp->p_as;
585 585 iovec_t *iov = uiop->uio_iov;
586 586 int32_t iovcnt = uiop->uio_iovcnt;
587 587 uioa_page_t *locked = uioap->uioa_locked;
588 588 dcopy_handle_t channel;
589 589 int error;
590 590
591 591 if (! (uioap->uioa_state & UIOA_ALLOC)) {
592 592 /* Can only init() a freshly allocated uioa_t */
593 593 return (EINVAL);
594 594 }
595 595
596 596 error = dcopy_alloc(DCOPY_NOSLEEP, &channel);
597 597 if (error == DCOPY_NORESOURCES) {
598 598 /* Turn off uioa */
599 599 uioasync.enabled = B_FALSE;
600 600 return (ENODEV);
601 601 }
602 602 if (error != DCOPY_SUCCESS) {
603 603 /* Alloc failed */
604 604 return (EIO);
605 605 }
606 606
607 607 uioap->uioa_hwst[UIO_DCOPY_CHANNEL] = channel;
608 608 uioap->uioa_hwst[UIO_DCOPY_CMD] = NULL;
609 609
610 610 /* Indicate uioa_t (will be) initialized */
611 611 uioap->uioa_state = UIOA_INIT;
612 612
613 613 uioap->uioa_mbytes = 0;
614 614
615 615 /* uio_t/uioa_t uio_t common struct copy */
616 616 *((uio_t *)uioap) = *uiop;
617 617
618 618 /* initialize *uiop->uio_iov */
619 619 if (iovcnt > UIOA_IOV_MAX) {
620 620 /* Too big? */
621 621 return (E2BIG);
622 622 }
623 623 uioap->uio_iov = iov;
624 624 uioap->uio_iovcnt = iovcnt;
625 625
626 626 /* Mark the uioap as such */
627 627 uioap->uio_extflg |= UIO_ASYNC;
628 628
629 629 /*
630 630 * For each iovec_t, lock-down the page(s) backing the iovec_t
631 631 * and save the page_t list for phys addr use in uioamove().
632 632 */
633 633 iov = uiop->uio_iov;
634 634 iovcnt = uiop->uio_iovcnt;
635 635 while (iovcnt > 0) {
636 636 addr = iov->iov_base;
637 637 off = (uintptr_t)addr & PAGEOFFSET;
638 638 addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
639 639 len = iov->iov_len + off;
640 640
641 641 /* Lock down page(s) for the iov span */
642 642 if ((error = as_pagelock(as, &pages,
643 643 iov->iov_base, iov->iov_len, S_WRITE)) != 0) {
644 644 /* Error */
645 645 goto cleanup;
646 646 }
647 647
648 648 if (pages == NULL) {
649 649 /*
650 650 * Need page_t list, really only need
651 651 * a pfn list so build one.
652 652 */
653 653 pfn_t *pfnp;
654 654 int pcnt = len >> PAGESHIFT;
|
↓ open down ↓ |
654 lines elided |
↑ open up ↑ |
655 655
656 656 if (off)
657 657 pcnt++;
658 658 if ((pfnp = kmem_alloc(pcnt * sizeof (pfnp),
659 659 KM_NOSLEEP)) == NULL) {
660 660 error = ENOMEM;
661 661 goto cleanup;
662 662 }
663 663 locked->uioa_ppp = (void **)pfnp;
664 664 locked->uioa_pfncnt = pcnt;
665 - AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
665 + AS_LOCK_ENTER(as, RW_READER);
666 666 while (pcnt-- > 0) {
667 667 *pfnp++ = hat_getpfnum(as->a_hat, addr);
668 668 addr += PAGESIZE;
669 669 }
670 - AS_LOCK_EXIT(as, &as->a_lock);
670 + AS_LOCK_EXIT(as);
671 671 } else {
672 672 /* Have a page_t list, save it */
673 673 locked->uioa_ppp = (void **)pages;
674 674 locked->uioa_pfncnt = 0;
675 675 }
676 676 /* Save for as_pageunlock() in uioafini() */
677 677 locked->uioa_base = iov->iov_base;
678 678 locked->uioa_len = iov->iov_len;
679 679 locked++;
680 680
681 681 /* Next iovec_t */
682 682 iov++;
683 683 iovcnt--;
684 684 }
685 685 /* Initialize curret pointer into uioa_locked[] and it's uioa_ppp */
686 686 uioap->uioa_lcur = uioap->uioa_locked;
687 687 uioap->uioa_lppp = uioap->uioa_lcur->uioa_ppp;
688 688 return (0);
689 689
690 690 cleanup:
691 691 /* Unlock any previously locked page_t(s) */
692 692 while (locked > uioap->uioa_locked) {
693 693 locked--;
694 694 as_pageunlock(as, (page_t **)locked->uioa_ppp,
695 695 locked->uioa_base, locked->uioa_len, S_WRITE);
696 696 }
697 697
698 698 /* Last indicate uioa_t still in alloc state */
699 699 uioap->uioa_state = UIOA_ALLOC;
700 700 uioap->uioa_mbytes = 0;
701 701
702 702 return (error);
703 703 }
704 704
705 705 /*
706 706 * Finish processing of a uioa_t by cleanup any pending "uioap" actions.
707 707 */
708 708 int
709 709 uioafini(uio_t *uiop, uioa_t *uioap)
710 710 {
711 711 int32_t iovcnt = uiop->uio_iovcnt;
712 712 uioa_page_t *locked = uioap->uioa_locked;
713 713 struct as *as = ttoproc(curthread)->p_as;
714 714 dcopy_handle_t channel;
715 715 dcopy_cmd_t cmd;
716 716 int ret = 0;
717 717
718 718 ASSERT(uioap->uio_extflg & UIO_ASYNC);
719 719
720 720 if (!(uioap->uioa_state & (UIOA_ENABLED|UIOA_FINI))) {
721 721 /* Must be an active uioa_t */
722 722 return (EINVAL);
723 723 }
724 724
725 725 channel = uioap->uioa_hwst[UIO_DCOPY_CHANNEL];
726 726 cmd = uioap->uioa_hwst[UIO_DCOPY_CMD];
727 727
728 728 /* XXX - why do we get cmd == NULL sometimes? */
729 729 if (cmd != NULL) {
730 730 if (uioap->uioa_state & UIOA_POLL) {
731 731 /* Wait for last dcopy() to finish */
732 732 int64_t poll = 1;
733 733 int poll_flag = DCOPY_POLL_NOFLAGS;
734 734
735 735 do {
736 736 if (uioa_maxpoll == 0 ||
737 737 (uioa_maxpoll > 0 &&
738 738 poll >= uioa_maxpoll)) {
739 739 /* Always block or after maxpoll */
740 740 poll_flag = DCOPY_POLL_BLOCK;
741 741 } else {
742 742 /* No block, poll */
743 743 poll++;
744 744 }
745 745 ret = dcopy_cmd_poll(cmd, poll_flag);
746 746 } while (ret == DCOPY_PENDING);
747 747
748 748 if (ret == DCOPY_COMPLETED) {
749 749 /* Poll/block succeeded */
750 750 ret = 0;
751 751 } else {
752 752 /* Poll/block failed */
753 753 ret = EIO;
754 754 }
755 755 }
756 756 dcopy_cmd_free(&cmd);
757 757 }
758 758
759 759 dcopy_free(&channel);
760 760
761 761 /* Unlock all page(s) iovec_t by iovec_t */
762 762 while (iovcnt-- > 0) {
763 763 page_t **pages;
764 764
765 765 if (locked->uioa_pfncnt == 0) {
766 766 /* A as_pagelock() returned (page_t **) */
767 767 pages = (page_t **)locked->uioa_ppp;
768 768 } else {
769 769 /* Our pfn_t array */
770 770 pages = NULL;
771 771 kmem_free(locked->uioa_ppp, locked->uioa_pfncnt *
772 772 sizeof (pfn_t *));
773 773 }
774 774 as_pageunlock(as, pages, locked->uioa_base, locked->uioa_len,
775 775 S_WRITE);
776 776
777 777 locked++;
778 778 }
779 779 /* uioa_t->uio_t common struct copy */
780 780 *uiop = *((uio_t *)uioap);
781 781
782 782 /*
783 783 * Last, reset uioa state to alloc.
784 784 *
785 785 * Note, we only initialize the state here, all other members
786 786 * will be initialized in a subsequent uioainit().
787 787 */
788 788 uioap->uioa_state = UIOA_ALLOC;
789 789 uioap->uioa_mbytes = 0;
790 790
791 791 uioap->uioa_hwst[UIO_DCOPY_CMD] = NULL;
792 792 uioap->uioa_hwst[UIO_DCOPY_CHANNEL] = NULL;
793 793
794 794 return (ret);
795 795 }
|
↓ open down ↓ |
115 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX