1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1995, 2010, Oracle and/or its affiliates. All rights reserved.
  24  */
  25 
  26 /* Copyright (c) 2013, OmniTI Computer Consulting, Inc. All rights reserved. */
  27 
  28 #include <sys/types.h>
  29 #include <sys/t_lock.h>
  30 #include <sys/param.h>
  31 #include <sys/systm.h>
  32 #include <sys/buf.h>
  33 #include <sys/conf.h>
  34 #include <sys/cred.h>
  35 #include <sys/kmem.h>
  36 #include <sys/sysmacros.h>
  37 #include <sys/vfs.h>
  38 #include <sys/vnode.h>
  39 #include <sys/debug.h>
  40 #include <sys/errno.h>
  41 #include <sys/time.h>
  42 #include <sys/file.h>
  43 #include <sys/user.h>
  44 #include <sys/stream.h>
  45 #include <sys/strsubr.h>
  46 #include <sys/strsun.h>
  47 #include <sys/sunddi.h>
  48 #include <sys/esunddi.h>
  49 #include <sys/flock.h>
  50 #include <sys/modctl.h>
  51 #include <sys/cmn_err.h>
  52 #include <sys/vmsystm.h>
  53 #include <sys/policy.h>
  54 
  55 #include <sys/socket.h>
  56 #include <sys/socketvar.h>
  57 
  58 #include <sys/isa_defs.h>
  59 #include <sys/inttypes.h>
  60 #include <sys/systm.h>
  61 #include <sys/cpuvar.h>
  62 #include <sys/filio.h>
  63 #include <sys/sendfile.h>
  64 #include <sys/ddi.h>
  65 #include <vm/seg.h>
  66 #include <vm/seg_map.h>
  67 #include <vm/seg_kpm.h>
  68 
  69 #include <fs/sockfs/nl7c.h>
  70 #include <fs/sockfs/sockcommon.h>
  71 #include <fs/sockfs/sockfilter_impl.h>
  72 #include <fs/sockfs/socktpi.h>
  73 
  74 #ifdef SOCK_TEST
  75 int do_useracc = 1;             /* Controlled by setting SO_DEBUG to 4 */
  76 #else
  77 #define do_useracc      1
  78 #endif /* SOCK_TEST */
  79 
  80 extern int      xnet_truncate_print;
  81 
  82 extern void     nl7c_init(void);
  83 extern int      sockfs_defer_nl7c_init;
  84 
  85 /*
  86  * Note: DEF_IOV_MAX is defined and used as it is in "fs/vncalls.c"
  87  *       as there isn't a formal definition of IOV_MAX ???
  88  */
  89 #define MSG_MAXIOVLEN   16
  90 
  91 /*
  92  * Kernel component of socket creation.
  93  *
  94  * The socket library determines which version number to use.
  95  * First the library calls this with a NULL devpath. If this fails
  96  * to find a transport (using solookup) the library will look in /etc/netconfig
  97  * for the appropriate transport. If one is found it will pass in the
  98  * devpath for the kernel to use.
  99  */
 100 int
 101 so_socket(int family, int type_w_flags, int protocol, char *devpath,
 102     int version)
 103 {
 104         struct sonode *so;
 105         vnode_t *vp;
 106         struct file *fp;
 107         int fd;
 108         int error;
 109         int type;
 110 
 111         type = type_w_flags & SOCK_TYPE_MASK;
 112         type_w_flags &= ~SOCK_TYPE_MASK;
 113         if (type_w_flags & ~(SOCK_CLOEXEC|SOCK_NDELAY|SOCK_NONBLOCK))
 114                 return (set_errno(EINVAL));
 115 
 116         if (devpath != NULL) {
 117                 char *buf;
 118                 size_t kdevpathlen = 0;
 119 
 120                 buf = kmem_alloc(MAXPATHLEN, KM_SLEEP);
 121                 if ((error = copyinstr(devpath, buf,
 122                     MAXPATHLEN, &kdevpathlen)) != 0) {
 123                         kmem_free(buf, MAXPATHLEN);
 124                         return (set_errno(error));
 125                 }
 126                 so = socket_create(family, type, protocol, buf, NULL,
 127                     SOCKET_SLEEP, version, CRED(), &error);
 128                 kmem_free(buf, MAXPATHLEN);
 129         } else {
 130                 so = socket_create(family, type, protocol, NULL, NULL,
 131                     SOCKET_SLEEP, version, CRED(), &error);
 132         }
 133         if (so == NULL)
 134                 return (set_errno(error));
 135 
 136         /* Allocate a file descriptor for the socket */
 137         vp = SOTOV(so);
 138         if (error = falloc(vp, FWRITE|FREAD, &fp, &fd)) {
 139                 (void) socket_close(so, 0, CRED());
 140                 socket_destroy(so);
 141                 return (set_errno(error));
 142         }
 143 
 144         /*
 145          * Now fill in the entries that falloc reserved
 146          */
 147         if (type_w_flags & SOCK_NDELAY) {
 148                 so->so_state |= SS_NDELAY;
 149                 fp->f_flag |= FNDELAY;
 150         }
 151         if (type_w_flags & SOCK_NONBLOCK) {
 152                 so->so_state |= SS_NONBLOCK;
 153                 fp->f_flag |= FNONBLOCK;
 154         }
 155         mutex_exit(&fp->f_tlock);
 156         setf(fd, fp);
 157         if ((type_w_flags & SOCK_CLOEXEC) != 0) {
 158                 f_setfd(fd, FD_CLOEXEC);
 159         }
 160 
 161         return (fd);
 162 }
 163 
 164 /*
 165  * Map from a file descriptor to a socket node.
 166  * Returns with the file descriptor held i.e. the caller has to
 167  * use releasef when done with the file descriptor.
 168  */
 169 struct sonode *
 170 getsonode(int sock, int *errorp, file_t **fpp)
 171 {
 172         file_t *fp;
 173         vnode_t *vp;
 174         struct sonode *so;
 175 
 176         if ((fp = getf(sock)) == NULL) {
 177                 *errorp = EBADF;
 178                 eprintline(*errorp);
 179                 return (NULL);
 180         }
 181         vp = fp->f_vnode;
 182         /* Check if it is a socket */
 183         if (vp->v_type != VSOCK) {
 184                 releasef(sock);
 185                 *errorp = ENOTSOCK;
 186                 eprintline(*errorp);
 187                 return (NULL);
 188         }
 189         /*
 190          * Use the stream head to find the real socket vnode.
 191          * This is needed when namefs sits above sockfs.
 192          */
 193         if (vp->v_stream) {
 194                 ASSERT(vp->v_stream->sd_vnode);
 195                 vp = vp->v_stream->sd_vnode;
 196 
 197                 so = VTOSO(vp);
 198                 if (so->so_version == SOV_STREAM) {
 199                         releasef(sock);
 200                         *errorp = ENOTSOCK;
 201                         eprintsoline(so, *errorp);
 202                         return (NULL);
 203                 }
 204         } else {
 205                 so = VTOSO(vp);
 206         }
 207         if (fpp)
 208                 *fpp = fp;
 209         return (so);
 210 }
 211 
 212 /*
 213  * Allocate and copyin a sockaddr.
 214  * Ensures NULL termination for AF_UNIX addresses by extending them
 215  * with one NULL byte if need be. Verifies that the length is not
 216  * excessive to prevent an application from consuming all of kernel
 217  * memory. Returns NULL when an error occurred.
 218  */
 219 static struct sockaddr *
 220 copyin_name(struct sonode *so, struct sockaddr *name, socklen_t *namelenp,
 221             int *errorp)
 222 {
 223         char    *faddr;
 224         size_t  namelen = (size_t)*namelenp;
 225 
 226         ASSERT(namelen != 0);
 227         if (namelen > SO_MAXARGSIZE) {
 228                 *errorp = EINVAL;
 229                 eprintsoline(so, *errorp);
 230                 return (NULL);
 231         }
 232 
 233         faddr = (char *)kmem_alloc(namelen, KM_SLEEP);
 234         if (copyin(name, faddr, namelen)) {
 235                 kmem_free(faddr, namelen);
 236                 *errorp = EFAULT;
 237                 eprintsoline(so, *errorp);
 238                 return (NULL);
 239         }
 240 
 241         /*
 242          * Add space for NULL termination if needed.
 243          * Do a quick check if the last byte is NUL.
 244          */
 245         if (so->so_family == AF_UNIX && faddr[namelen - 1] != '\0') {
 246                 /* Check if there is any NULL termination */
 247                 size_t  i;
 248                 int foundnull = 0;
 249 
 250                 for (i = sizeof (name->sa_family); i < namelen; i++) {
 251                         if (faddr[i] == '\0') {
 252                                 foundnull = 1;
 253                                 break;
 254                         }
 255                 }
 256                 if (!foundnull) {
 257                         /* Add extra byte for NUL padding */
 258                         char *nfaddr;
 259 
 260                         nfaddr = (char *)kmem_alloc(namelen + 1, KM_SLEEP);
 261                         bcopy(faddr, nfaddr, namelen);
 262                         kmem_free(faddr, namelen);
 263 
 264                         /* NUL terminate */
 265                         nfaddr[namelen] = '\0';
 266                         namelen++;
 267                         ASSERT((socklen_t)namelen == namelen);
 268                         *namelenp = (socklen_t)namelen;
 269                         faddr = nfaddr;
 270                 }
 271         }
 272         return ((struct sockaddr *)faddr);
 273 }
 274 
 275 /*
 276  * Copy from kaddr/klen to uaddr/ulen. Updates ulenp if non-NULL.
 277  */
 278 static int
 279 copyout_arg(void *uaddr, socklen_t ulen, void *ulenp,
 280                 void *kaddr, socklen_t klen)
 281 {
 282         if (uaddr != NULL) {
 283                 if (ulen > klen)
 284                         ulen = klen;
 285 
 286                 if (ulen != 0) {
 287                         if (copyout(kaddr, uaddr, ulen))
 288                                 return (EFAULT);
 289                 }
 290         } else
 291                 ulen = 0;
 292 
 293         if (ulenp != NULL) {
 294                 if (copyout(&ulen, ulenp, sizeof (ulen)))
 295                         return (EFAULT);
 296         }
 297         return (0);
 298 }
 299 
 300 /*
 301  * Copy from kaddr/klen to uaddr/ulen. Updates ulenp if non-NULL.
 302  * If klen is greater than ulen it still uses the non-truncated
 303  * klen to update ulenp.
 304  */
 305 static int
 306 copyout_name(void *uaddr, socklen_t ulen, void *ulenp,
 307                 void *kaddr, socklen_t klen)
 308 {
 309         if (uaddr != NULL) {
 310                 if (ulen >= klen)
 311                         ulen = klen;
 312                 else if (ulen != 0 && xnet_truncate_print) {
 313                         printf("sockfs: truncating copyout of address using "
 314                             "XNET semantics for pid = %d. Lengths %d, %d\n",
 315                             curproc->p_pid, klen, ulen);
 316                 }
 317 
 318                 if (ulen != 0) {
 319                         if (copyout(kaddr, uaddr, ulen))
 320                                 return (EFAULT);
 321                 } else
 322                         klen = 0;
 323         } else
 324                 klen = 0;
 325 
 326         if (ulenp != NULL) {
 327                 if (copyout(&klen, ulenp, sizeof (klen)))
 328                         return (EFAULT);
 329         }
 330         return (0);
 331 }
 332 
 333 /*
 334  * The socketpair() code in libsocket creates two sockets (using
 335  * the /etc/netconfig fallback if needed) before calling this routine
 336  * to connect the two sockets together.
 337  *
 338  * For a SOCK_STREAM socketpair a listener is needed - in that case this
 339  * routine will create a new file descriptor as part of accepting the
 340  * connection. The library socketpair() will check if svs[2] has changed
 341  * in which case it will close the changed fd.
 342  *
 343  * Note that this code could use the TPI feature of accepting the connection
 344  * on the listening endpoint. However, that would require significant changes
 345  * to soaccept.
 346  */
 347 int
 348 so_socketpair(int sv[2])
 349 {
 350         int svs[2];
 351         struct sonode *so1, *so2;
 352         int error;
 353         int orig_flags;
 354         struct sockaddr_ux *name;
 355         size_t namelen;
 356         sotpi_info_t *sti1;
 357         sotpi_info_t *sti2;
 358 
 359         dprint(1, ("so_socketpair(%p)\n", (void *)sv));
 360 
 361         error = useracc(sv, sizeof (svs), B_WRITE);
 362         if (error && do_useracc)
 363                 return (set_errno(EFAULT));
 364 
 365         if (copyin(sv, svs, sizeof (svs)))
 366                 return (set_errno(EFAULT));
 367 
 368         if ((so1 = getsonode(svs[0], &error, NULL)) == NULL)
 369                 return (set_errno(error));
 370 
 371         if ((so2 = getsonode(svs[1], &error, NULL)) == NULL) {
 372                 releasef(svs[0]);
 373                 return (set_errno(error));
 374         }
 375 
 376         if (so1->so_family != AF_UNIX || so2->so_family != AF_UNIX) {
 377                 error = EOPNOTSUPP;
 378                 goto done;
 379         }
 380 
 381         sti1 = SOTOTPI(so1);
 382         sti2 = SOTOTPI(so2);
 383 
 384         /*
 385          * The code below makes assumptions about the "sockfs" implementation.
 386          * So make sure that the correct implementation is really used.
 387          */
 388         ASSERT(so1->so_ops == &sotpi_sonodeops);
 389         ASSERT(so2->so_ops == &sotpi_sonodeops);
 390 
 391         if (so1->so_type == SOCK_DGRAM) {
 392                 /*
 393                  * Bind both sockets and connect them with each other.
 394                  * Need to allocate name/namelen for soconnect.
 395                  */
 396                 error = socket_bind(so1, NULL, 0, _SOBIND_UNSPEC, CRED());
 397                 if (error) {
 398                         eprintsoline(so1, error);
 399                         goto done;
 400                 }
 401                 error = socket_bind(so2, NULL, 0, _SOBIND_UNSPEC, CRED());
 402                 if (error) {
 403                         eprintsoline(so2, error);
 404                         goto done;
 405                 }
 406                 namelen = sizeof (struct sockaddr_ux);
 407                 name = kmem_alloc(namelen, KM_SLEEP);
 408                 name->sou_family = AF_UNIX;
 409                 name->sou_addr = sti2->sti_ux_laddr;
 410                 error = socket_connect(so1,
 411                     (struct sockaddr *)name,
 412                     (socklen_t)namelen,
 413                     0, _SOCONNECT_NOXLATE, CRED());
 414                 if (error) {
 415                         kmem_free(name, namelen);
 416                         eprintsoline(so1, error);
 417                         goto done;
 418                 }
 419                 name->sou_addr = sti1->sti_ux_laddr;
 420                 error = socket_connect(so2,
 421                     (struct sockaddr *)name,
 422                     (socklen_t)namelen,
 423                     0, _SOCONNECT_NOXLATE, CRED());
 424                 kmem_free(name, namelen);
 425                 if (error) {
 426                         eprintsoline(so2, error);
 427                         goto done;
 428                 }
 429                 releasef(svs[0]);
 430                 releasef(svs[1]);
 431         } else {
 432                 /*
 433                  * Bind both sockets, with so1 being a listener.
 434                  * Connect so2 to so1 - nonblocking to avoid waiting for
 435                  * soaccept to complete.
 436                  * Accept a connection on so1. Pass out the new fd as sv[0].
 437                  * The library will detect the changed fd and close
 438                  * the original one.
 439                  */
 440                 struct sonode *nso;
 441                 struct vnode *nvp;
 442                 struct file *nfp;
 443                 int nfd;
 444 
 445                 /*
 446                  * We could simply call socket_listen() here (which would do the
 447                  * binding automatically) if the code didn't rely on passing
 448                  * _SOBIND_NOXLATE to the TPI implementation of socket_bind().
 449                  */
 450                 error = socket_bind(so1, NULL, 0, _SOBIND_UNSPEC|
 451                     _SOBIND_NOXLATE|_SOBIND_LISTEN|_SOBIND_SOCKETPAIR,
 452                     CRED());
 453                 if (error) {
 454                         eprintsoline(so1, error);
 455                         goto done;
 456                 }
 457                 error = socket_bind(so2, NULL, 0, _SOBIND_UNSPEC, CRED());
 458                 if (error) {
 459                         eprintsoline(so2, error);
 460                         goto done;
 461                 }
 462 
 463                 namelen = sizeof (struct sockaddr_ux);
 464                 name = kmem_alloc(namelen, KM_SLEEP);
 465                 name->sou_family = AF_UNIX;
 466                 name->sou_addr = sti1->sti_ux_laddr;
 467                 error = socket_connect(so2,
 468                     (struct sockaddr *)name,
 469                     (socklen_t)namelen,
 470                     FNONBLOCK, _SOCONNECT_NOXLATE, CRED());
 471                 kmem_free(name, namelen);
 472                 if (error) {
 473                         if (error != EINPROGRESS) {
 474                                 eprintsoline(so2, error); goto done;
 475                         }
 476                 }
 477 
 478                 error = socket_accept(so1, 0, CRED(), &nso);
 479                 if (error) {
 480                         eprintsoline(so1, error);
 481                         goto done;
 482                 }
 483 
 484                 /* wait for so2 being SS_CONNECTED ignoring signals */
 485                 mutex_enter(&so2->so_lock);
 486                 error = sowaitconnected(so2, 0, 1);
 487                 mutex_exit(&so2->so_lock);
 488                 if (error != 0) {
 489                         (void) socket_close(nso, 0, CRED());
 490                         socket_destroy(nso);
 491                         eprintsoline(so2, error);
 492                         goto done;
 493                 }
 494 
 495                 nvp = SOTOV(nso);
 496                 if (error = falloc(nvp, FWRITE|FREAD, &nfp, &nfd)) {
 497                         (void) socket_close(nso, 0, CRED());
 498                         socket_destroy(nso);
 499                         eprintsoline(nso, error);
 500                         goto done;
 501                 }
 502                 /*
 503                  * copy over FNONBLOCK and FNDELAY flags should they exist
 504                  */
 505                 if (so1->so_state & SS_NONBLOCK)
 506                         nfp->f_flag |= FNONBLOCK;
 507                 if (so1->so_state & SS_NDELAY)
 508                         nfp->f_flag |= FNDELAY;
 509 
 510                 /*
 511                  * fill in the entries that falloc reserved
 512                  */
 513                 mutex_exit(&nfp->f_tlock);
 514                 setf(nfd, nfp);
 515 
 516                 /*
 517                  * get the original flags before we release
 518                  */
 519                 VERIFY(f_getfd_error(svs[0], &orig_flags) == 0);
 520 
 521                 releasef(svs[0]);
 522                 releasef(svs[1]);
 523 
 524                 /*
 525                  * If FD_CLOEXEC was set on the filedescriptor we're
 526                  * swapping out, we should set it on the new one too.
 527                  */
 528                 if (orig_flags & FD_CLOEXEC) {
 529                         f_setfd(nfd, FD_CLOEXEC);
 530                 }
 531 
 532                 /*
 533                  * The socketpair library routine will close the original
 534                  * svs[0] when this code passes out a different file
 535                  * descriptor.
 536                  */
 537                 svs[0] = nfd;
 538 
 539                 if (copyout(svs, sv, sizeof (svs))) {
 540                         (void) closeandsetf(nfd, NULL);
 541                         eprintline(EFAULT);
 542                         return (set_errno(EFAULT));
 543                 }
 544         }
 545         return (0);
 546 
 547 done:
 548         releasef(svs[0]);
 549         releasef(svs[1]);
 550         return (set_errno(error));
 551 }
 552 
 553 int
 554 bind(int sock, struct sockaddr *name, socklen_t namelen, int version)
 555 {
 556         struct sonode *so;
 557         int error;
 558 
 559         dprint(1, ("bind(%d, %p, %d)\n",
 560             sock, (void *)name, namelen));
 561 
 562         if ((so = getsonode(sock, &error, NULL)) == NULL)
 563                 return (set_errno(error));
 564 
 565         /* Allocate and copyin name */
 566         /*
 567          * X/Open test does not expect EFAULT with NULL name and non-zero
 568          * namelen.
 569          */
 570         if (name != NULL && namelen != 0) {
 571                 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
 572                 name = copyin_name(so, name, &namelen, &error);
 573                 if (name == NULL) {
 574                         releasef(sock);
 575                         return (set_errno(error));
 576                 }
 577         } else {
 578                 name = NULL;
 579                 namelen = 0;
 580         }
 581 
 582         switch (version) {
 583         default:
 584                 error = socket_bind(so, name, namelen, 0, CRED());
 585                 break;
 586         case SOV_XPG4_2:
 587                 error = socket_bind(so, name, namelen, _SOBIND_XPG4_2, CRED());
 588                 break;
 589         case SOV_SOCKBSD:
 590                 error = socket_bind(so, name, namelen, _SOBIND_SOCKBSD, CRED());
 591                 break;
 592         }
 593 done:
 594         releasef(sock);
 595         if (name != NULL)
 596                 kmem_free(name, (size_t)namelen);
 597 
 598         if (error)
 599                 return (set_errno(error));
 600         return (0);
 601 }
 602 
 603 /* ARGSUSED2 */
 604 int
 605 listen(int sock, int backlog, int version)
 606 {
 607         struct sonode *so;
 608         int error;
 609 
 610         dprint(1, ("listen(%d, %d)\n",
 611             sock, backlog));
 612 
 613         if ((so = getsonode(sock, &error, NULL)) == NULL)
 614                 return (set_errno(error));
 615 
 616         error = socket_listen(so, backlog, CRED());
 617 
 618         releasef(sock);
 619         if (error)
 620                 return (set_errno(error));
 621         return (0);
 622 }
 623 
 624 /*ARGSUSED3*/
 625 int
 626 accept(int sock, struct sockaddr *name, socklen_t *namelenp, int version,
 627     int flags)
 628 {
 629         struct sonode *so;
 630         file_t *fp;
 631         int error;
 632         socklen_t namelen;
 633         struct sonode *nso;
 634         struct vnode *nvp;
 635         struct file *nfp;
 636         int nfd;
 637         int ssflags;
 638         struct sockaddr *addrp;
 639         socklen_t addrlen;
 640 
 641         dprint(1, ("accept(%d, %p, %p)\n",
 642             sock, (void *)name, (void *)namelenp));
 643 
 644         if (flags & ~(SOCK_CLOEXEC|SOCK_NONBLOCK|SOCK_NDELAY)) {
 645                 return (set_errno(EINVAL));
 646         }
 647 
 648         /* Translate SOCK_ flags to their SS_ variant */
 649         ssflags = 0;
 650         if (flags & SOCK_NONBLOCK)
 651                 ssflags |= SS_NONBLOCK;
 652         if (flags & SOCK_NDELAY)
 653                 ssflags |= SS_NDELAY;
 654 
 655         if ((so = getsonode(sock, &error, &fp)) == NULL)
 656                 return (set_errno(error));
 657 
 658         if (name != NULL) {
 659                 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
 660                 if (copyin(namelenp, &namelen, sizeof (namelen))) {
 661                         releasef(sock);
 662                         return (set_errno(EFAULT));
 663                 }
 664                 if (namelen != 0) {
 665                         error = useracc(name, (size_t)namelen, B_WRITE);
 666                         if (error && do_useracc) {
 667                                 releasef(sock);
 668                                 return (set_errno(EFAULT));
 669                         }
 670                 } else
 671                         name = NULL;
 672         } else {
 673                 namelen = 0;
 674         }
 675 
 676         /*
 677          * Allocate the user fd before socket_accept() in order to
 678          * catch EMFILE errors before calling socket_accept().
 679          */
 680         if ((nfd = ufalloc(0)) == -1) {
 681                 eprintsoline(so, EMFILE);
 682                 releasef(sock);
 683                 return (set_errno(EMFILE));
 684         }
 685         error = socket_accept(so, fp->f_flag, CRED(), &nso);
 686         if (error) {
 687                 setf(nfd, NULL);
 688                 releasef(sock);
 689                 return (set_errno(error));
 690         }
 691 
 692         nvp = SOTOV(nso);
 693 
 694         ASSERT(MUTEX_NOT_HELD(&nso->so_lock));
 695         if (namelen != 0) {
 696                 addrlen = so->so_max_addr_len;
 697                 addrp = (struct sockaddr *)kmem_alloc(addrlen, KM_SLEEP);
 698 
 699                 if ((error = socket_getpeername(nso, (struct sockaddr *)addrp,
 700                     &addrlen, B_TRUE, CRED())) == 0) {
 701                         error = copyout_name(name, namelen, namelenp,
 702                             addrp, addrlen);
 703                 } else {
 704                         ASSERT(error == EINVAL || error == ENOTCONN);
 705                         error = ECONNABORTED;
 706                 }
 707                 kmem_free(addrp, so->so_max_addr_len);
 708         }
 709 
 710         if (error) {
 711                 setf(nfd, NULL);
 712                 (void) socket_close(nso, 0, CRED());
 713                 socket_destroy(nso);
 714                 releasef(sock);
 715                 return (set_errno(error));
 716         }
 717         if (error = falloc(NULL, FWRITE|FREAD, &nfp, NULL)) {
 718                 setf(nfd, NULL);
 719                 (void) socket_close(nso, 0, CRED());
 720                 socket_destroy(nso);
 721                 eprintsoline(so, error);
 722                 releasef(sock);
 723                 return (set_errno(error));
 724         }
 725         /*
 726          * fill in the entries that falloc reserved
 727          */
 728         nfp->f_vnode = nvp;
 729         mutex_exit(&nfp->f_tlock);
 730         setf(nfd, nfp);
 731 
 732         /*
 733          * Act on SOCK_CLOEXEC from flags
 734          */
 735         if (flags & SOCK_CLOEXEC) {
 736                 f_setfd(nfd, FD_CLOEXEC);
 737         }
 738 
 739         /*
 740          * Copy FNDELAY and FNONBLOCK from listener to acceptor
 741          * and from ssflags
 742          */
 743         if ((ssflags | so->so_state) & (SS_NDELAY|SS_NONBLOCK)) {
 744                 uint_t oflag = nfp->f_flag;
 745                 int arg = 0;
 746 
 747                 if ((ssflags | so->so_state) & SS_NONBLOCK)
 748                         arg |= FNONBLOCK;
 749                 else if ((ssflags | so->so_state) & SS_NDELAY)
 750                         arg |= FNDELAY;
 751 
 752                 /*
 753                  * This code is a simplification of the F_SETFL code in fcntl()
 754                  * Ignore any errors from VOP_SETFL.
 755                  */
 756                 if ((error = VOP_SETFL(nvp, oflag, arg, nfp->f_cred, NULL))
 757                     != 0) {
 758                         eprintsoline(so, error);
 759                         error = 0;
 760                 } else {
 761                         mutex_enter(&nfp->f_tlock);
 762                         nfp->f_flag &= ~FMASK | (FREAD|FWRITE);
 763                         nfp->f_flag |= arg;
 764                         mutex_exit(&nfp->f_tlock);
 765                 }
 766         }
 767         releasef(sock);
 768         return (nfd);
 769 }
 770 
 771 int
 772 connect(int sock, struct sockaddr *name, socklen_t namelen, int version)
 773 {
 774         struct sonode *so;
 775         file_t *fp;
 776         int error;
 777 
 778         dprint(1, ("connect(%d, %p, %d)\n",
 779             sock, (void *)name, namelen));
 780 
 781         if ((so = getsonode(sock, &error, &fp)) == NULL)
 782                 return (set_errno(error));
 783 
 784         /* Allocate and copyin name */
 785         if (namelen != 0) {
 786                 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
 787                 name = copyin_name(so, name, &namelen, &error);
 788                 if (name == NULL) {
 789                         releasef(sock);
 790                         return (set_errno(error));
 791                 }
 792         } else
 793                 name = NULL;
 794 
 795         error = socket_connect(so, name, namelen, fp->f_flag,
 796             (version != SOV_XPG4_2) ? 0 : _SOCONNECT_XPG4_2, CRED());
 797         releasef(sock);
 798         if (name)
 799                 kmem_free(name, (size_t)namelen);
 800         if (error)
 801                 return (set_errno(error));
 802         return (0);
 803 }
 804 
 805 /*ARGSUSED2*/
 806 int
 807 shutdown(int sock, int how, int version)
 808 {
 809         struct sonode *so;
 810         int error;
 811 
 812         dprint(1, ("shutdown(%d, %d)\n",
 813             sock, how));
 814 
 815         if ((so = getsonode(sock, &error, NULL)) == NULL)
 816                 return (set_errno(error));
 817 
 818         error = socket_shutdown(so, how, CRED());
 819 
 820         releasef(sock);
 821         if (error)
 822                 return (set_errno(error));
 823         return (0);
 824 }
 825 
 826 /*
 827  * Common receive routine.
 828  */
 829 static ssize_t
 830 recvit(int sock,
 831         struct nmsghdr *msg,
 832         struct uio *uiop,
 833         int flags,
 834         socklen_t *namelenp,
 835         socklen_t *controllenp,
 836         int *flagsp)
 837 {
 838         struct sonode *so;
 839         file_t *fp;
 840         void *name;
 841         socklen_t namelen;
 842         void *control;
 843         socklen_t controllen;
 844         ssize_t len;
 845         int error;
 846 
 847         if ((so = getsonode(sock, &error, &fp)) == NULL)
 848                 return (set_errno(error));
 849 
 850         len = uiop->uio_resid;
 851         uiop->uio_fmode = fp->f_flag;
 852         uiop->uio_extflg = UIO_COPY_CACHED;
 853 
 854         name = msg->msg_name;
 855         namelen = msg->msg_namelen;
 856         control = msg->msg_control;
 857         controllen = msg->msg_controllen;
 858 
 859         msg->msg_flags = flags & (MSG_OOB | MSG_PEEK | MSG_WAITALL |
 860             MSG_DONTWAIT | MSG_XPG4_2);
 861 
 862         error = socket_recvmsg(so, msg, uiop, CRED());
 863         if (error) {
 864                 releasef(sock);
 865                 return (set_errno(error));
 866         }
 867         lwp_stat_update(LWP_STAT_MSGRCV, 1);
 868         releasef(sock);
 869 
 870         error = copyout_name(name, namelen, namelenp,
 871             msg->msg_name, msg->msg_namelen);
 872         if (error)
 873                 goto err;
 874 
 875         if (flagsp != NULL) {
 876                 /*
 877                  * Clear internal flag.
 878                  */
 879                 msg->msg_flags &= ~MSG_XPG4_2;
 880 
 881                 /*
 882                  * Determine MSG_CTRUNC. sorecvmsg sets MSG_CTRUNC only
 883                  * when controllen is zero and there is control data to
 884                  * copy out.
 885                  */
 886                 if (controllen != 0 &&
 887                     (msg->msg_controllen > controllen || control == NULL)) {
 888                         dprint(1, ("recvit: CTRUNC %d %d %p\n",
 889                             msg->msg_controllen, controllen, control));
 890 
 891                         msg->msg_flags |= MSG_CTRUNC;
 892                 }
 893                 if (copyout(&msg->msg_flags, flagsp,
 894                     sizeof (msg->msg_flags))) {
 895                         error = EFAULT;
 896                         goto err;
 897                 }
 898         }
 899         /*
 900          * Note: This MUST be done last. There can be no "goto err" after this
 901          * point since it could make so_closefds run twice on some part
 902          * of the file descriptor array.
 903          */
 904         if (controllen != 0) {
 905                 if (!(flags & MSG_XPG4_2)) {
 906                         /*
 907                          * Good old msg_accrights can only return a multiple
 908                          * of 4 bytes.
 909                          */
 910                         controllen &= ~((int)sizeof (uint32_t) - 1);
 911                 }
 912                 error = copyout_arg(control, controllen, controllenp,
 913                     msg->msg_control, msg->msg_controllen);
 914                 if (error)
 915                         goto err;
 916 
 917                 if (msg->msg_controllen > controllen || control == NULL) {
 918                         if (control == NULL)
 919                                 controllen = 0;
 920                         so_closefds(msg->msg_control, msg->msg_controllen,
 921                             !(flags & MSG_XPG4_2), controllen);
 922                 }
 923         }
 924         if (msg->msg_namelen != 0)
 925                 kmem_free(msg->msg_name, (size_t)msg->msg_namelen);
 926         if (msg->msg_controllen != 0)
 927                 kmem_free(msg->msg_control, (size_t)msg->msg_controllen);
 928         return (len - uiop->uio_resid);
 929 
 930 err:
 931         /*
 932          * If we fail and the control part contains file descriptors
 933          * we have to close the fd's.
 934          */
 935         if (msg->msg_controllen != 0)
 936                 so_closefds(msg->msg_control, msg->msg_controllen,
 937                     !(flags & MSG_XPG4_2), 0);
 938         if (msg->msg_namelen != 0)
 939                 kmem_free(msg->msg_name, (size_t)msg->msg_namelen);
 940         if (msg->msg_controllen != 0)
 941                 kmem_free(msg->msg_control, (size_t)msg->msg_controllen);
 942         return (set_errno(error));
 943 }
 944 
 945 /*
 946  * Native system call
 947  */
 948 ssize_t
 949 recv(int sock, void *buffer, size_t len, int flags)
 950 {
 951         struct nmsghdr lmsg;
 952         struct uio auio;
 953         struct iovec aiov[1];
 954 
 955         dprint(1, ("recv(%d, %p, %ld, %d)\n",
 956             sock, buffer, len, flags));
 957 
 958         if ((ssize_t)len < 0) {
 959                 return (set_errno(EINVAL));
 960         }
 961 
 962         aiov[0].iov_base = buffer;
 963         aiov[0].iov_len = len;
 964         auio.uio_loffset = 0;
 965         auio.uio_iov = aiov;
 966         auio.uio_iovcnt = 1;
 967         auio.uio_resid = len;
 968         auio.uio_segflg = UIO_USERSPACE;
 969         auio.uio_limit = 0;
 970 
 971         lmsg.msg_namelen = 0;
 972         lmsg.msg_controllen = 0;
 973         lmsg.msg_flags = 0;
 974         return (recvit(sock, &lmsg, &auio, flags, NULL, NULL, NULL));
 975 }
 976 
 977 ssize_t
 978 recvfrom(int sock, void *buffer, size_t len, int flags,
 979         struct sockaddr *name, socklen_t *namelenp)
 980 {
 981         struct nmsghdr lmsg;
 982         struct uio auio;
 983         struct iovec aiov[1];
 984 
 985         dprint(1, ("recvfrom(%d, %p, %ld, %d, %p, %p)\n",
 986             sock, buffer, len, flags, (void *)name, (void *)namelenp));
 987 
 988         if ((ssize_t)len < 0) {
 989                 return (set_errno(EINVAL));
 990         }
 991 
 992         aiov[0].iov_base = buffer;
 993         aiov[0].iov_len = len;
 994         auio.uio_loffset = 0;
 995         auio.uio_iov = aiov;
 996         auio.uio_iovcnt = 1;
 997         auio.uio_resid = len;
 998         auio.uio_segflg = UIO_USERSPACE;
 999         auio.uio_limit = 0;
1000 
1001         lmsg.msg_name = (char *)name;
1002         if (namelenp != NULL) {
1003                 if (copyin(namelenp, &lmsg.msg_namelen,
1004                     sizeof (lmsg.msg_namelen)))
1005                         return (set_errno(EFAULT));
1006         } else {
1007                 lmsg.msg_namelen = 0;
1008         }
1009         lmsg.msg_controllen = 0;
1010         lmsg.msg_flags = 0;
1011 
1012         return (recvit(sock, &lmsg, &auio, flags, namelenp, NULL, NULL));
1013 }
1014 
1015 /*
1016  * Uses the MSG_XPG4_2 flag to determine if the caller is using
1017  * struct omsghdr or struct nmsghdr.
1018  */
1019 ssize_t
1020 recvmsg(int sock, struct nmsghdr *msg, int flags)
1021 {
1022         STRUCT_DECL(nmsghdr, u_lmsg);
1023         STRUCT_HANDLE(nmsghdr, umsgptr);
1024         struct nmsghdr lmsg;
1025         struct uio auio;
1026         struct iovec aiov[MSG_MAXIOVLEN];
1027         int iovcnt;
1028         ssize_t len;
1029         int i;
1030         int *flagsp;
1031         model_t model;
1032 
1033         dprint(1, ("recvmsg(%d, %p, %d)\n",
1034             sock, (void *)msg, flags));
1035 
1036         model = get_udatamodel();
1037         STRUCT_INIT(u_lmsg, model);
1038         STRUCT_SET_HANDLE(umsgptr, model, msg);
1039 
1040         if (flags & MSG_XPG4_2) {
1041                 if (copyin(msg, STRUCT_BUF(u_lmsg), STRUCT_SIZE(u_lmsg)))
1042                         return (set_errno(EFAULT));
1043                 flagsp = STRUCT_FADDR(umsgptr, msg_flags);
1044         } else {
1045                 /*
1046                  * Assumes that nmsghdr and omsghdr are identically shaped
1047                  * except for the added msg_flags field.
1048                  */
1049                 if (copyin(msg, STRUCT_BUF(u_lmsg),
1050                     SIZEOF_STRUCT(omsghdr, model)))
1051                         return (set_errno(EFAULT));
1052                 STRUCT_FSET(u_lmsg, msg_flags, 0);
1053                 flagsp = NULL;
1054         }
1055 
1056         /*
1057          * Code below us will kmem_alloc memory and hang it
1058          * off msg_control and msg_name fields. This forces
1059          * us to copy the structure to its native form.
1060          */
1061         lmsg.msg_name = STRUCT_FGETP(u_lmsg, msg_name);
1062         lmsg.msg_namelen = STRUCT_FGET(u_lmsg, msg_namelen);
1063         lmsg.msg_iov = STRUCT_FGETP(u_lmsg, msg_iov);
1064         lmsg.msg_iovlen = STRUCT_FGET(u_lmsg, msg_iovlen);
1065         lmsg.msg_control = STRUCT_FGETP(u_lmsg, msg_control);
1066         lmsg.msg_controllen = STRUCT_FGET(u_lmsg, msg_controllen);
1067         lmsg.msg_flags = STRUCT_FGET(u_lmsg, msg_flags);
1068 
1069         iovcnt = lmsg.msg_iovlen;
1070 
1071         if (iovcnt <= 0 || iovcnt > MSG_MAXIOVLEN) {
1072                 return (set_errno(EMSGSIZE));
1073         }
1074 
1075 #ifdef _SYSCALL32_IMPL
1076         /*
1077          * 32-bit callers need to have their iovec expanded, while ensuring
1078          * that they can't move more than 2Gbytes of data in a single call.
1079          */
1080         if (model == DATAMODEL_ILP32) {
1081                 struct iovec32 aiov32[MSG_MAXIOVLEN];
1082                 ssize32_t count32;
1083 
1084                 if (copyin((struct iovec32 *)lmsg.msg_iov, aiov32,
1085                     iovcnt * sizeof (struct iovec32)))
1086                         return (set_errno(EFAULT));
1087 
1088                 count32 = 0;
1089                 for (i = 0; i < iovcnt; i++) {
1090                         ssize32_t iovlen32;
1091 
1092                         iovlen32 = aiov32[i].iov_len;
1093                         count32 += iovlen32;
1094                         if (iovlen32 < 0 || count32 < 0)
1095                                 return (set_errno(EINVAL));
1096                         aiov[i].iov_len = iovlen32;
1097                         aiov[i].iov_base =
1098                             (caddr_t)(uintptr_t)aiov32[i].iov_base;
1099                 }
1100         } else
1101 #endif /* _SYSCALL32_IMPL */
1102         if (copyin(lmsg.msg_iov, aiov, iovcnt * sizeof (struct iovec))) {
1103                 return (set_errno(EFAULT));
1104         }
1105         len = 0;
1106         for (i = 0; i < iovcnt; i++) {
1107                 ssize_t iovlen = aiov[i].iov_len;
1108                 len += iovlen;
1109                 if (iovlen < 0 || len < 0) {
1110                         return (set_errno(EINVAL));
1111                 }
1112         }
1113         auio.uio_loffset = 0;
1114         auio.uio_iov = aiov;
1115         auio.uio_iovcnt = iovcnt;
1116         auio.uio_resid = len;
1117         auio.uio_segflg = UIO_USERSPACE;
1118         auio.uio_limit = 0;
1119 
1120         if (lmsg.msg_control != NULL &&
1121             (do_useracc == 0 ||
1122             useracc(lmsg.msg_control, lmsg.msg_controllen,
1123             B_WRITE) != 0)) {
1124                 return (set_errno(EFAULT));
1125         }
1126 
1127         return (recvit(sock, &lmsg, &auio, flags,
1128             STRUCT_FADDR(umsgptr, msg_namelen),
1129             STRUCT_FADDR(umsgptr, msg_controllen), flagsp));
1130 }
1131 
1132 /*
1133  * Common send function.
1134  */
1135 static ssize_t
1136 sendit(int sock, struct nmsghdr *msg, struct uio *uiop, int flags)
1137 {
1138         struct sonode *so;
1139         file_t *fp;
1140         void *name;
1141         socklen_t namelen;
1142         void *control;
1143         socklen_t controllen;
1144         ssize_t len;
1145         int error;
1146 
1147         if ((so = getsonode(sock, &error, &fp)) == NULL)
1148                 return (set_errno(error));
1149 
1150         uiop->uio_fmode = fp->f_flag;
1151 
1152         if (so->so_family == AF_UNIX)
1153                 uiop->uio_extflg = UIO_COPY_CACHED;
1154         else
1155                 uiop->uio_extflg = UIO_COPY_DEFAULT;
1156 
1157         /* Allocate and copyin name and control */
1158         name = msg->msg_name;
1159         namelen = msg->msg_namelen;
1160         if (name != NULL && namelen != 0) {
1161                 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1162                 name = copyin_name(so,
1163                     (struct sockaddr *)name,
1164                     &namelen, &error);
1165                 if (name == NULL)
1166                         goto done3;
1167                 /* copyin_name null terminates addresses for AF_UNIX */
1168                 msg->msg_namelen = namelen;
1169                 msg->msg_name = name;
1170         } else {
1171                 msg->msg_name = name = NULL;
1172                 msg->msg_namelen = namelen = 0;
1173         }
1174 
1175         control = msg->msg_control;
1176         controllen = msg->msg_controllen;
1177         if ((control != NULL) && (controllen != 0)) {
1178                 /*
1179                  * Verify that the length is not excessive to prevent
1180                  * an application from consuming all of kernel memory.
1181                  */
1182                 if (controllen > SO_MAXARGSIZE) {
1183                         error = EINVAL;
1184                         goto done2;
1185                 }
1186                 control = kmem_alloc(controllen, KM_SLEEP);
1187 
1188                 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1189                 if (copyin(msg->msg_control, control, controllen)) {
1190                         error = EFAULT;
1191                         goto done1;
1192                 }
1193                 msg->msg_control = control;
1194         } else {
1195                 msg->msg_control = control = NULL;
1196                 msg->msg_controllen = controllen = 0;
1197         }
1198 
1199         len = uiop->uio_resid;
1200         msg->msg_flags = flags;
1201 
1202         error = socket_sendmsg(so, msg, uiop, CRED());
1203 done1:
1204         if (control != NULL)
1205                 kmem_free(control, controllen);
1206 done2:
1207         if (name != NULL)
1208                 kmem_free(name, namelen);
1209 done3:
1210         if (error != 0) {
1211                 releasef(sock);
1212                 return (set_errno(error));
1213         }
1214         lwp_stat_update(LWP_STAT_MSGSND, 1);
1215         releasef(sock);
1216         return (len - uiop->uio_resid);
1217 }
1218 
1219 /*
1220  * Native system call
1221  */
1222 ssize_t
1223 send(int sock, void *buffer, size_t len, int flags)
1224 {
1225         struct nmsghdr lmsg;
1226         struct uio auio;
1227         struct iovec aiov[1];
1228 
1229         dprint(1, ("send(%d, %p, %ld, %d)\n",
1230             sock, buffer, len, flags));
1231 
1232         if ((ssize_t)len < 0) {
1233                 return (set_errno(EINVAL));
1234         }
1235 
1236         aiov[0].iov_base = buffer;
1237         aiov[0].iov_len = len;
1238         auio.uio_loffset = 0;
1239         auio.uio_iov = aiov;
1240         auio.uio_iovcnt = 1;
1241         auio.uio_resid = len;
1242         auio.uio_segflg = UIO_USERSPACE;
1243         auio.uio_limit = 0;
1244 
1245         lmsg.msg_name = NULL;
1246         lmsg.msg_control = NULL;
1247         if (!(flags & MSG_XPG4_2)) {
1248                 /*
1249                  * In order to be compatible with the libsocket/sockmod
1250                  * implementation we set EOR for all send* calls.
1251                  */
1252                 flags |= MSG_EOR;
1253         }
1254         return (sendit(sock, &lmsg, &auio, flags));
1255 }
1256 
1257 /*
1258  * Uses the MSG_XPG4_2 flag to determine if the caller is using
1259  * struct omsghdr or struct nmsghdr.
1260  */
1261 ssize_t
1262 sendmsg(int sock, struct nmsghdr *msg, int flags)
1263 {
1264         struct nmsghdr lmsg;
1265         STRUCT_DECL(nmsghdr, u_lmsg);
1266         struct uio auio;
1267         struct iovec aiov[MSG_MAXIOVLEN];
1268         int iovcnt;
1269         ssize_t len;
1270         int i;
1271         model_t model;
1272 
1273         dprint(1, ("sendmsg(%d, %p, %d)\n", sock, (void *)msg, flags));
1274 
1275         model = get_udatamodel();
1276         STRUCT_INIT(u_lmsg, model);
1277 
1278         if (flags & MSG_XPG4_2) {
1279                 if (copyin(msg, (char *)STRUCT_BUF(u_lmsg),
1280                     STRUCT_SIZE(u_lmsg)))
1281                         return (set_errno(EFAULT));
1282         } else {
1283                 /*
1284                  * Assumes that nmsghdr and omsghdr are identically shaped
1285                  * except for the added msg_flags field.
1286                  */
1287                 if (copyin(msg, (char *)STRUCT_BUF(u_lmsg),
1288                     SIZEOF_STRUCT(omsghdr, model)))
1289                         return (set_errno(EFAULT));
1290                 /*
1291                  * In order to be compatible with the libsocket/sockmod
1292                  * implementation we set EOR for all send* calls.
1293                  */
1294                 flags |= MSG_EOR;
1295         }
1296 
1297         /*
1298          * Code below us will kmem_alloc memory and hang it
1299          * off msg_control and msg_name fields. This forces
1300          * us to copy the structure to its native form.
1301          */
1302         lmsg.msg_name = STRUCT_FGETP(u_lmsg, msg_name);
1303         lmsg.msg_namelen = STRUCT_FGET(u_lmsg, msg_namelen);
1304         lmsg.msg_iov = STRUCT_FGETP(u_lmsg, msg_iov);
1305         lmsg.msg_iovlen = STRUCT_FGET(u_lmsg, msg_iovlen);
1306         lmsg.msg_control = STRUCT_FGETP(u_lmsg, msg_control);
1307         lmsg.msg_controllen = STRUCT_FGET(u_lmsg, msg_controllen);
1308         lmsg.msg_flags = STRUCT_FGET(u_lmsg, msg_flags);
1309 
1310         iovcnt = lmsg.msg_iovlen;
1311 
1312         if (iovcnt <= 0 || iovcnt > MSG_MAXIOVLEN) {
1313                 /*
1314                  * Unless this is XPG 4.2 we allow iovcnt == 0 to
1315                  * be compatible with SunOS 4.X and 4.4BSD.
1316                  */
1317                 if (iovcnt != 0 || (flags & MSG_XPG4_2))
1318                         return (set_errno(EMSGSIZE));
1319         }
1320 
1321 #ifdef _SYSCALL32_IMPL
1322         /*
1323          * 32-bit callers need to have their iovec expanded, while ensuring
1324          * that they can't move more than 2Gbytes of data in a single call.
1325          */
1326         if (model == DATAMODEL_ILP32) {
1327                 struct iovec32 aiov32[MSG_MAXIOVLEN];
1328                 ssize32_t count32;
1329 
1330                 if (iovcnt != 0 &&
1331                     copyin((struct iovec32 *)lmsg.msg_iov, aiov32,
1332                     iovcnt * sizeof (struct iovec32)))
1333                         return (set_errno(EFAULT));
1334 
1335                 count32 = 0;
1336                 for (i = 0; i < iovcnt; i++) {
1337                         ssize32_t iovlen32;
1338 
1339                         iovlen32 = aiov32[i].iov_len;
1340                         count32 += iovlen32;
1341                         if (iovlen32 < 0 || count32 < 0)
1342                                 return (set_errno(EINVAL));
1343                         aiov[i].iov_len = iovlen32;
1344                         aiov[i].iov_base =
1345                             (caddr_t)(uintptr_t)aiov32[i].iov_base;
1346                 }
1347         } else
1348 #endif /* _SYSCALL32_IMPL */
1349         if (iovcnt != 0 &&
1350             copyin(lmsg.msg_iov, aiov,
1351             (unsigned)iovcnt * sizeof (struct iovec))) {
1352                 return (set_errno(EFAULT));
1353         }
1354         len = 0;
1355         for (i = 0; i < iovcnt; i++) {
1356                 ssize_t iovlen = aiov[i].iov_len;
1357                 len += iovlen;
1358                 if (iovlen < 0 || len < 0) {
1359                         return (set_errno(EINVAL));
1360                 }
1361         }
1362         auio.uio_loffset = 0;
1363         auio.uio_iov = aiov;
1364         auio.uio_iovcnt = iovcnt;
1365         auio.uio_resid = len;
1366         auio.uio_segflg = UIO_USERSPACE;
1367         auio.uio_limit = 0;
1368 
1369         return (sendit(sock, &lmsg, &auio, flags));
1370 }
1371 
1372 ssize_t
1373 sendto(int sock, void *buffer, size_t len, int flags,
1374     struct sockaddr *name, socklen_t namelen)
1375 {
1376         struct nmsghdr lmsg;
1377         struct uio auio;
1378         struct iovec aiov[1];
1379 
1380         dprint(1, ("sendto(%d, %p, %ld, %d, %p, %d)\n",
1381             sock, buffer, len, flags, (void *)name, namelen));
1382 
1383         if ((ssize_t)len < 0) {
1384                 return (set_errno(EINVAL));
1385         }
1386 
1387         aiov[0].iov_base = buffer;
1388         aiov[0].iov_len = len;
1389         auio.uio_loffset = 0;
1390         auio.uio_iov = aiov;
1391         auio.uio_iovcnt = 1;
1392         auio.uio_resid = len;
1393         auio.uio_segflg = UIO_USERSPACE;
1394         auio.uio_limit = 0;
1395 
1396         lmsg.msg_name = (char *)name;
1397         lmsg.msg_namelen = namelen;
1398         lmsg.msg_control = NULL;
1399         if (!(flags & MSG_XPG4_2)) {
1400                 /*
1401                  * In order to be compatible with the libsocket/sockmod
1402                  * implementation we set EOR for all send* calls.
1403                  */
1404                 flags |= MSG_EOR;
1405         }
1406         return (sendit(sock, &lmsg, &auio, flags));
1407 }
1408 
1409 /*ARGSUSED3*/
1410 int
1411 getpeername(int sock, struct sockaddr *name, socklen_t *namelenp, int version)
1412 {
1413         struct sonode *so;
1414         int error;
1415         socklen_t namelen;
1416         socklen_t sock_addrlen;
1417         struct sockaddr *sock_addrp;
1418 
1419         dprint(1, ("getpeername(%d, %p, %p)\n",
1420             sock, (void *)name, (void *)namelenp));
1421 
1422         if ((so = getsonode(sock, &error, NULL)) == NULL)
1423                 goto bad;
1424 
1425         ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1426         if (copyin(namelenp, &namelen, sizeof (namelen)) ||
1427             (name == NULL && namelen != 0)) {
1428                 error = EFAULT;
1429                 goto rel_out;
1430         }
1431         sock_addrlen = so->so_max_addr_len;
1432         sock_addrp = (struct sockaddr *)kmem_alloc(sock_addrlen, KM_SLEEP);
1433 
1434         if ((error = socket_getpeername(so, sock_addrp, &sock_addrlen,
1435             B_FALSE, CRED())) == 0) {
1436                 ASSERT(sock_addrlen <= so->so_max_addr_len);
1437                 error = copyout_name(name, namelen, namelenp,
1438                     (void *)sock_addrp, sock_addrlen);
1439         }
1440         kmem_free(sock_addrp, so->so_max_addr_len);
1441 rel_out:
1442         releasef(sock);
1443 bad:    return (error != 0 ? set_errno(error) : 0);
1444 }
1445 
1446 /*ARGSUSED3*/
1447 int
1448 getsockname(int sock, struct sockaddr *name,
1449                 socklen_t *namelenp, int version)
1450 {
1451         struct sonode *so;
1452         int error;
1453         socklen_t namelen, sock_addrlen;
1454         struct sockaddr *sock_addrp;
1455 
1456         dprint(1, ("getsockname(%d, %p, %p)\n",
1457             sock, (void *)name, (void *)namelenp));
1458 
1459         if ((so = getsonode(sock, &error, NULL)) == NULL)
1460                 goto bad;
1461 
1462         ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1463         if (copyin(namelenp, &namelen, sizeof (namelen)) ||
1464             (name == NULL && namelen != 0)) {
1465                 error = EFAULT;
1466                 goto rel_out;
1467         }
1468 
1469         sock_addrlen = so->so_max_addr_len;
1470         sock_addrp = (struct sockaddr *)kmem_alloc(sock_addrlen, KM_SLEEP);
1471         if ((error = socket_getsockname(so, sock_addrp, &sock_addrlen,
1472             CRED())) == 0) {
1473                 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1474                 ASSERT(sock_addrlen <= so->so_max_addr_len);
1475                 error = copyout_name(name, namelen, namelenp,
1476                     (void *)sock_addrp, sock_addrlen);
1477         }
1478         kmem_free(sock_addrp, so->so_max_addr_len);
1479 rel_out:
1480         releasef(sock);
1481 bad:    return (error != 0 ? set_errno(error) : 0);
1482 }
1483 
1484 /*ARGSUSED5*/
1485 int
1486 getsockopt(int sock,
1487         int level,
1488         int option_name,
1489         void *option_value,
1490         socklen_t *option_lenp,
1491         int version)
1492 {
1493         struct sonode *so;
1494         socklen_t optlen, optlen_res;
1495         void *optval;
1496         int error;
1497 
1498         dprint(1, ("getsockopt(%d, %d, %d, %p, %p)\n",
1499             sock, level, option_name, option_value, (void *)option_lenp));
1500 
1501         if ((so = getsonode(sock, &error, NULL)) == NULL)
1502                 return (set_errno(error));
1503 
1504         ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1505         if (copyin(option_lenp, &optlen, sizeof (optlen))) {
1506                 releasef(sock);
1507                 return (set_errno(EFAULT));
1508         }
1509         /*
1510          * Verify that the length is not excessive to prevent
1511          * an application from consuming all of kernel memory.
1512          */
1513         if (optlen > SO_MAXARGSIZE) {
1514                 error = EINVAL;
1515                 releasef(sock);
1516                 return (set_errno(error));
1517         }
1518         optval = kmem_alloc(optlen, KM_SLEEP);
1519         optlen_res = optlen;
1520         error = socket_getsockopt(so, level, option_name, optval,
1521             &optlen_res, (version != SOV_XPG4_2) ? 0 : _SOGETSOCKOPT_XPG4_2,
1522             CRED());
1523         releasef(sock);
1524         if (error) {
1525                 kmem_free(optval, optlen);
1526                 return (set_errno(error));
1527         }
1528         error = copyout_arg(option_value, optlen, option_lenp,
1529             optval, optlen_res);
1530         kmem_free(optval, optlen);
1531         if (error)
1532                 return (set_errno(error));
1533         return (0);
1534 }
1535 
1536 /*ARGSUSED5*/
1537 int
1538 setsockopt(int sock,
1539         int level,
1540         int option_name,
1541         void *option_value,
1542         socklen_t option_len,
1543         int version)
1544 {
1545         struct sonode *so;
1546         intptr_t buffer[2];
1547         void *optval = NULL;
1548         int error;
1549 
1550         dprint(1, ("setsockopt(%d, %d, %d, %p, %d)\n",
1551             sock, level, option_name, option_value, option_len));
1552 
1553         if ((so = getsonode(sock, &error, NULL)) == NULL)
1554                 return (set_errno(error));
1555 
1556         if (option_value != NULL) {
1557                 if (option_len != 0) {
1558                         /*
1559                          * Verify that the length is not excessive to prevent
1560                          * an application from consuming all of kernel memory.
1561                          */
1562                         if (option_len > SO_MAXARGSIZE) {
1563                                 error = EINVAL;
1564                                 goto done2;
1565                         }
1566                         optval = option_len <= sizeof (buffer) ?
1567                             &buffer : kmem_alloc((size_t)option_len, KM_SLEEP);
1568                         ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1569                         if (copyin(option_value, optval, (size_t)option_len)) {
1570                                 error = EFAULT;
1571                                 goto done1;
1572                         }
1573                 }
1574         } else
1575                 option_len = 0;
1576 
1577         error = socket_setsockopt(so, level, option_name, optval,
1578             (t_uscalar_t)option_len, CRED());
1579 done1:
1580         if (optval != buffer)
1581                 kmem_free(optval, (size_t)option_len);
1582 done2:
1583         releasef(sock);
1584         if (error)
1585                 return (set_errno(error));
1586         return (0);
1587 }
1588 
1589 static int
1590 sockconf_add_sock(int family, int type, int protocol, char *name)
1591 {
1592         int error = 0;
1593         char *kdevpath = NULL;
1594         char *kmodule = NULL;
1595         char *buf = NULL;
1596         size_t pathlen = 0;
1597         struct sockparams *sp;
1598 
1599         if (name == NULL)
1600                 return (EINVAL);
1601         /*
1602          * Copyin the name.
1603          * This also makes it possible to check for too long pathnames.
1604          * Compress the space needed for the name before passing it
1605          * to soconfig - soconfig will store the string until
1606          * the configuration is removed.
1607          */
1608         buf = kmem_alloc(MAXPATHLEN, KM_SLEEP);
1609         if ((error = copyinstr(name, buf, MAXPATHLEN, &pathlen)) != 0) {
1610                 kmem_free(buf, MAXPATHLEN);
1611                 return (error);
1612         }
1613         if (strncmp(buf, "/dev", strlen("/dev")) == 0) {
1614                 /* For device */
1615 
1616                 /*
1617                  * Special handling for NCA:
1618                  *
1619                  * DEV_NCA is never opened even if an application
1620                  * requests for AF_NCA. The device opened is instead a
1621                  * predefined AF_INET transport (NCA_INET_DEV).
1622                  *
1623                  * Prior to Volo (PSARC/2007/587) NCA would determine
1624                  * the device using a lookup, which worked then because
1625                  * all protocols were based on TPI. Since TPI is no
1626                  * longer the default, we have to explicitly state
1627                  * which device to use.
1628                  */
1629                 if (strcmp(buf, NCA_DEV) == 0) {
1630                         /* only support entry <28, 2, 0> */
1631                         if (family != AF_NCA || type != SOCK_STREAM ||
1632                             protocol != 0) {
1633                                 kmem_free(buf, MAXPATHLEN);
1634                                 return (EINVAL);
1635                         }
1636 
1637                         pathlen = strlen(NCA_INET_DEV) + 1;
1638                         kdevpath = kmem_alloc(pathlen, KM_SLEEP);
1639                         bcopy(NCA_INET_DEV, kdevpath, pathlen);
1640                         kdevpath[pathlen - 1] = '\0';
1641                 } else {
1642                         kdevpath = kmem_alloc(pathlen, KM_SLEEP);
1643                         bcopy(buf, kdevpath, pathlen);
1644                         kdevpath[pathlen - 1] = '\0';
1645                 }
1646         } else {
1647                 /* For socket module */
1648                 kmodule = kmem_alloc(pathlen, KM_SLEEP);
1649                 bcopy(buf, kmodule, pathlen);
1650                 kmodule[pathlen - 1] = '\0';
1651                 pathlen = 0;
1652         }
1653         kmem_free(buf, MAXPATHLEN);
1654 
1655         /* sockparams_create frees mod name and devpath upon failure */
1656         sp = sockparams_create(family, type, protocol, kmodule,
1657             kdevpath, pathlen, 0, KM_SLEEP, &error);
1658         if (sp != NULL) {
1659                 error = sockparams_add(sp);
1660                 if (error != 0)
1661                         sockparams_destroy(sp);
1662         }
1663 
1664         return (error);
1665 }
1666 
1667 static int
1668 sockconf_remove_sock(int family, int type, int protocol)
1669 {
1670         return (sockparams_delete(family, type, protocol));
1671 }
1672 
1673 static int
1674 sockconfig_remove_filter(const char *uname)
1675 {
1676         char kname[SOF_MAXNAMELEN];
1677         size_t len;
1678         int error;
1679         sof_entry_t *ent;
1680 
1681         if ((error = copyinstr(uname, kname, SOF_MAXNAMELEN, &len)) != 0)
1682                 return (error);
1683 
1684         ent = sof_entry_remove_by_name(kname);
1685         if (ent == NULL)
1686                 return (ENXIO);
1687 
1688         mutex_enter(&ent->sofe_lock);
1689         ASSERT(!(ent->sofe_flags & SOFEF_CONDEMED));
1690         if (ent->sofe_refcnt == 0) {
1691                 mutex_exit(&ent->sofe_lock);
1692                 sof_entry_free(ent);
1693         } else {
1694                 /* let the last socket free the filter */
1695                 ent->sofe_flags |= SOFEF_CONDEMED;
1696                 mutex_exit(&ent->sofe_lock);
1697         }
1698 
1699         return (0);
1700 }
1701 
1702 static int
1703 sockconfig_add_filter(const char *uname, void *ufilpropp)
1704 {
1705         struct sockconfig_filter_props filprop;
1706         sof_entry_t *ent;
1707         int error;
1708         size_t tuplesz, len;
1709         char hintbuf[SOF_MAXNAMELEN];
1710 
1711         ent = kmem_zalloc(sizeof (sof_entry_t), KM_SLEEP);
1712         mutex_init(&ent->sofe_lock, NULL, MUTEX_DEFAULT, NULL);
1713 
1714         if ((error = copyinstr(uname, ent->sofe_name, SOF_MAXNAMELEN,
1715             &len)) != 0) {
1716                 sof_entry_free(ent);
1717                 return (error);
1718         }
1719 
1720         if (get_udatamodel() == DATAMODEL_NATIVE) {
1721                 if (copyin(ufilpropp, &filprop, sizeof (filprop)) != 0) {
1722                         sof_entry_free(ent);
1723                         return (EFAULT);
1724                 }
1725         }
1726 #ifdef  _SYSCALL32_IMPL
1727         else {
1728                 struct sockconfig_filter_props32 filprop32;
1729 
1730                 if (copyin(ufilpropp, &filprop32, sizeof (filprop32)) != 0) {
1731                         sof_entry_free(ent);
1732                         return (EFAULT);
1733                 }
1734                 filprop.sfp_modname = (char *)(uintptr_t)filprop32.sfp_modname;
1735                 filprop.sfp_autoattach = filprop32.sfp_autoattach;
1736                 filprop.sfp_hint = filprop32.sfp_hint;
1737                 filprop.sfp_hintarg = (char *)(uintptr_t)filprop32.sfp_hintarg;
1738                 filprop.sfp_socktuple_cnt = filprop32.sfp_socktuple_cnt;
1739                 filprop.sfp_socktuple =
1740                     (sof_socktuple_t *)(uintptr_t)filprop32.sfp_socktuple;
1741         }
1742 #endif  /* _SYSCALL32_IMPL */
1743 
1744         if ((error = copyinstr(filprop.sfp_modname, ent->sofe_modname,
1745             sizeof (ent->sofe_modname), &len)) != 0) {
1746                 sof_entry_free(ent);
1747                 return (error);
1748         }
1749 
1750         /*
1751          * A filter must specify at least one socket tuple.
1752          */
1753         if (filprop.sfp_socktuple_cnt == 0 ||
1754             filprop.sfp_socktuple_cnt > SOF_MAXSOCKTUPLECNT) {
1755                 sof_entry_free(ent);
1756                 return (EINVAL);
1757         }
1758         ent->sofe_flags = filprop.sfp_autoattach ? SOFEF_AUTO : SOFEF_PROG;
1759         ent->sofe_hint = filprop.sfp_hint;
1760 
1761         /*
1762          * Verify the hint, and copy in the hint argument, if necessary.
1763          */
1764         switch (ent->sofe_hint) {
1765         case SOF_HINT_BEFORE:
1766         case SOF_HINT_AFTER:
1767                 if ((error = copyinstr(filprop.sfp_hintarg, hintbuf,
1768                     sizeof (hintbuf), &len)) != 0) {
1769                         sof_entry_free(ent);
1770                         return (error);
1771                 }
1772                 ent->sofe_hintarg = kmem_alloc(len, KM_SLEEP);
1773                 bcopy(hintbuf, ent->sofe_hintarg, len);
1774                 /* FALLTHRU */
1775         case SOF_HINT_TOP:
1776         case SOF_HINT_BOTTOM:
1777                 /* hints cannot be used with programmatic filters */
1778                 if (ent->sofe_flags & SOFEF_PROG) {
1779                         sof_entry_free(ent);
1780                         return (EINVAL);
1781                 }
1782                 break;
1783         case SOF_HINT_NONE:
1784                 break;
1785         default:
1786                 /* bad hint value */
1787                 sof_entry_free(ent);
1788                 return (EINVAL);
1789         }
1790 
1791         ent->sofe_socktuple_cnt = filprop.sfp_socktuple_cnt;
1792         tuplesz = sizeof (sof_socktuple_t) * ent->sofe_socktuple_cnt;
1793         ent->sofe_socktuple = kmem_alloc(tuplesz, KM_SLEEP);
1794 
1795         if (get_udatamodel() == DATAMODEL_NATIVE) {
1796                 if (copyin(filprop.sfp_socktuple, ent->sofe_socktuple,
1797                     tuplesz)) {
1798                         sof_entry_free(ent);
1799                         return (EFAULT);
1800                 }
1801         }
1802 #ifdef  _SYSCALL32_IMPL
1803         else {
1804                 int i;
1805                 caddr_t data = (caddr_t)filprop.sfp_socktuple;
1806                 sof_socktuple_t *tup = ent->sofe_socktuple;
1807                 sof_socktuple32_t tup32;
1808 
1809                 tup = ent->sofe_socktuple;
1810                 for (i = 0; i < ent->sofe_socktuple_cnt; i++, tup++) {
1811                         ASSERT(tup < ent->sofe_socktuple + tuplesz);
1812 
1813                         if (copyin(data, &tup32, sizeof (tup32)) != 0) {
1814                                 sof_entry_free(ent);
1815                                 return (EFAULT);
1816                         }
1817                         tup->sofst_family = tup32.sofst_family;
1818                         tup->sofst_type = tup32.sofst_type;
1819                         tup->sofst_protocol = tup32.sofst_protocol;
1820 
1821                         data += sizeof (tup32);
1822                 }
1823         }
1824 #endif  /* _SYSCALL32_IMPL */
1825 
1826         /* Sockets can start using the filter as soon as the filter is added */
1827         if ((error = sof_entry_add(ent)) != 0)
1828                 sof_entry_free(ent);
1829 
1830         return (error);
1831 }
1832 
1833 /*
1834  * Socket configuration system call. It is used to add and remove
1835  * socket types.
1836  */
1837 int
1838 sockconfig(int cmd, void *arg1, void *arg2, void *arg3, void *arg4)
1839 {
1840         int error = 0;
1841 
1842         if (secpolicy_net_config(CRED(), B_FALSE) != 0)
1843                 return (set_errno(EPERM));
1844 
1845         if (sockfs_defer_nl7c_init) {
1846                 nl7c_init();
1847                 sockfs_defer_nl7c_init = 0;
1848         }
1849 
1850         switch (cmd) {
1851         case SOCKCONFIG_ADD_SOCK:
1852                 error = sockconf_add_sock((int)(uintptr_t)arg1,
1853                     (int)(uintptr_t)arg2, (int)(uintptr_t)arg3, arg4);
1854                 break;
1855         case SOCKCONFIG_REMOVE_SOCK:
1856                 error = sockconf_remove_sock((int)(uintptr_t)arg1,
1857                     (int)(uintptr_t)arg2, (int)(uintptr_t)arg3);
1858                 break;
1859         case SOCKCONFIG_ADD_FILTER:
1860                 error = sockconfig_add_filter((const char *)arg1, arg2);
1861                 break;
1862         case SOCKCONFIG_REMOVE_FILTER:
1863                 error = sockconfig_remove_filter((const char *)arg1);
1864                 break;
1865         default:
1866 #ifdef  DEBUG
1867                 cmn_err(CE_NOTE, "sockconfig: unkonwn subcommand %d", cmd);
1868 #endif
1869                 error = EINVAL;
1870                 break;
1871         }
1872 
1873         if (error != 0) {
1874                 eprintline(error);
1875                 return (set_errno(error));
1876         }
1877         return (0);
1878 }
1879 
1880 
1881 /*
1882  * Sendfile is implemented through two schemes, direct I/O or by
1883  * caching in the filesystem page cache. We cache the input file by
1884  * default and use direct I/O only if sendfile_max_size is set
1885  * appropriately as explained below. Note that this logic is consistent
1886  * with other filesystems where caching is turned on by default
1887  * unless explicitly turned off by using the DIRECTIO ioctl.
1888  *
1889  * We choose a slightly different scheme here. One can turn off
1890  * caching by setting sendfile_max_size to 0. One can also enable
1891  * caching of files <= sendfile_max_size by setting sendfile_max_size
1892  * to an appropriate value. By default sendfile_max_size is set to the
1893  * maximum value so that all files are cached. In future, we may provide
1894  * better interfaces for caching the file.
1895  *
1896  * Sendfile through Direct I/O (Zero copy)
1897  * --------------------------------------
1898  *
1899  * As disks are normally slower than the network, we can't have a
1900  * single thread that reads the disk and writes to the network. We
1901  * need to have parallelism. This is done by having the sendfile
1902  * thread create another thread that reads from the filesystem
1903  * and queues it for network processing. In this scheme, the data
1904  * is never copied anywhere i.e it is zero copy unlike the other
1905  * scheme.
1906  *
1907  * We have a sendfile queue (snfq) where each sendfile
1908  * request (snf_req_t) is queued for processing by a thread. Number
1909  * of threads is dynamically allocated and they exit if they are idling
1910  * beyond a specified amount of time. When each request (snf_req_t) is
1911  * processed by a thread, it produces a number of mblk_t structures to
1912  * be consumed by the sendfile thread. snf_deque and snf_enque are
1913  * used for consuming and producing mblks. Size of the filesystem
1914  * read is determined by the tunable (sendfile_read_size). A single
1915  * mblk holds sendfile_read_size worth of data (except the last
1916  * read of the file) which is sent down as a whole to the network.
1917  * sendfile_read_size is set to 1 MB as this seems to be the optimal
1918  * value for the UFS filesystem backed by a striped storage array.
1919  *
1920  * Synchronisation between read (producer) and write (consumer) threads.
1921  * --------------------------------------------------------------------
1922  *
1923  * sr_lock protects sr_ib_head and sr_ib_tail. The lock is held while
1924  * adding and deleting items in this list. Error can happen anytime
1925  * during read or write. There could be unprocessed mblks in the
1926  * sr_ib_XXX list when a read or write error occurs. Whenever error
1927  * is encountered, we need two things to happen :
1928  *
1929  * a) One of the threads need to clean the mblks.
1930  * b) When one thread encounters an error, the other should stop.
1931  *
1932  * For (a), we don't want to penalize the reader thread as it could do
1933  * some useful work processing other requests. For (b), the error can
1934  * be detected by examining sr_read_error or sr_write_error.
1935  * sr_lock protects sr_read_error and sr_write_error. If both reader and
1936  * writer encounters error, we need to report the write error back to
1937  * the application as that's what would have happened if the operations
1938  * were done sequentially. With this in mind, following should work :
1939  *
1940  *      - Check for errors before read or write.
1941  *      - If the reader encounters error, set the error in sr_read_error.
1942  *        Check sr_write_error, if it is set, send cv_signal as it is
1943  *        waiting for reader to complete. If it is not set, the writer
1944  *        is either running sinking data to the network or blocked
1945  *        because of flow control. For handling the latter case, we
1946  *        always send a signal. In any case, it will examine sr_read_error
1947  *        and return. sr_read_error is marked with SR_READ_DONE to tell
1948  *        the writer that the reader is done in all the cases.
1949  *      - If the writer encounters error, set the error in sr_write_error.
1950  *        The reader thread is either blocked because of flow control or
1951  *        running reading data from the disk. For the former, we need to
1952  *        wakeup the thread. Again to keep it simple, we always wake up
1953  *        the reader thread. Then, wait for the read thread to complete
1954  *        if it is not done yet. Cleanup and return.
1955  *
1956  * High and low water marks for the read thread.
1957  * --------------------------------------------
1958  *
1959  * If sendfile() is used to send data over a slow network, we need to
1960  * make sure that the read thread does not produce data at a faster
1961  * rate than the network. This can happen if the disk is faster than
1962  * the network. In such a case, we don't want to build a very large queue.
1963  * But we would still like to get all of the network throughput possible.
1964  * This implies that network should never block waiting for data.
1965  * As there are lot of disk throughput/network throughput combinations
1966  * possible, it is difficult to come up with an accurate number.
1967  * A typical 10K RPM disk has a max seek latency 17ms and rotational
1968  * latency of 3ms for reading a disk block. Thus, the total latency to
1969  * initiate a new read, transfer data from the disk and queue for
1970  * transmission would take about a max of 25ms. Todays max transfer rate
1971  * for network is 100MB/sec. If the thread is blocked because of flow
1972  * control, it would take 25ms to get new data ready for transmission.
1973  * We have to make sure that network is not idling, while we are initiating
1974  * new transfers. So, at 100MB/sec, to keep network busy we would need
1975  * 2.5MB of data. Rounding off, we keep the low water mark to be 3MB of data.
1976  * We need to pick a high water mark so that the woken up thread would
1977  * do considerable work before blocking again to prevent thrashing. Currently,
1978  * we pick this to be 10 times that of the low water mark.
1979  *
1980  * Sendfile with segmap caching (One copy from page cache to mblks).
1981  * ----------------------------------------------------------------
1982  *
1983  * We use the segmap cache for caching the file, if the size of file
1984  * is <= sendfile_max_size. In this case we don't use threads as VM
1985  * is reasonably fast enough to keep up with the network. If the underlying
1986  * transport allows, we call segmap_getmapflt() to map MAXBSIZE (8K) worth
1987  * of data into segmap space, and use the virtual address from segmap
1988  * directly through desballoc() to avoid copy. Once the transport is done
1989  * with the data, the mapping will be released through segmap_release()
1990  * called by the call-back routine.
1991  *
1992  * If zero-copy is not allowed by the transport, we simply call VOP_READ()
1993  * to copy the data from the filesystem into our temporary network buffer.
1994  *
1995  * To disable caching, set sendfile_max_size to 0.
1996  */
1997 
1998 uint_t sendfile_read_size = 1024 * 1024;
1999 #define SENDFILE_REQ_LOWAT      3 * 1024 * 1024
2000 uint_t sendfile_req_lowat = SENDFILE_REQ_LOWAT;
2001 uint_t sendfile_req_hiwat = 10 * SENDFILE_REQ_LOWAT;
2002 struct sendfile_stats sf_stats;
2003 struct sendfile_queue *snfq;
2004 clock_t snfq_timeout;
2005 off64_t sendfile_max_size;
2006 
2007 static void snf_enque(snf_req_t *, mblk_t *);
2008 static mblk_t *snf_deque(snf_req_t *);
2009 
2010 void
2011 sendfile_init(void)
2012 {
2013         snfq = kmem_zalloc(sizeof (struct sendfile_queue), KM_SLEEP);
2014 
2015         mutex_init(&snfq->snfq_lock, NULL, MUTEX_DEFAULT, NULL);
2016         cv_init(&snfq->snfq_cv, NULL, CV_DEFAULT, NULL);
2017         snfq->snfq_max_threads = max_ncpus;
2018         snfq_timeout = SNFQ_TIMEOUT;
2019         /* Cache all files by default. */
2020         sendfile_max_size = MAXOFFSET_T;
2021 }
2022 
2023 /*
2024  * Queues a mblk_t for network processing.
2025  */
2026 static void
2027 snf_enque(snf_req_t *sr, mblk_t *mp)
2028 {
2029         mp->b_next = NULL;
2030         mutex_enter(&sr->sr_lock);
2031         if (sr->sr_mp_head == NULL) {
2032                 sr->sr_mp_head = sr->sr_mp_tail = mp;
2033                 cv_signal(&sr->sr_cv);
2034         } else {
2035                 sr->sr_mp_tail->b_next = mp;
2036                 sr->sr_mp_tail = mp;
2037         }
2038         sr->sr_qlen += MBLKL(mp);
2039         while ((sr->sr_qlen > sr->sr_hiwat) &&
2040             (sr->sr_write_error == 0)) {
2041                 sf_stats.ss_full_waits++;
2042                 cv_wait(&sr->sr_cv, &sr->sr_lock);
2043         }
2044         mutex_exit(&sr->sr_lock);
2045 }
2046 
2047 /*
2048  * De-queues a mblk_t for network processing.
2049  */
2050 static mblk_t *
2051 snf_deque(snf_req_t *sr)
2052 {
2053         mblk_t *mp;
2054 
2055         mutex_enter(&sr->sr_lock);
2056         /*
2057          * If we have encountered an error on read or read is
2058          * completed and no more mblks, return NULL.
2059          * We need to check for NULL sr_mp_head also as
2060          * the reads could have completed and there is
2061          * nothing more to come.
2062          */
2063         if (((sr->sr_read_error & ~SR_READ_DONE) != 0) ||
2064             ((sr->sr_read_error & SR_READ_DONE) &&
2065             sr->sr_mp_head == NULL)) {
2066                 mutex_exit(&sr->sr_lock);
2067                 return (NULL);
2068         }
2069         /*
2070          * To start with neither SR_READ_DONE is marked nor
2071          * the error is set. When we wake up from cv_wait,
2072          * following are the possibilities :
2073          *
2074          *      a) sr_read_error is zero and mblks are queued.
2075          *      b) sr_read_error is set to SR_READ_DONE
2076          *         and mblks are queued.
2077          *      c) sr_read_error is set to SR_READ_DONE
2078          *         and no mblks.
2079          *      d) sr_read_error is set to some error other
2080          *         than SR_READ_DONE.
2081          */
2082 
2083         while ((sr->sr_read_error == 0) && (sr->sr_mp_head == NULL)) {
2084                 sf_stats.ss_empty_waits++;
2085                 cv_wait(&sr->sr_cv, &sr->sr_lock);
2086         }
2087         /* Handle (a) and (b) first  - the normal case. */
2088         if (((sr->sr_read_error & ~SR_READ_DONE) == 0) &&
2089             (sr->sr_mp_head != NULL)) {
2090                 mp = sr->sr_mp_head;
2091                 sr->sr_mp_head = mp->b_next;
2092                 sr->sr_qlen -= MBLKL(mp);
2093                 if (sr->sr_qlen < sr->sr_lowat)
2094                         cv_signal(&sr->sr_cv);
2095                 mutex_exit(&sr->sr_lock);
2096                 mp->b_next = NULL;
2097                 return (mp);
2098         }
2099         /* Handle (c) and (d). */
2100         mutex_exit(&sr->sr_lock);
2101         return (NULL);
2102 }
2103 
2104 /*
2105  * Reads data from the filesystem and queues it for network processing.
2106  */
2107 void
2108 snf_async_read(snf_req_t *sr)
2109 {
2110         size_t iosize;
2111         u_offset_t fileoff;
2112         u_offset_t size;
2113         int ret_size;
2114         int error;
2115         file_t *fp;
2116         mblk_t *mp;
2117         struct vnode *vp;
2118         int extra = 0;
2119         int maxblk = 0;
2120         int wroff = 0;
2121         struct sonode *so;
2122 
2123         fp = sr->sr_fp;
2124         size = sr->sr_file_size;
2125         fileoff = sr->sr_file_off;
2126 
2127         /*
2128          * Ignore the error for filesystems that doesn't support DIRECTIO.
2129          */
2130         (void) VOP_IOCTL(fp->f_vnode, _FIODIRECTIO, DIRECTIO_ON, 0,
2131             kcred, NULL, NULL);
2132 
2133         vp = sr->sr_vp;
2134         if (vp->v_type == VSOCK) {
2135                 stdata_t *stp;
2136 
2137                 /*
2138                  * Get the extra space to insert a header and a trailer.
2139                  */
2140                 so = VTOSO(vp);
2141                 stp = vp->v_stream;
2142                 if (stp == NULL) {
2143                         wroff = so->so_proto_props.sopp_wroff;
2144                         maxblk = so->so_proto_props.sopp_maxblk;
2145                         extra = wroff + so->so_proto_props.sopp_tail;
2146                 } else {
2147                         wroff = (int)(stp->sd_wroff);
2148                         maxblk = (int)(stp->sd_maxblk);
2149                         extra = wroff + (int)(stp->sd_tail);
2150                 }
2151         }
2152 
2153         while ((size != 0) && (sr->sr_write_error == 0)) {
2154 
2155                 iosize = (int)MIN(sr->sr_maxpsz, size);
2156 
2157                 /*
2158                  * Socket filters can limit the mblk size,
2159                  * so limit reads to maxblk if there are
2160                  * filters present.
2161                  */
2162                 if (vp->v_type == VSOCK &&
2163                     so->so_filter_active > 0 && maxblk != INFPSZ)
2164                         iosize = (int)MIN(iosize, maxblk);
2165 
2166                 if (is_system_labeled()) {
2167                         mp = allocb_cred(iosize + extra, CRED(),
2168                             curproc->p_pid);
2169                 } else {
2170                         mp = allocb(iosize + extra, BPRI_MED);
2171                 }
2172                 if (mp == NULL) {
2173                         error = EAGAIN;
2174                         break;
2175                 }
2176 
2177                 mp->b_rptr += wroff;
2178 
2179                 ret_size = soreadfile(fp, mp->b_rptr, fileoff, &error, iosize);
2180 
2181                 /* Error or Reached EOF ? */
2182                 if ((error != 0) || (ret_size == 0)) {
2183                         freeb(mp);
2184                         break;
2185                 }
2186                 mp->b_wptr = mp->b_rptr + ret_size;
2187 
2188                 snf_enque(sr, mp);
2189                 size -= ret_size;
2190                 fileoff += ret_size;
2191         }
2192         (void) VOP_IOCTL(fp->f_vnode, _FIODIRECTIO, DIRECTIO_OFF, 0,
2193             kcred, NULL, NULL);
2194         mutex_enter(&sr->sr_lock);
2195         sr->sr_read_error = error;
2196         sr->sr_read_error |= SR_READ_DONE;
2197         cv_signal(&sr->sr_cv);
2198         mutex_exit(&sr->sr_lock);
2199 }
2200 
2201 void
2202 snf_async_thread(void)
2203 {
2204         snf_req_t *sr;
2205         callb_cpr_t cprinfo;
2206         clock_t time_left = 1;
2207 
2208         CALLB_CPR_INIT(&cprinfo, &snfq->snfq_lock, callb_generic_cpr, "snfq");
2209 
2210         mutex_enter(&snfq->snfq_lock);
2211         for (;;) {
2212                 /*
2213                  * If we didn't find a entry, then block until woken up
2214                  * again and then look through the queues again.
2215                  */
2216                 while ((sr = snfq->snfq_req_head) == NULL) {
2217                         CALLB_CPR_SAFE_BEGIN(&cprinfo);
2218                         if (time_left <= 0) {
2219                                 snfq->snfq_svc_threads--;
2220                                 CALLB_CPR_EXIT(&cprinfo);
2221                                 thread_exit();
2222                                 /* NOTREACHED */
2223                         }
2224                         snfq->snfq_idle_cnt++;
2225 
2226                         time_left = cv_reltimedwait(&snfq->snfq_cv,
2227                             &snfq->snfq_lock, snfq_timeout, TR_CLOCK_TICK);
2228                         snfq->snfq_idle_cnt--;
2229 
2230                         CALLB_CPR_SAFE_END(&cprinfo, &snfq->snfq_lock);
2231                 }
2232                 snfq->snfq_req_head = sr->sr_next;
2233                 snfq->snfq_req_cnt--;
2234                 mutex_exit(&snfq->snfq_lock);
2235                 snf_async_read(sr);
2236                 mutex_enter(&snfq->snfq_lock);
2237         }
2238 }
2239 
2240 
2241 snf_req_t *
2242 create_thread(int operation, struct vnode *vp, file_t *fp,
2243     u_offset_t fileoff, u_offset_t size)
2244 {
2245         snf_req_t *sr;
2246         stdata_t *stp;
2247 
2248         sr = (snf_req_t *)kmem_zalloc(sizeof (snf_req_t), KM_SLEEP);
2249 
2250         sr->sr_vp = vp;
2251         sr->sr_fp = fp;
2252         stp = vp->v_stream;
2253 
2254         /*
2255          * store sd_qn_maxpsz into sr_maxpsz while we have stream head.
2256          * stream might be closed before thread returns from snf_async_read.
2257          */
2258         if (stp != NULL && stp->sd_qn_maxpsz > 0) {
2259                 sr->sr_maxpsz = MIN(MAXBSIZE, stp->sd_qn_maxpsz);
2260         } else {
2261                 sr->sr_maxpsz = MAXBSIZE;
2262         }
2263 
2264         sr->sr_operation = operation;
2265         sr->sr_file_off = fileoff;
2266         sr->sr_file_size = size;
2267         sr->sr_hiwat = sendfile_req_hiwat;
2268         sr->sr_lowat = sendfile_req_lowat;
2269         mutex_init(&sr->sr_lock, NULL, MUTEX_DEFAULT, NULL);
2270         cv_init(&sr->sr_cv, NULL, CV_DEFAULT, NULL);
2271         /*
2272          * See whether we need another thread for servicing this
2273          * request. If there are already enough requests queued
2274          * for the threads, create one if not exceeding
2275          * snfq_max_threads.
2276          */
2277         mutex_enter(&snfq->snfq_lock);
2278         if (snfq->snfq_req_cnt >= snfq->snfq_idle_cnt &&
2279             snfq->snfq_svc_threads < snfq->snfq_max_threads) {
2280                 (void) thread_create(NULL, 0, &snf_async_thread, 0, 0, &p0,
2281                     TS_RUN, minclsyspri);
2282                 snfq->snfq_svc_threads++;
2283         }
2284         if (snfq->snfq_req_head == NULL) {
2285                 snfq->snfq_req_head = snfq->snfq_req_tail = sr;
2286                 cv_signal(&snfq->snfq_cv);
2287         } else {
2288                 snfq->snfq_req_tail->sr_next = sr;
2289                 snfq->snfq_req_tail = sr;
2290         }
2291         snfq->snfq_req_cnt++;
2292         mutex_exit(&snfq->snfq_lock);
2293         return (sr);
2294 }
2295 
2296 int
2297 snf_direct_io(file_t *fp, file_t *rfp, u_offset_t fileoff, u_offset_t size,
2298     ssize_t *count)
2299 {
2300         snf_req_t *sr;
2301         mblk_t *mp;
2302         int iosize;
2303         int error = 0;
2304         short fflag;
2305         struct vnode *vp;
2306         int ksize;
2307         struct nmsghdr msg;
2308 
2309         ksize = 0;
2310         *count = 0;
2311         bzero(&msg, sizeof (msg));
2312 
2313         vp = fp->f_vnode;
2314         fflag = fp->f_flag;
2315         if ((sr = create_thread(READ_OP, vp, rfp, fileoff, size)) == NULL)
2316                 return (EAGAIN);
2317 
2318         /*
2319          * We check for read error in snf_deque. It has to check
2320          * for successful READ_DONE and return NULL, and we might
2321          * as well make an additional check there.
2322          */
2323         while ((mp = snf_deque(sr)) != NULL) {
2324 
2325                 if (ISSIG(curthread, JUSTLOOKING)) {
2326                         freeb(mp);
2327                         error = EINTR;
2328                         break;
2329                 }
2330                 iosize = MBLKL(mp);
2331 
2332                 error = socket_sendmblk(VTOSO(vp), &msg, fflag, CRED(), &mp);
2333 
2334                 if (error != 0) {
2335                         if (mp != NULL)
2336                                 freeb(mp);
2337                         break;
2338                 }
2339                 ksize += iosize;
2340         }
2341         *count = ksize;
2342 
2343         mutex_enter(&sr->sr_lock);
2344         sr->sr_write_error = error;
2345         /* Look at the big comments on why we cv_signal here. */
2346         cv_signal(&sr->sr_cv);
2347 
2348         /* Wait for the reader to complete always. */
2349         while (!(sr->sr_read_error & SR_READ_DONE)) {
2350                 cv_wait(&sr->sr_cv, &sr->sr_lock);
2351         }
2352         /* If there is no write error, check for read error. */
2353         if (error == 0)
2354                 error = (sr->sr_read_error & ~SR_READ_DONE);
2355 
2356         if (error != 0) {
2357                 mblk_t *next_mp;
2358 
2359                 mp = sr->sr_mp_head;
2360                 while (mp != NULL) {
2361                         next_mp = mp->b_next;
2362                         mp->b_next = NULL;
2363                         freeb(mp);
2364                         mp = next_mp;
2365                 }
2366         }
2367         mutex_exit(&sr->sr_lock);
2368         kmem_free(sr, sizeof (snf_req_t));
2369         return (error);
2370 }
2371 
2372 /* Maximum no.of pages allocated by vpm for sendfile at a time */
2373 #define SNF_VPMMAXPGS   (VPMMAXPGS/2)
2374 
2375 /*
2376  * Maximum no.of elements in the list returned by vpm, including
2377  * NULL for the last entry
2378  */
2379 #define SNF_MAXVMAPS    (SNF_VPMMAXPGS + 1)
2380 
2381 typedef struct {
2382         unsigned int    snfv_ref;
2383         frtn_t          snfv_frtn;
2384         vnode_t         *snfv_vp;
2385         struct vmap     snfv_vml[SNF_MAXVMAPS];
2386 } snf_vmap_desbinfo;
2387 
2388 typedef struct {
2389         frtn_t          snfi_frtn;
2390         caddr_t         snfi_base;
2391         uint_t          snfi_mapoff;
2392         size_t          snfi_len;
2393         vnode_t         *snfi_vp;
2394 } snf_smap_desbinfo;
2395 
2396 /*
2397  * The callback function used for vpm mapped mblks called when the last ref of
2398  * the mblk is dropped which normally occurs when TCP receives the ack. But it
2399  * can be the driver too due to lazy reclaim.
2400  */
2401 void
2402 snf_vmap_desbfree(snf_vmap_desbinfo *snfv)
2403 {
2404         ASSERT(snfv->snfv_ref != 0);
2405         if (atomic_dec_32_nv(&snfv->snfv_ref) == 0) {
2406                 vpm_unmap_pages(snfv->snfv_vml, S_READ);
2407                 VN_RELE(snfv->snfv_vp);
2408                 kmem_free(snfv, sizeof (snf_vmap_desbinfo));
2409         }
2410 }
2411 
2412 /*
2413  * The callback function used for segmap'ped mblks called when the last ref of
2414  * the mblk is dropped which normally occurs when TCP receives the ack. But it
2415  * can be the driver too due to lazy reclaim.
2416  */
2417 void
2418 snf_smap_desbfree(snf_smap_desbinfo *snfi)
2419 {
2420         if (! IS_KPM_ADDR(snfi->snfi_base)) {
2421                 /*
2422                  * We don't need to call segmap_fault(F_SOFTUNLOCK) for
2423                  * segmap_kpm as long as the latter never falls back to
2424                  * "use_segmap_range". (See segmap_getmapflt().)
2425                  *
2426                  * Using S_OTHER saves an redundant hat_setref() in
2427                  * segmap_unlock()
2428                  */
2429                 (void) segmap_fault(kas.a_hat, segkmap,
2430                     (caddr_t)(uintptr_t)(((uintptr_t)snfi->snfi_base +
2431                     snfi->snfi_mapoff) & PAGEMASK), snfi->snfi_len,
2432                     F_SOFTUNLOCK, S_OTHER);
2433         }
2434         (void) segmap_release(segkmap, snfi->snfi_base, SM_DONTNEED);
2435         VN_RELE(snfi->snfi_vp);
2436         kmem_free(snfi, sizeof (*snfi));
2437 }
2438 
2439 /*
2440  * Use segmap or vpm instead of bcopy to send down a desballoca'ed, mblk.
2441  * When segmap is used, the mblk contains a segmap slot of no more
2442  * than MAXBSIZE.
2443  *
2444  * With vpm, a maximum of SNF_MAXVMAPS page-sized mappings can be obtained
2445  * in each iteration and sent by socket_sendmblk until an error occurs or
2446  * the requested size has been transferred. An mblk is esballoca'ed from
2447  * each mapped page and a chain of these mblk is sent to the transport layer.
2448  * vpm will be called to unmap the pages when all mblks have been freed by
2449  * free_func.
2450  *
2451  * At the end of the whole sendfile() operation, we wait till the data from
2452  * the last mblk is ack'ed by the transport before returning so that the
2453  * caller of sendfile() can safely modify the file content.
2454  */
2455 int
2456 snf_segmap(file_t *fp, vnode_t *fvp, u_offset_t fileoff, u_offset_t total_size,
2457     ssize_t *count, boolean_t nowait)
2458 {
2459         caddr_t base;
2460         int mapoff;
2461         vnode_t *vp;
2462         mblk_t *mp = NULL;
2463         int chain_size;
2464         int error;
2465         clock_t deadlk_wait;
2466         short fflag;
2467         int ksize;
2468         struct vattr va;
2469         boolean_t dowait = B_FALSE;
2470         struct nmsghdr msg;
2471 
2472         vp = fp->f_vnode;
2473         fflag = fp->f_flag;
2474         ksize = 0;
2475         bzero(&msg, sizeof (msg));
2476 
2477         for (;;) {
2478                 if (ISSIG(curthread, JUSTLOOKING)) {
2479                         error = EINTR;
2480                         break;
2481                 }
2482 
2483                 if (vpm_enable) {
2484                         snf_vmap_desbinfo *snfv;
2485                         mblk_t *nmp;
2486                         int mblk_size;
2487                         int maxsize;
2488                         int i;
2489 
2490                         mapoff = fileoff & PAGEOFFSET;
2491                         maxsize = MIN((SNF_VPMMAXPGS * PAGESIZE), total_size);
2492 
2493                         snfv = kmem_zalloc(sizeof (snf_vmap_desbinfo),
2494                             KM_SLEEP);
2495 
2496                         /*
2497                          * Get vpm mappings for maxsize with read access.
2498                          * If the pages aren't available yet, we get
2499                          * DEADLK, so wait and try again a little later using
2500                          * an increasing wait. We might be here a long time.
2501                          *
2502                          * If delay_sig returns EINTR, be sure to exit and
2503                          * pass it up to the caller.
2504                          */
2505                         deadlk_wait = 0;
2506                         while ((error = vpm_map_pages(fvp, fileoff,
2507                             (size_t)maxsize, (VPM_FETCHPAGE), snfv->snfv_vml,
2508                             SNF_MAXVMAPS, NULL, S_READ)) == EDEADLK) {
2509                                 deadlk_wait += (deadlk_wait < 5) ? 1 : 4;
2510                                 if ((error = delay_sig(deadlk_wait)) != 0) {
2511                                         break;
2512                                 }
2513                         }
2514                         if (error != 0) {
2515                                 kmem_free(snfv, sizeof (snf_vmap_desbinfo));
2516                                 error = (error == EINTR) ? EINTR : EIO;
2517                                 goto out;
2518                         }
2519                         snfv->snfv_frtn.free_func = snf_vmap_desbfree;
2520                         snfv->snfv_frtn.free_arg = (caddr_t)snfv;
2521 
2522                         /* Construct the mblk chain from the page mappings */
2523                         chain_size = 0;
2524                         for (i = 0; (snfv->snfv_vml[i].vs_addr != NULL) &&
2525                             total_size > 0; i++) {
2526                                 ASSERT(chain_size < maxsize);
2527                                 mblk_size = MIN(snfv->snfv_vml[i].vs_len -
2528                                     mapoff, total_size);
2529                                 nmp = esballoca(
2530                                     (uchar_t *)snfv->snfv_vml[i].vs_addr +
2531                                     mapoff, mblk_size, BPRI_HI,
2532                                     &snfv->snfv_frtn);
2533 
2534                                 /*
2535                                  * We return EAGAIN after unmapping the pages
2536                                  * if we cannot allocate the the head of the
2537                                  * chain. Otherwise, we continue sending the
2538                                  * mblks constructed so far.
2539                                  */
2540                                 if (nmp == NULL) {
2541                                         if (i == 0) {
2542                                                 vpm_unmap_pages(snfv->snfv_vml,
2543                                                     S_READ);
2544                                                 kmem_free(snfv,
2545                                                     sizeof (snf_vmap_desbinfo));
2546                                                 error = EAGAIN;
2547                                                 goto out;
2548                                         }
2549                                         break;
2550                                 }
2551                                 /* Mark this dblk with the zero-copy flag */
2552                                 nmp->b_datap->db_struioflag |= STRUIO_ZC;
2553                                 nmp->b_wptr += mblk_size;
2554                                 chain_size += mblk_size;
2555                                 fileoff += mblk_size;
2556                                 total_size -= mblk_size;
2557                                 snfv->snfv_ref++;
2558                                 mapoff = 0;
2559                                 if (i > 0)
2560                                         linkb(mp, nmp);
2561                                 else
2562                                         mp = nmp;
2563                         }
2564                         VN_HOLD(fvp);
2565                         snfv->snfv_vp = fvp;
2566                 } else {
2567                         /* vpm not supported. fallback to segmap */
2568                         snf_smap_desbinfo *snfi;
2569 
2570                         mapoff = fileoff & MAXBOFFSET;
2571                         chain_size = MAXBSIZE - mapoff;
2572                         if (chain_size > total_size)
2573                                 chain_size = total_size;
2574                         /*
2575                          * we don't forcefault because we'll call
2576                          * segmap_fault(F_SOFTLOCK) next.
2577                          *
2578                          * S_READ will get the ref bit set (by either
2579                          * segmap_getmapflt() or segmap_fault()) and page
2580                          * shared locked.
2581                          */
2582                         base = segmap_getmapflt(segkmap, fvp, fileoff,
2583                             chain_size, segmap_kpm ? SM_FAULT : 0, S_READ);
2584 
2585                         snfi = kmem_alloc(sizeof (*snfi), KM_SLEEP);
2586                         snfi->snfi_len = (size_t)roundup(mapoff+chain_size,
2587                             PAGESIZE)- (mapoff & PAGEMASK);
2588                         /*
2589                          * We must call segmap_fault() even for segmap_kpm
2590                          * because that's how error gets returned.
2591                          * (segmap_getmapflt() never fails but segmap_fault()
2592                          * does.)
2593                          *
2594                          * If the pages aren't available yet, we get
2595                          * DEADLK, so wait and try again a little later using
2596                          * an increasing wait. We might be here a long time.
2597                          *
2598                          * If delay_sig returns EINTR, be sure to exit and
2599                          * pass it up to the caller.
2600                          */
2601                         deadlk_wait = 0;
2602                         while ((error = FC_ERRNO(segmap_fault(kas.a_hat,
2603                             segkmap, (caddr_t)(uintptr_t)(((uintptr_t)base +
2604                             mapoff) & PAGEMASK), snfi->snfi_len, F_SOFTLOCK,
2605                             S_READ))) == EDEADLK) {
2606                                 deadlk_wait += (deadlk_wait < 5) ? 1 : 4;
2607                                 if ((error = delay_sig(deadlk_wait)) != 0) {
2608                                         break;
2609                                 }
2610                         }
2611                         if (error != 0) {
2612                                 (void) segmap_release(segkmap, base, 0);
2613                                 kmem_free(snfi, sizeof (*snfi));
2614                                 error = (error == EINTR) ? EINTR : EIO;
2615                                 goto out;
2616                         }
2617                         snfi->snfi_frtn.free_func = snf_smap_desbfree;
2618                         snfi->snfi_frtn.free_arg = (caddr_t)snfi;
2619                         snfi->snfi_base = base;
2620                         snfi->snfi_mapoff = mapoff;
2621                         mp = esballoca((uchar_t *)base + mapoff, chain_size,
2622                             BPRI_HI, &snfi->snfi_frtn);
2623 
2624                         if (mp == NULL) {
2625                                 (void) segmap_fault(kas.a_hat, segkmap,
2626                                     (caddr_t)(uintptr_t)(((uintptr_t)base +
2627                                     mapoff) & PAGEMASK), snfi->snfi_len,
2628                                     F_SOFTUNLOCK, S_OTHER);
2629                                 (void) segmap_release(segkmap, base, 0);
2630                                 kmem_free(snfi, sizeof (*snfi));
2631                                 freemsg(mp);
2632                                 error = EAGAIN;
2633                                 goto out;
2634                         }
2635                         VN_HOLD(fvp);
2636                         snfi->snfi_vp = fvp;
2637                         mp->b_wptr += chain_size;
2638 
2639                         /* Mark this dblk with the zero-copy flag */
2640                         mp->b_datap->db_struioflag |= STRUIO_ZC;
2641                         fileoff += chain_size;
2642                         total_size -= chain_size;
2643                 }
2644 
2645                 if (total_size == 0 && !nowait) {
2646                         ASSERT(!dowait);
2647                         dowait = B_TRUE;
2648                         mp->b_datap->db_struioflag |= STRUIO_ZCNOTIFY;
2649                 }
2650                 VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2651                 error = socket_sendmblk(VTOSO(vp), &msg, fflag, CRED(), &mp);
2652                 if (error != 0) {
2653                         /*
2654                          * mp contains the mblks that were not sent by
2655                          * socket_sendmblk. Use its size to update *count
2656                          */
2657                         *count = ksize + (chain_size - msgdsize(mp));
2658                         if (mp != NULL)
2659                                 freemsg(mp);
2660                         return (error);
2661                 }
2662                 ksize += chain_size;
2663                 if (total_size == 0)
2664                         goto done;
2665 
2666                 (void) VOP_RWLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2667                 va.va_mask = AT_SIZE;
2668                 error = VOP_GETATTR(fvp, &va, 0, kcred, NULL);
2669                 if (error)
2670                         break;
2671                 /* Read as much as possible. */
2672                 if (fileoff >= va.va_size)
2673                         break;
2674                 if (total_size + fileoff > va.va_size)
2675                         total_size = va.va_size - fileoff;
2676         }
2677 out:
2678         VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2679 done:
2680         *count = ksize;
2681         if (dowait) {
2682                 stdata_t *stp;
2683 
2684                 stp = vp->v_stream;
2685                 if (stp == NULL) {
2686                         struct sonode *so;
2687                         so = VTOSO(vp);
2688                         error = so_zcopy_wait(so);
2689                 } else {
2690                         mutex_enter(&stp->sd_lock);
2691                         while (!(stp->sd_flag & STZCNOTIFY)) {
2692                                 if (cv_wait_sig(&stp->sd_zcopy_wait,
2693                                     &stp->sd_lock) == 0) {
2694                                         error = EINTR;
2695                                         break;
2696                                 }
2697                         }
2698                         stp->sd_flag &= ~STZCNOTIFY;
2699                         mutex_exit(&stp->sd_lock);
2700                 }
2701         }
2702         return (error);
2703 }
2704 
2705 int
2706 snf_cache(file_t *fp, vnode_t *fvp, u_offset_t fileoff, u_offset_t size,
2707     uint_t maxpsz, ssize_t *count)
2708 {
2709         struct vnode *vp;
2710         mblk_t *mp;
2711         int iosize;
2712         int extra = 0;
2713         int error;
2714         short fflag;
2715         int ksize;
2716         int ioflag;
2717         struct uio auio;
2718         struct iovec aiov;
2719         struct vattr va;
2720         int maxblk = 0;
2721         int wroff = 0;
2722         struct sonode *so;
2723         struct nmsghdr msg;
2724 
2725         vp = fp->f_vnode;
2726         if (vp->v_type == VSOCK) {
2727                 stdata_t *stp;
2728 
2729                 /*
2730                  * Get the extra space to insert a header and a trailer.
2731                  */
2732                 so = VTOSO(vp);
2733                 stp = vp->v_stream;
2734                 if (stp == NULL) {
2735                         wroff = so->so_proto_props.sopp_wroff;
2736                         maxblk = so->so_proto_props.sopp_maxblk;
2737                         extra = wroff + so->so_proto_props.sopp_tail;
2738                 } else {
2739                         wroff = (int)(stp->sd_wroff);
2740                         maxblk = (int)(stp->sd_maxblk);
2741                         extra = wroff + (int)(stp->sd_tail);
2742                 }
2743         }
2744         bzero(&msg, sizeof (msg));
2745         fflag = fp->f_flag;
2746         ksize = 0;
2747         auio.uio_iov = &aiov;
2748         auio.uio_iovcnt = 1;
2749         auio.uio_segflg = UIO_SYSSPACE;
2750         auio.uio_llimit = MAXOFFSET_T;
2751         auio.uio_fmode = fflag;
2752         auio.uio_extflg = UIO_COPY_CACHED;
2753         ioflag = auio.uio_fmode & (FSYNC|FDSYNC|FRSYNC);
2754         /* If read sync is not asked for, filter sync flags */
2755         if ((ioflag & FRSYNC) == 0)
2756                 ioflag &= ~(FSYNC|FDSYNC);
2757         for (;;) {
2758                 if (ISSIG(curthread, JUSTLOOKING)) {
2759                         error = EINTR;
2760                         break;
2761                 }
2762                 iosize = (int)MIN(maxpsz, size);
2763 
2764                 /*
2765                  * Socket filters can limit the mblk size,
2766                  * so limit reads to maxblk if there are
2767                  * filters present.
2768                  */
2769                 if (vp->v_type == VSOCK &&
2770                     so->so_filter_active > 0 && maxblk != INFPSZ)
2771                         iosize = (int)MIN(iosize, maxblk);
2772 
2773                 if (is_system_labeled()) {
2774                         mp = allocb_cred(iosize + extra, CRED(),
2775                             curproc->p_pid);
2776                 } else {
2777                         mp = allocb(iosize + extra, BPRI_MED);
2778                 }
2779                 if (mp == NULL) {
2780                         error = EAGAIN;
2781                         break;
2782                 }
2783 
2784                 mp->b_rptr += wroff;
2785 
2786                 aiov.iov_base = (caddr_t)mp->b_rptr;
2787                 aiov.iov_len = iosize;
2788                 auio.uio_loffset = fileoff;
2789                 auio.uio_resid = iosize;
2790 
2791                 error = VOP_READ(fvp, &auio, ioflag, fp->f_cred, NULL);
2792                 iosize -= auio.uio_resid;
2793 
2794                 if (error == EINTR && iosize != 0)
2795                         error = 0;
2796 
2797                 if (error != 0 || iosize == 0) {
2798                         freeb(mp);
2799                         break;
2800                 }
2801                 mp->b_wptr = mp->b_rptr + iosize;
2802 
2803                 VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2804 
2805                 error = socket_sendmblk(VTOSO(vp), &msg, fflag, CRED(), &mp);
2806 
2807                 if (error != 0) {
2808                         *count = ksize;
2809                         if (mp != NULL)
2810                                 freeb(mp);
2811                         return (error);
2812                 }
2813                 ksize += iosize;
2814                 size -= iosize;
2815                 if (size == 0)
2816                         goto done;
2817 
2818                 fileoff += iosize;
2819                 (void) VOP_RWLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2820                 va.va_mask = AT_SIZE;
2821                 error = VOP_GETATTR(fvp, &va, 0, kcred, NULL);
2822                 if (error)
2823                         break;
2824                 /* Read as much as possible. */
2825                 if (fileoff >= va.va_size)
2826                         size = 0;
2827                 else if (size + fileoff > va.va_size)
2828                         size = va.va_size - fileoff;
2829         }
2830         VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2831 done:
2832         *count = ksize;
2833         return (error);
2834 }
2835 
2836 #if defined(_SYSCALL32_IMPL) || defined(_ILP32)
2837 /*
2838  * Largefile support for 32 bit applications only.
2839  */
2840 int
2841 sosendfile64(file_t *fp, file_t *rfp, const struct ksendfilevec64 *sfv,
2842     ssize32_t *count32)
2843 {
2844         ssize32_t sfv_len;
2845         u_offset_t sfv_off, va_size;
2846         struct vnode *vp, *fvp, *realvp;
2847         struct vattr va;
2848         stdata_t *stp;
2849         ssize_t count = 0;
2850         int error = 0;
2851         boolean_t dozcopy = B_FALSE;
2852         uint_t maxpsz;
2853 
2854         sfv_len = (ssize32_t)sfv->sfv_len;
2855         if (sfv_len < 0) {
2856                 error = EINVAL;
2857                 goto out;
2858         }
2859 
2860         if (sfv_len == 0) goto out;
2861 
2862         sfv_off = (u_offset_t)sfv->sfv_off;
2863 
2864         /* Same checks as in pread */
2865         if (sfv_off > MAXOFFSET_T) {
2866                 error = EINVAL;
2867                 goto out;
2868         }
2869         if (sfv_off + sfv_len > MAXOFFSET_T)
2870                 sfv_len = (ssize32_t)(MAXOFFSET_T - sfv_off);
2871 
2872         /*
2873          * There are no more checks on sfv_len. So, we cast it to
2874          * u_offset_t and share the snf_direct_io/snf_cache code between
2875          * 32 bit and 64 bit.
2876          *
2877          * TODO: should do nbl_need_check() like read()?
2878          */
2879         if (sfv_len > sendfile_max_size) {
2880                 sf_stats.ss_file_not_cached++;
2881                 error = snf_direct_io(fp, rfp, sfv_off, (u_offset_t)sfv_len,
2882                     &count);
2883                 goto out;
2884         }
2885         fvp = rfp->f_vnode;
2886         if (VOP_REALVP(fvp, &realvp, NULL) == 0)
2887                 fvp = realvp;
2888         /*
2889          * Grab the lock as a reader to prevent the file size
2890          * from changing underneath.
2891          */
2892         (void) VOP_RWLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2893         va.va_mask = AT_SIZE;
2894         error = VOP_GETATTR(fvp, &va, 0, kcred, NULL);
2895         va_size = va.va_size;
2896         if ((error != 0) || (va_size == 0) || (sfv_off >= va_size)) {
2897                 VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2898                 goto out;
2899         }
2900         /* Read as much as possible. */
2901         if (sfv_off + sfv_len > va_size)
2902                 sfv_len = va_size - sfv_off;
2903 
2904         vp = fp->f_vnode;
2905         stp = vp->v_stream;
2906         /*
2907          * When the NOWAIT flag is not set, we enable zero-copy only if the
2908          * transfer size is large enough. This prevents performance loss
2909          * when the caller sends the file piece by piece.
2910          */
2911         if (sfv_len >= MAXBSIZE && (sfv_len >= (va_size >> 1) ||
2912             (sfv->sfv_flag & SFV_NOWAIT) || sfv_len >= 0x1000000) &&
2913             !vn_has_flocks(fvp) && !(fvp->v_flag & VNOMAP)) {
2914                 uint_t copyflag;
2915                 copyflag = stp != NULL ? stp->sd_copyflag :
2916                     VTOSO(vp)->so_proto_props.sopp_zcopyflag;
2917                 if ((copyflag & (STZCVMSAFE|STZCVMUNSAFE)) == 0) {
2918                         int on = 1;
2919 
2920                         if (socket_setsockopt(VTOSO(vp), SOL_SOCKET,
2921                             SO_SND_COPYAVOID, &on, sizeof (on), CRED()) == 0)
2922                                 dozcopy = B_TRUE;
2923                 } else {
2924                         dozcopy = copyflag & STZCVMSAFE;
2925                 }
2926         }
2927         if (dozcopy) {
2928                 sf_stats.ss_file_segmap++;
2929                 error = snf_segmap(fp, fvp, sfv_off, (u_offset_t)sfv_len,
2930                     &count, ((sfv->sfv_flag & SFV_NOWAIT) != 0));
2931         } else {
2932                 if (vp->v_type == VSOCK && stp == NULL) {
2933                         sonode_t *so = VTOSO(vp);
2934                         maxpsz = so->so_proto_props.sopp_maxpsz;
2935                 } else if (stp != NULL) {
2936                         maxpsz = stp->sd_qn_maxpsz;
2937                 } else {
2938                         maxpsz = maxphys;
2939                 }
2940 
2941                 if (maxpsz == INFPSZ)
2942                         maxpsz = maxphys;
2943                 else
2944                         maxpsz = roundup(maxpsz, MAXBSIZE);
2945                 sf_stats.ss_file_cached++;
2946                 error = snf_cache(fp, fvp, sfv_off, (u_offset_t)sfv_len,
2947                     maxpsz, &count);
2948         }
2949 out:
2950         releasef(sfv->sfv_fd);
2951         *count32 = (ssize32_t)count;
2952         return (error);
2953 }
2954 #endif
2955 
2956 #ifdef _SYSCALL32_IMPL
2957 /*
2958  * recv32(), recvfrom32(), send32(), sendto32(): intentionally return a
2959  * ssize_t rather than ssize32_t; see the comments above read32 for details.
2960  */
2961 
2962 ssize_t
2963 recv32(int32_t sock, caddr32_t buffer, size32_t len, int32_t flags)
2964 {
2965         return (recv(sock, (void *)(uintptr_t)buffer, (ssize32_t)len, flags));
2966 }
2967 
2968 ssize_t
2969 recvfrom32(int32_t sock, caddr32_t buffer, size32_t len, int32_t flags,
2970         caddr32_t name, caddr32_t namelenp)
2971 {
2972         return (recvfrom(sock, (void *)(uintptr_t)buffer, (ssize32_t)len, flags,
2973             (void *)(uintptr_t)name, (void *)(uintptr_t)namelenp));
2974 }
2975 
2976 ssize_t
2977 send32(int32_t sock, caddr32_t buffer, size32_t len, int32_t flags)
2978 {
2979         return (send(sock, (void *)(uintptr_t)buffer, (ssize32_t)len, flags));
2980 }
2981 
2982 ssize_t
2983 sendto32(int32_t sock, caddr32_t buffer, size32_t len, int32_t flags,
2984         caddr32_t name, socklen_t namelen)
2985 {
2986         return (sendto(sock, (void *)(uintptr_t)buffer, (ssize32_t)len, flags,
2987             (void *)(uintptr_t)name, namelen));
2988 }
2989 #endif  /* _SYSCALL32_IMPL */
2990 
2991 /*
2992  * Function wrappers (mostly around the sonode switch) for
2993  * backward compatibility.
2994  */
2995 
2996 int
2997 soaccept(struct sonode *so, int fflag, struct sonode **nsop)
2998 {
2999         return (socket_accept(so, fflag, CRED(), nsop));
3000 }
3001 
3002 int
3003 sobind(struct sonode *so, struct sockaddr *name, socklen_t namelen,
3004     int backlog, int flags)
3005 {
3006         int     error;
3007 
3008         error = socket_bind(so, name, namelen, flags, CRED());
3009         if (error == 0 && backlog != 0)
3010                 return (socket_listen(so, backlog, CRED()));
3011 
3012         return (error);
3013 }
3014 
3015 int
3016 solisten(struct sonode *so, int backlog)
3017 {
3018         return (socket_listen(so, backlog, CRED()));
3019 }
3020 
3021 int
3022 soconnect(struct sonode *so, struct sockaddr *name, socklen_t namelen,
3023     int fflag, int flags)
3024 {
3025         return (socket_connect(so, name, namelen, fflag, flags, CRED()));
3026 }
3027 
3028 int
3029 sorecvmsg(struct sonode *so, struct nmsghdr *msg, struct uio *uiop)
3030 {
3031         return (socket_recvmsg(so, msg, uiop, CRED()));
3032 }
3033 
3034 int
3035 sosendmsg(struct sonode *so, struct nmsghdr *msg, struct uio *uiop)
3036 {
3037         return (socket_sendmsg(so, msg, uiop, CRED()));
3038 }
3039 
3040 int
3041 soshutdown(struct sonode *so, int how)
3042 {
3043         return (socket_shutdown(so, how, CRED()));
3044 }
3045 
3046 int
3047 sogetsockopt(struct sonode *so, int level, int option_name, void *optval,
3048     socklen_t *optlenp, int flags)
3049 {
3050         return (socket_getsockopt(so, level, option_name, optval, optlenp,
3051             flags, CRED()));
3052 }
3053 
3054 int
3055 sosetsockopt(struct sonode *so, int level, int option_name, const void *optval,
3056     t_uscalar_t optlen)
3057 {
3058         return (socket_setsockopt(so, level, option_name, optval, optlen,
3059             CRED()));
3060 }
3061 
3062 /*
3063  * Because this is backward compatibility interface it only needs to be
3064  * able to handle the creation of TPI sockfs sockets.
3065  */
3066 struct sonode *
3067 socreate(struct sockparams *sp, int family, int type, int protocol, int version,
3068     int *errorp)
3069 {
3070         struct sonode *so;
3071 
3072         ASSERT(sp != NULL);
3073 
3074         so = sp->sp_smod_info->smod_sock_create_func(sp, family, type, protocol,
3075             version, SOCKET_SLEEP, errorp, CRED());
3076         if (so == NULL) {
3077                 SOCKPARAMS_DEC_REF(sp);
3078         } else {
3079                 if ((*errorp = SOP_INIT(so, NULL, CRED(), SOCKET_SLEEP)) == 0) {
3080                         /* Cannot fail, only bumps so_count */
3081                         (void) VOP_OPEN(&SOTOV(so), FREAD|FWRITE, CRED(), NULL);
3082                 } else {
3083                         socket_destroy(so);
3084                         so = NULL;
3085                 }
3086         }
3087         return (so);
3088 }