1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  23  * Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
  24  */
  25 
  26 #include <sys/systm.h>
  27 #include <sys/kmem.h>
  28 #include <sys/cmn_err.h>
  29 #include <sys/atomic.h>
  30 #include <sys/clconf.h>
  31 #include <sys/cladm.h>
  32 #include <sys/flock.h>
  33 #include <nfs/export.h>
  34 #include <nfs/nfs.h>
  35 #include <nfs/nfs4.h>
  36 #include <nfs/nfssys.h>
  37 #include <nfs/lm.h>
  38 #include <sys/pathname.h>
  39 #include <sys/sdt.h>
  40 #include <sys/nvpair.h>
  41 
  42 extern u_longlong_t nfs4_srv_caller_id;
  43 
  44 extern time_t rfs4_start_time;
  45 extern uint_t nfs4_srv_vkey;
  46 
  47 stateid4 special0 = {
  48         0,
  49         { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
  50 };
  51 
  52 stateid4 special1 = {
  53         0xffffffff,
  54         {
  55                 (char)0xff, (char)0xff, (char)0xff, (char)0xff,
  56                 (char)0xff, (char)0xff, (char)0xff, (char)0xff,
  57                 (char)0xff, (char)0xff, (char)0xff, (char)0xff
  58         }
  59 };
  60 
  61 
  62 #define ISSPECIAL(id)  (stateid4_cmp(id, &special0) || \
  63                         stateid4_cmp(id, &special1))
  64 
  65 /* For embedding the cluster nodeid into our clientid */
  66 #define CLUSTER_NODEID_SHIFT    24
  67 #define CLUSTER_MAX_NODEID      255
  68 
  69 #ifdef DEBUG
  70 int rfs4_debug;
  71 #endif
  72 
  73 static uint32_t rfs4_database_debug = 0x00;
  74 
  75 static void rfs4_ss_clid_write(rfs4_client_t *cp, char *leaf);
  76 static void rfs4_ss_clid_write_one(rfs4_client_t *cp, char *dir, char *leaf);
  77 static void rfs4_dss_clear_oldstate(rfs4_servinst_t *sip);
  78 static void rfs4_ss_chkclid_sip(rfs4_client_t *cp, rfs4_servinst_t *sip);
  79 
  80 /*
  81  * Couple of simple init/destroy functions for a general waiter
  82  */
  83 void
  84 rfs4_sw_init(rfs4_state_wait_t *swp)
  85 {
  86         mutex_init(swp->sw_cv_lock, NULL, MUTEX_DEFAULT, NULL);
  87         cv_init(swp->sw_cv, NULL, CV_DEFAULT, NULL);
  88         swp->sw_active = FALSE;
  89         swp->sw_wait_count = 0;
  90 }
  91 
  92 void
  93 rfs4_sw_destroy(rfs4_state_wait_t *swp)
  94 {
  95         mutex_destroy(swp->sw_cv_lock);
  96         cv_destroy(swp->sw_cv);
  97 }
  98 
  99 void
 100 rfs4_sw_enter(rfs4_state_wait_t *swp)
 101 {
 102         mutex_enter(swp->sw_cv_lock);
 103         while (swp->sw_active) {
 104                 swp->sw_wait_count++;
 105                 cv_wait(swp->sw_cv, swp->sw_cv_lock);
 106                 swp->sw_wait_count--;
 107         }
 108         ASSERT(swp->sw_active == FALSE);
 109         swp->sw_active = TRUE;
 110         mutex_exit(swp->sw_cv_lock);
 111 }
 112 
 113 void
 114 rfs4_sw_exit(rfs4_state_wait_t *swp)
 115 {
 116         mutex_enter(swp->sw_cv_lock);
 117         ASSERT(swp->sw_active == TRUE);
 118         swp->sw_active = FALSE;
 119         if (swp->sw_wait_count != 0)
 120                 cv_broadcast(swp->sw_cv);
 121         mutex_exit(swp->sw_cv_lock);
 122 }
 123 
 124 /*
 125  * CPR callback id -- not related to v4 callbacks
 126  */
 127 static callb_id_t cpr_id = 0;
 128 
 129 static void
 130 deep_lock_copy(LOCK4res *dres, LOCK4res *sres)
 131 {
 132         lock_owner4 *slo = &sres->LOCK4res_u.denied.owner;
 133         lock_owner4 *dlo = &dres->LOCK4res_u.denied.owner;
 134 
 135         if (sres->status == NFS4ERR_DENIED) {
 136                 dlo->owner_val = kmem_alloc(slo->owner_len, KM_SLEEP);
 137                 bcopy(slo->owner_val, dlo->owner_val, slo->owner_len);
 138         }
 139 }
 140 
 141 static void
 142 deep_lock_free(LOCK4res *res)
 143 {
 144         lock_owner4 *lo = &res->LOCK4res_u.denied.owner;
 145 
 146         if (res->status == NFS4ERR_DENIED)
 147                 kmem_free(lo->owner_val, lo->owner_len);
 148 }
 149 
 150 static void
 151 deep_open_copy(OPEN4res *dres, OPEN4res *sres)
 152 {
 153         nfsace4 *sacep, *dacep;
 154 
 155         if (sres->status != NFS4_OK) {
 156                 return;
 157         }
 158 
 159         dres->attrset = sres->attrset;
 160 
 161         switch (sres->delegation.delegation_type) {
 162         case OPEN_DELEGATE_NONE:
 163                 return;
 164         case OPEN_DELEGATE_READ:
 165                 sacep = &sres->delegation.open_delegation4_u.read.permissions;
 166                 dacep = &dres->delegation.open_delegation4_u.read.permissions;
 167                 break;
 168         case OPEN_DELEGATE_WRITE:
 169                 sacep = &sres->delegation.open_delegation4_u.write.permissions;
 170                 dacep = &dres->delegation.open_delegation4_u.write.permissions;
 171                 break;
 172         }
 173         dacep->who.utf8string_val =
 174             kmem_alloc(sacep->who.utf8string_len, KM_SLEEP);
 175         bcopy(sacep->who.utf8string_val, dacep->who.utf8string_val,
 176             sacep->who.utf8string_len);
 177 }
 178 
 179 static void
 180 deep_open_free(OPEN4res *res)
 181 {
 182         nfsace4 *acep;
 183         if (res->status != NFS4_OK)
 184                 return;
 185 
 186         switch (res->delegation.delegation_type) {
 187         case OPEN_DELEGATE_NONE:
 188                 return;
 189         case OPEN_DELEGATE_READ:
 190                 acep = &res->delegation.open_delegation4_u.read.permissions;
 191                 break;
 192         case OPEN_DELEGATE_WRITE:
 193                 acep = &res->delegation.open_delegation4_u.write.permissions;
 194                 break;
 195         }
 196 
 197         if (acep->who.utf8string_val) {
 198                 kmem_free(acep->who.utf8string_val, acep->who.utf8string_len);
 199                 acep->who.utf8string_val = NULL;
 200         }
 201 }
 202 
 203 void
 204 rfs4_free_reply(nfs_resop4 *rp)
 205 {
 206         switch (rp->resop) {
 207         case OP_LOCK:
 208                 deep_lock_free(&rp->nfs_resop4_u.oplock);
 209                 break;
 210         case OP_OPEN:
 211                 deep_open_free(&rp->nfs_resop4_u.opopen);
 212         default:
 213                 break;
 214         }
 215 }
 216 
 217 void
 218 rfs4_copy_reply(nfs_resop4 *dst, nfs_resop4 *src)
 219 {
 220         *dst = *src;
 221 
 222         /* Handle responses that need deep copy */
 223         switch (src->resop) {
 224         case OP_LOCK:
 225                 deep_lock_copy(&dst->nfs_resop4_u.oplock,
 226                     &src->nfs_resop4_u.oplock);
 227                 break;
 228         case OP_OPEN:
 229                 deep_open_copy(&dst->nfs_resop4_u.opopen,
 230                     &src->nfs_resop4_u.opopen);
 231                 break;
 232         default:
 233                 break;
 234         };
 235 }
 236 
 237 /*
 238  * This is the implementation of the underlying state engine. The
 239  * public interface to this engine is described by
 240  * nfs4_state.h. Callers to the engine should hold no state engine
 241  * locks when they call in to it. If the protocol needs to lock data
 242  * structures it should do so after acquiring all references to them
 243  * first and then follow the following lock order:
 244  *
 245  *      client > openowner > state > lo_state > lockowner > file.
 246  *
 247  * Internally we only allow a thread to hold one hash bucket lock at a
 248  * time and the lock is higher in the lock order (must be acquired
 249  * first) than the data structure that is on that hash list.
 250  *
 251  * If a new reference was acquired by the caller, that reference needs
 252  * to be released after releasing all acquired locks with the
 253  * corresponding rfs4_*_rele routine.
 254  */
 255 
 256 /*
 257  * This code is some what prototypical for now. Its purpose currently is to
 258  * implement the interfaces sufficiently to finish the higher protocol
 259  * elements. This will be replaced by a dynamically resizeable tables
 260  * backed by kmem_cache allocator. However synchronization is handled
 261  * correctly (I hope) and will not change by much.  The mutexes for
 262  * the hash buckets that can be used to create new instances of data
 263  * structures  might be good candidates to evolve into reader writer
 264  * locks. If it has to do a creation, it would be holding the
 265  * mutex across a kmem_alloc with KM_SLEEP specified.
 266  */
 267 
 268 #ifdef DEBUG
 269 #define TABSIZE 17
 270 #else
 271 #define TABSIZE 2047
 272 #endif
 273 
 274 #define ADDRHASH(key) ((unsigned long)(key) >> 3)
 275 
 276 /* Used to serialize create/destroy of rfs4_server_state database */
 277 kmutex_t        rfs4_state_lock;
 278 static rfs4_database_t *rfs4_server_state = NULL;
 279 
 280 /* Used to serialize lookups of clientids */
 281 static  krwlock_t       rfs4_findclient_lock;
 282 
 283 /*
 284  * For now this "table" is exposed so that the CPR callback
 285  * function can tromp through it..
 286  */
 287 rfs4_table_t *rfs4_client_tab;
 288 
 289 static rfs4_index_t *rfs4_clientid_idx;
 290 static rfs4_index_t *rfs4_nfsclnt_idx;
 291 static rfs4_table_t *rfs4_clntip_tab;
 292 static rfs4_index_t *rfs4_clntip_idx;
 293 static rfs4_table_t *rfs4_openowner_tab;
 294 static rfs4_index_t *rfs4_openowner_idx;
 295 static rfs4_table_t *rfs4_state_tab;
 296 static rfs4_index_t *rfs4_state_idx;
 297 static rfs4_index_t *rfs4_state_owner_file_idx;
 298 static rfs4_index_t *rfs4_state_file_idx;
 299 static rfs4_table_t *rfs4_lo_state_tab;
 300 static rfs4_index_t *rfs4_lo_state_idx;
 301 static rfs4_index_t *rfs4_lo_state_owner_idx;
 302 static rfs4_table_t *rfs4_lockowner_tab;
 303 static rfs4_index_t *rfs4_lockowner_idx;
 304 static rfs4_index_t *rfs4_lockowner_pid_idx;
 305 static rfs4_table_t *rfs4_file_tab;
 306 static rfs4_index_t *rfs4_file_idx;
 307 static rfs4_table_t *rfs4_deleg_state_tab;
 308 static rfs4_index_t *rfs4_deleg_idx;
 309 static rfs4_index_t *rfs4_deleg_state_idx;
 310 
 311 #define MAXTABSZ 1024*1024
 312 
 313 /* The values below are rfs4_lease_time units */
 314 
 315 #ifdef DEBUG
 316 #define CLIENT_CACHE_TIME 1
 317 #define OPENOWNER_CACHE_TIME 1
 318 #define STATE_CACHE_TIME 1
 319 #define LO_STATE_CACHE_TIME 1
 320 #define LOCKOWNER_CACHE_TIME 1
 321 #define FILE_CACHE_TIME 3
 322 #define DELEG_STATE_CACHE_TIME 1
 323 #else
 324 #define CLIENT_CACHE_TIME 10
 325 #define OPENOWNER_CACHE_TIME 5
 326 #define STATE_CACHE_TIME 1
 327 #define LO_STATE_CACHE_TIME 1
 328 #define LOCKOWNER_CACHE_TIME 3
 329 #define FILE_CACHE_TIME 40
 330 #define DELEG_STATE_CACHE_TIME 1
 331 #endif
 332 
 333 
 334 static time_t rfs4_client_cache_time = 0;
 335 static time_t rfs4_clntip_cache_time = 0;
 336 static time_t rfs4_openowner_cache_time = 0;
 337 static time_t rfs4_state_cache_time = 0;
 338 static time_t rfs4_lo_state_cache_time = 0;
 339 static time_t rfs4_lockowner_cache_time = 0;
 340 static time_t rfs4_file_cache_time = 0;
 341 static time_t rfs4_deleg_state_cache_time = 0;
 342 
 343 static bool_t rfs4_client_create(rfs4_entry_t, void *);
 344 static void rfs4_dss_remove_cpleaf(rfs4_client_t *);
 345 static void rfs4_dss_remove_leaf(rfs4_servinst_t *, char *, char *);
 346 static void rfs4_client_destroy(rfs4_entry_t);
 347 static bool_t rfs4_client_expiry(rfs4_entry_t);
 348 static uint32_t clientid_hash(void *);
 349 static bool_t clientid_compare(rfs4_entry_t, void *);
 350 static void *clientid_mkkey(rfs4_entry_t);
 351 static uint32_t nfsclnt_hash(void *);
 352 static bool_t nfsclnt_compare(rfs4_entry_t, void *);
 353 static void *nfsclnt_mkkey(rfs4_entry_t);
 354 static bool_t rfs4_clntip_expiry(rfs4_entry_t);
 355 static void rfs4_clntip_destroy(rfs4_entry_t);
 356 static bool_t rfs4_clntip_create(rfs4_entry_t, void *);
 357 static uint32_t clntip_hash(void *);
 358 static bool_t clntip_compare(rfs4_entry_t, void *);
 359 static void *clntip_mkkey(rfs4_entry_t);
 360 static bool_t rfs4_openowner_create(rfs4_entry_t, void *);
 361 static void rfs4_openowner_destroy(rfs4_entry_t);
 362 static bool_t rfs4_openowner_expiry(rfs4_entry_t);
 363 static uint32_t openowner_hash(void *);
 364 static bool_t openowner_compare(rfs4_entry_t, void *);
 365 static void *openowner_mkkey(rfs4_entry_t);
 366 static bool_t rfs4_state_create(rfs4_entry_t, void *);
 367 static void rfs4_state_destroy(rfs4_entry_t);
 368 static bool_t rfs4_state_expiry(rfs4_entry_t);
 369 static uint32_t state_hash(void *);
 370 static bool_t state_compare(rfs4_entry_t, void *);
 371 static void *state_mkkey(rfs4_entry_t);
 372 static uint32_t state_owner_file_hash(void *);
 373 static bool_t state_owner_file_compare(rfs4_entry_t, void *);
 374 static void *state_owner_file_mkkey(rfs4_entry_t);
 375 static uint32_t state_file_hash(void *);
 376 static bool_t state_file_compare(rfs4_entry_t, void *);
 377 static void *state_file_mkkey(rfs4_entry_t);
 378 static bool_t rfs4_lo_state_create(rfs4_entry_t, void *);
 379 static void rfs4_lo_state_destroy(rfs4_entry_t);
 380 static bool_t rfs4_lo_state_expiry(rfs4_entry_t);
 381 static uint32_t lo_state_hash(void *);
 382 static bool_t lo_state_compare(rfs4_entry_t, void *);
 383 static void *lo_state_mkkey(rfs4_entry_t);
 384 static uint32_t lo_state_lo_hash(void *);
 385 static bool_t lo_state_lo_compare(rfs4_entry_t, void *);
 386 static void *lo_state_lo_mkkey(rfs4_entry_t);
 387 static bool_t rfs4_lockowner_create(rfs4_entry_t, void *);
 388 static void rfs4_lockowner_destroy(rfs4_entry_t);
 389 static bool_t rfs4_lockowner_expiry(rfs4_entry_t);
 390 static uint32_t lockowner_hash(void *);
 391 static bool_t lockowner_compare(rfs4_entry_t, void *);
 392 static void *lockowner_mkkey(rfs4_entry_t);
 393 static uint32_t pid_hash(void *);
 394 static bool_t pid_compare(rfs4_entry_t, void *);
 395 static void *pid_mkkey(rfs4_entry_t);
 396 static bool_t rfs4_file_create(rfs4_entry_t, void *);
 397 static void rfs4_file_destroy(rfs4_entry_t);
 398 static uint32_t file_hash(void *);
 399 static bool_t file_compare(rfs4_entry_t, void *);
 400 static void *file_mkkey(rfs4_entry_t);
 401 static bool_t rfs4_deleg_state_create(rfs4_entry_t, void *);
 402 static void rfs4_deleg_state_destroy(rfs4_entry_t);
 403 static bool_t rfs4_deleg_state_expiry(rfs4_entry_t);
 404 static uint32_t deleg_hash(void *);
 405 static bool_t deleg_compare(rfs4_entry_t, void *);
 406 static void *deleg_mkkey(rfs4_entry_t);
 407 static uint32_t deleg_state_hash(void *);
 408 static bool_t deleg_state_compare(rfs4_entry_t, void *);
 409 static void *deleg_state_mkkey(rfs4_entry_t);
 410 
 411 static void rfs4_state_rele_nounlock(rfs4_state_t *);
 412 
 413 static int rfs4_ss_enabled = 0;
 414 
 415 extern void (*rfs4_client_clrst)(struct nfs4clrst_args *);
 416 
 417 void
 418 rfs4_ss_pnfree(rfs4_ss_pn_t *ss_pn)
 419 {
 420         kmem_free(ss_pn, sizeof (rfs4_ss_pn_t));
 421 }
 422 
 423 static rfs4_ss_pn_t *
 424 rfs4_ss_pnalloc(char *dir, char *leaf)
 425 {
 426         rfs4_ss_pn_t *ss_pn;
 427         int     dir_len, leaf_len;
 428 
 429         /*
 430          * validate we have a resonable path
 431          * (account for the '/' and trailing null)
 432          */
 433         if ((dir_len = strlen(dir)) > MAXPATHLEN ||
 434             (leaf_len = strlen(leaf)) > MAXNAMELEN ||
 435             (dir_len + leaf_len + 2) > MAXPATHLEN) {
 436                 return (NULL);
 437         }
 438 
 439         ss_pn = kmem_alloc(sizeof (rfs4_ss_pn_t), KM_SLEEP);
 440 
 441         (void) snprintf(ss_pn->pn, MAXPATHLEN, "%s/%s", dir, leaf);
 442         /* Handy pointer to just the leaf name */
 443         ss_pn->leaf = ss_pn->pn + dir_len + 1;
 444         return (ss_pn);
 445 }
 446 
 447 
 448 /*
 449  * Move the "leaf" filename from "sdir" directory
 450  * to the "ddir" directory. Return the pathname of
 451  * the destination unless the rename fails in which
 452  * case we need to return the source pathname.
 453  */
 454 static rfs4_ss_pn_t *
 455 rfs4_ss_movestate(char *sdir, char *ddir, char *leaf)
 456 {
 457         rfs4_ss_pn_t *src, *dst;
 458 
 459         if ((src = rfs4_ss_pnalloc(sdir, leaf)) == NULL)
 460                 return (NULL);
 461 
 462         if ((dst = rfs4_ss_pnalloc(ddir, leaf)) == NULL) {
 463                 rfs4_ss_pnfree(src);
 464                 return (NULL);
 465         }
 466 
 467         /*
 468          * If the rename fails we shall return the src
 469          * pathname and free the dst. Otherwise we need
 470          * to free the src and return the dst pathanme.
 471          */
 472         if (vn_rename(src->pn, dst->pn, UIO_SYSSPACE)) {
 473                 rfs4_ss_pnfree(dst);
 474                 return (src);
 475         }
 476         rfs4_ss_pnfree(src);
 477         return (dst);
 478 }
 479 
 480 
 481 static rfs4_oldstate_t *
 482 rfs4_ss_getstate(vnode_t *dvp, rfs4_ss_pn_t *ss_pn)
 483 {
 484         struct uio uio;
 485         struct iovec iov[3];
 486 
 487         rfs4_oldstate_t *cl_ss = NULL;
 488         vnode_t *vp;
 489         vattr_t va;
 490         uint_t id_len;
 491         int err, kill_file, file_vers;
 492 
 493         if (ss_pn == NULL)
 494                 return (NULL);
 495 
 496         /*
 497          * open the state file.
 498          */
 499         if (vn_open(ss_pn->pn, UIO_SYSSPACE, FREAD, 0, &vp, 0, 0) != 0) {
 500                 return (NULL);
 501         }
 502 
 503         if (vp->v_type != VREG) {
 504                 (void) VOP_CLOSE(vp, FREAD, 1, (offset_t)0, CRED(), NULL);
 505                 VN_RELE(vp);
 506                 return (NULL);
 507         }
 508 
 509         err = VOP_ACCESS(vp, VREAD, 0, CRED(), NULL);
 510         if (err) {
 511                 /*
 512                  * We don't have read access? better get the heck out.
 513                  */
 514                 (void) VOP_CLOSE(vp, FREAD, 1, (offset_t)0, CRED(), NULL);
 515                 VN_RELE(vp);
 516                 return (NULL);
 517         }
 518 
 519         (void) VOP_RWLOCK(vp, V_WRITELOCK_FALSE, NULL);
 520         /*
 521          * get the file size to do some basic validation
 522          */
 523         va.va_mask = AT_SIZE;
 524         err = VOP_GETATTR(vp, &va, 0, CRED(), NULL);
 525 
 526         kill_file = (va.va_size == 0 || va.va_size <
 527             (NFS4_VERIFIER_SIZE + sizeof (uint_t)+1));
 528 
 529         if (err || kill_file) {
 530                 VOP_RWUNLOCK(vp, V_WRITELOCK_FALSE, NULL);
 531                 (void) VOP_CLOSE(vp, FREAD, 1, (offset_t)0, CRED(), NULL);
 532                 VN_RELE(vp);
 533                 if (kill_file) {
 534                         (void) VOP_REMOVE(dvp, ss_pn->leaf, CRED(), NULL, 0);
 535                 }
 536                 return (NULL);
 537         }
 538 
 539         cl_ss = kmem_alloc(sizeof (rfs4_oldstate_t), KM_SLEEP);
 540 
 541         /*
 542          * build iovecs to read in the file_version, verifier and id_len
 543          */
 544         iov[0].iov_base = (caddr_t)&file_vers;
 545         iov[0].iov_len = sizeof (int);
 546         iov[1].iov_base = (caddr_t)&cl_ss->cl_id4.verifier;
 547         iov[1].iov_len = NFS4_VERIFIER_SIZE;
 548         iov[2].iov_base = (caddr_t)&id_len;
 549         iov[2].iov_len = sizeof (uint_t);
 550 
 551         uio.uio_iov = iov;
 552         uio.uio_iovcnt = 3;
 553         uio.uio_segflg = UIO_SYSSPACE;
 554         uio.uio_loffset = 0;
 555         uio.uio_resid = sizeof (int) + NFS4_VERIFIER_SIZE + sizeof (uint_t);
 556 
 557         if (err = VOP_READ(vp, &uio, FREAD, CRED(), NULL)) {
 558                 VOP_RWUNLOCK(vp, V_WRITELOCK_FALSE, NULL);
 559                 (void) VOP_CLOSE(vp, FREAD, 1, (offset_t)0, CRED(), NULL);
 560                 VN_RELE(vp);
 561                 kmem_free(cl_ss, sizeof (rfs4_oldstate_t));
 562                 return (NULL);
 563         }
 564 
 565         /*
 566          * if the file_version doesn't match or if the
 567          * id_len is zero or the combination of the verifier,
 568          * id_len and id_val is bigger than the file we have
 569          * a problem. If so ditch the file.
 570          */
 571         kill_file = (file_vers != NFS4_SS_VERSION || id_len == 0 ||
 572             (id_len + NFS4_VERIFIER_SIZE + sizeof (uint_t)) > va.va_size);
 573 
 574         if (err || kill_file) {
 575                 VOP_RWUNLOCK(vp, V_WRITELOCK_FALSE, NULL);
 576                 (void) VOP_CLOSE(vp, FREAD, 1, (offset_t)0, CRED(), NULL);
 577                 VN_RELE(vp);
 578                 kmem_free(cl_ss, sizeof (rfs4_oldstate_t));
 579                 if (kill_file) {
 580                         (void) VOP_REMOVE(dvp, ss_pn->leaf, CRED(), NULL, 0);
 581                 }
 582                 return (NULL);
 583         }
 584 
 585         /*
 586          * now get the client id value
 587          */
 588         cl_ss->cl_id4.id_val = kmem_alloc(id_len, KM_SLEEP);
 589         iov[0].iov_base = cl_ss->cl_id4.id_val;
 590         iov[0].iov_len = id_len;
 591 
 592         uio.uio_iov = iov;
 593         uio.uio_iovcnt = 1;
 594         uio.uio_segflg = UIO_SYSSPACE;
 595         uio.uio_resid = cl_ss->cl_id4.id_len = id_len;
 596 
 597         if (err = VOP_READ(vp, &uio, FREAD, CRED(), NULL)) {
 598                 VOP_RWUNLOCK(vp, V_WRITELOCK_FALSE, NULL);
 599                 (void) VOP_CLOSE(vp, FREAD, 1, (offset_t)0, CRED(), NULL);
 600                 VN_RELE(vp);
 601                 kmem_free(cl_ss->cl_id4.id_val, id_len);
 602                 kmem_free(cl_ss, sizeof (rfs4_oldstate_t));
 603                 return (NULL);
 604         }
 605 
 606         VOP_RWUNLOCK(vp, V_WRITELOCK_FALSE, NULL);
 607         (void) VOP_CLOSE(vp, FREAD, 1, (offset_t)0, CRED(), NULL);
 608         VN_RELE(vp);
 609         return (cl_ss);
 610 }
 611 
 612 #ifdef  nextdp
 613 #undef nextdp
 614 #endif
 615 #define nextdp(dp)      ((struct dirent64 *)((char *)(dp) + (dp)->d_reclen))
 616 
 617 /*
 618  * Add entries from statedir to supplied oldstate list.
 619  * Optionally, move all entries from statedir -> destdir.
 620  */
 621 void
 622 rfs4_ss_oldstate(rfs4_oldstate_t *oldstate, char *statedir, char *destdir)
 623 {
 624         rfs4_ss_pn_t *ss_pn;
 625         rfs4_oldstate_t *cl_ss = NULL;
 626         char    *dirt = NULL;
 627         int     err, dir_eof = 0, size = 0;
 628         vnode_t *dvp;
 629         struct iovec iov;
 630         struct uio uio;
 631         struct dirent64 *dep;
 632         offset_t dirchunk_offset = 0;
 633 
 634         /*
 635          * open the state directory
 636          */
 637         if (vn_open(statedir, UIO_SYSSPACE, FREAD, 0, &dvp, 0, 0))
 638                 return;
 639 
 640         if (dvp->v_type != VDIR || VOP_ACCESS(dvp, VREAD, 0, CRED(), NULL))
 641                 goto out;
 642 
 643         dirt = kmem_alloc(RFS4_SS_DIRSIZE, KM_SLEEP);
 644 
 645         /*
 646          * Get and process the directory entries
 647          */
 648         while (!dir_eof) {
 649                 (void) VOP_RWLOCK(dvp, V_WRITELOCK_FALSE, NULL);
 650                 iov.iov_base = dirt;
 651                 iov.iov_len = RFS4_SS_DIRSIZE;
 652                 uio.uio_iov = &iov;
 653                 uio.uio_iovcnt = 1;
 654                 uio.uio_segflg = UIO_SYSSPACE;
 655                 uio.uio_loffset = dirchunk_offset;
 656                 uio.uio_resid = RFS4_SS_DIRSIZE;
 657 
 658                 err = VOP_READDIR(dvp, &uio, CRED(), &dir_eof, NULL, 0);
 659                 VOP_RWUNLOCK(dvp, V_WRITELOCK_FALSE, NULL);
 660                 if (err)
 661                         goto out;
 662 
 663                 size = RFS4_SS_DIRSIZE - uio.uio_resid;
 664 
 665                 /*
 666                  * Process all the directory entries in this
 667                  * readdir chunk
 668                  */
 669                 for (dep = (struct dirent64 *)dirt; size > 0;
 670                     dep = nextdp(dep)) {
 671 
 672                         size -= dep->d_reclen;
 673                         dirchunk_offset = dep->d_off;
 674 
 675                         /*
 676                          * Skip '.' and '..'
 677                          */
 678                         if (NFS_IS_DOTNAME(dep->d_name))
 679                                 continue;
 680 
 681                         ss_pn = rfs4_ss_pnalloc(statedir, dep->d_name);
 682                         if (ss_pn == NULL)
 683                                 continue;
 684 
 685                         if (cl_ss = rfs4_ss_getstate(dvp, ss_pn)) {
 686                                 if (destdir != NULL) {
 687                                         rfs4_ss_pnfree(ss_pn);
 688                                         cl_ss->ss_pn = rfs4_ss_movestate(
 689                                             statedir, destdir, dep->d_name);
 690                                 } else {
 691                                         cl_ss->ss_pn = ss_pn;
 692                                 }
 693                                 insque(cl_ss, oldstate);
 694                         } else {
 695                                 rfs4_ss_pnfree(ss_pn);
 696                         }
 697                 }
 698         }
 699 
 700 out:
 701         (void) VOP_CLOSE(dvp, FREAD, 1, (offset_t)0, CRED(), NULL);
 702         VN_RELE(dvp);
 703         if (dirt)
 704                 kmem_free((caddr_t)dirt, RFS4_SS_DIRSIZE);
 705 }
 706 
 707 static void
 708 rfs4_ss_init(void)
 709 {
 710         int npaths = 1;
 711         char *default_dss_path = NFS4_DSS_VAR_DIR;
 712 
 713         /* read the default stable storage state */
 714         rfs4_dss_readstate(npaths, &default_dss_path);
 715 
 716         rfs4_ss_enabled = 1;
 717 }
 718 
 719 static void
 720 rfs4_ss_fini(void)
 721 {
 722         rfs4_servinst_t *sip;
 723 
 724         mutex_enter(&rfs4_servinst_lock);
 725         sip = rfs4_cur_servinst;
 726         while (sip != NULL) {
 727                 rfs4_dss_clear_oldstate(sip);
 728                 sip = sip->next;
 729         }
 730         mutex_exit(&rfs4_servinst_lock);
 731 }
 732 
 733 /*
 734  * Remove all oldstate files referenced by this servinst.
 735  */
 736 static void
 737 rfs4_dss_clear_oldstate(rfs4_servinst_t *sip)
 738 {
 739         rfs4_oldstate_t *os_head, *osp;
 740 
 741         rw_enter(&sip->oldstate_lock, RW_WRITER);
 742         os_head = sip->oldstate;
 743 
 744         if (os_head == NULL) {
 745                 rw_exit(&sip->oldstate_lock);
 746                 return;
 747         }
 748 
 749         /* skip dummy entry */
 750         osp = os_head->next;
 751         while (osp != os_head) {
 752                 char *leaf = osp->ss_pn->leaf;
 753                 rfs4_oldstate_t *os_next;
 754 
 755                 rfs4_dss_remove_leaf(sip, NFS4_DSS_OLDSTATE_LEAF, leaf);
 756 
 757                 if (osp->cl_id4.id_val)
 758                         kmem_free(osp->cl_id4.id_val, osp->cl_id4.id_len);
 759                 rfs4_ss_pnfree(osp->ss_pn);
 760 
 761                 os_next = osp->next;
 762                 remque(osp);
 763                 kmem_free(osp, sizeof (rfs4_oldstate_t));
 764                 osp = os_next;
 765         }
 766 
 767         rw_exit(&sip->oldstate_lock);
 768 }
 769 
 770 /*
 771  * Form the state and oldstate paths, and read in the stable storage files.
 772  */
 773 void
 774 rfs4_dss_readstate(int npaths, char **paths)
 775 {
 776         int i;
 777         char *state, *oldstate;
 778 
 779         state = kmem_alloc(MAXPATHLEN, KM_SLEEP);
 780         oldstate = kmem_alloc(MAXPATHLEN, KM_SLEEP);
 781 
 782         for (i = 0; i < npaths; i++) {
 783                 char *path = paths[i];
 784 
 785                 (void) sprintf(state, "%s/%s", path, NFS4_DSS_STATE_LEAF);
 786                 (void) sprintf(oldstate, "%s/%s", path, NFS4_DSS_OLDSTATE_LEAF);
 787 
 788                 /*
 789                  * Populate the current server instance's oldstate list.
 790                  *
 791                  * 1. Read stable storage data from old state directory,
 792                  *    leaving its contents alone.
 793                  *
 794                  * 2. Read stable storage data from state directory,
 795                  *    and move the latter's contents to old state
 796                  *    directory.
 797                  */
 798                 rfs4_ss_oldstate(rfs4_cur_servinst->oldstate, oldstate, NULL);
 799                 rfs4_ss_oldstate(rfs4_cur_servinst->oldstate, state, oldstate);
 800         }
 801 
 802         kmem_free(state, MAXPATHLEN);
 803         kmem_free(oldstate, MAXPATHLEN);
 804 }
 805 
 806 
 807 /*
 808  * Check if we are still in grace and if the client can be
 809  * granted permission to perform reclaims.
 810  */
 811 void
 812 rfs4_ss_chkclid(rfs4_client_t *cp)
 813 {
 814         rfs4_servinst_t *sip;
 815 
 816         /*
 817          * It should be sufficient to check the oldstate data for just
 818          * this client's instance. However, since our per-instance
 819          * client grouping is solely temporal, HA-NFSv4 RG failover
 820          * might result in clients of the same RG being partitioned into
 821          * separate instances.
 822          *
 823          * Until the client grouping is improved, we must check the
 824          * oldstate data for all instances with an active grace period.
 825          *
 826          * This also serves as the mechanism to remove stale oldstate data.
 827          * The first time we check an instance after its grace period has
 828          * expired, the oldstate data should be cleared.
 829          *
 830          * Start at the current instance, and walk the list backwards
 831          * to the first.
 832          */
 833         mutex_enter(&rfs4_servinst_lock);
 834         for (sip = rfs4_cur_servinst; sip != NULL; sip = sip->prev) {
 835                 rfs4_ss_chkclid_sip(cp, sip);
 836 
 837                 /* if the above check found this client, we're done */
 838                 if (cp->rc_can_reclaim)
 839                         break;
 840         }
 841         mutex_exit(&rfs4_servinst_lock);
 842 }
 843 
 844 static void
 845 rfs4_ss_chkclid_sip(rfs4_client_t *cp, rfs4_servinst_t *sip)
 846 {
 847         rfs4_oldstate_t *osp, *os_head;
 848 
 849         /* short circuit everything if this server instance has no oldstate */
 850         rw_enter(&sip->oldstate_lock, RW_READER);
 851         os_head = sip->oldstate;
 852         rw_exit(&sip->oldstate_lock);
 853         if (os_head == NULL)
 854                 return;
 855 
 856         /*
 857          * If this server instance is no longer in a grace period then
 858          * the client won't be able to reclaim. No further need for this
 859          * instance's oldstate data, so it can be cleared.
 860          */
 861         if (!rfs4_servinst_in_grace(sip))
 862                 return;
 863 
 864         /* this instance is still in grace; search for the clientid */
 865 
 866         rw_enter(&sip->oldstate_lock, RW_READER);
 867 
 868         os_head = sip->oldstate;
 869         /* skip dummy entry */
 870         osp = os_head->next;
 871         while (osp != os_head) {
 872                 if (osp->cl_id4.id_len == cp->rc_nfs_client.id_len) {
 873                         if (bcmp(osp->cl_id4.id_val, cp->rc_nfs_client.id_val,
 874                             osp->cl_id4.id_len) == 0) {
 875                                 cp->rc_can_reclaim = 1;
 876                                 break;
 877                         }
 878                 }
 879                 osp = osp->next;
 880         }
 881 
 882         rw_exit(&sip->oldstate_lock);
 883 }
 884 
 885 /*
 886  * Place client information into stable storage: 1/3.
 887  * First, generate the leaf filename, from the client's IP address and
 888  * the server-generated short-hand clientid.
 889  */
 890 void
 891 rfs4_ss_clid(rfs4_client_t *cp)
 892 {
 893         const char *kinet_ntop6(uchar_t *, char *, size_t);
 894         char leaf[MAXNAMELEN], buf[INET6_ADDRSTRLEN];
 895         struct sockaddr *ca;
 896         uchar_t *b;
 897 
 898         if (rfs4_ss_enabled == 0) {
 899                 return;
 900         }
 901 
 902         buf[0] = 0;
 903 
 904         ca = (struct sockaddr *)&cp->rc_addr;
 905 
 906         /*
 907          * Convert the caller's IP address to a dotted string
 908          */
 909         if (ca->sa_family == AF_INET) {
 910                 b = (uchar_t *)&((struct sockaddr_in *)ca)->sin_addr;
 911                 (void) sprintf(buf, "%03d.%03d.%03d.%03d", b[0] & 0xFF,
 912                     b[1] & 0xFF, b[2] & 0xFF, b[3] & 0xFF);
 913         } else if (ca->sa_family == AF_INET6) {
 914                 struct sockaddr_in6 *sin6;
 915 
 916                 sin6 = (struct sockaddr_in6 *)ca;
 917                 (void) kinet_ntop6((uchar_t *)&sin6->sin6_addr,
 918                     buf, INET6_ADDRSTRLEN);
 919         }
 920 
 921         (void) snprintf(leaf, MAXNAMELEN, "%s-%llx", buf,
 922             (longlong_t)cp->rc_clientid);
 923         rfs4_ss_clid_write(cp, leaf);
 924 }
 925 
 926 /*
 927  * Place client information into stable storage: 2/3.
 928  * DSS: distributed stable storage: the file may need to be written to
 929  * multiple directories.
 930  */
 931 static void
 932 rfs4_ss_clid_write(rfs4_client_t *cp, char *leaf)
 933 {
 934         rfs4_servinst_t *sip;
 935 
 936         /*
 937          * It should be sufficient to write the leaf file to (all) DSS paths
 938          * associated with just this client's instance. However, since our
 939          * per-instance client grouping is solely temporal, HA-NFSv4 RG
 940          * failover might result in us losing DSS data.
 941          *
 942          * Until the client grouping is improved, we must write the DSS data
 943          * to all instances' paths. Start at the current instance, and
 944          * walk the list backwards to the first.
 945          */
 946         mutex_enter(&rfs4_servinst_lock);
 947         for (sip = rfs4_cur_servinst; sip != NULL; sip = sip->prev) {
 948                 int i, npaths = sip->dss_npaths;
 949 
 950                 /* write the leaf file to all DSS paths */
 951                 for (i = 0; i < npaths; i++) {
 952                         rfs4_dss_path_t *dss_path = sip->dss_paths[i];
 953 
 954                         /* HA-NFSv4 path might have been failed-away from us */
 955                         if (dss_path == NULL)
 956                                 continue;
 957 
 958                         rfs4_ss_clid_write_one(cp, dss_path->path, leaf);
 959                 }
 960         }
 961         mutex_exit(&rfs4_servinst_lock);
 962 }
 963 
 964 /*
 965  * Place client information into stable storage: 3/3.
 966  * Write the stable storage data to the requested file.
 967  */
 968 static void
 969 rfs4_ss_clid_write_one(rfs4_client_t *cp, char *dss_path, char *leaf)
 970 {
 971         int ioflag;
 972         int file_vers = NFS4_SS_VERSION;
 973         size_t dirlen;
 974         struct uio uio;
 975         struct iovec iov[4];
 976         char *dir;
 977         rfs4_ss_pn_t *ss_pn;
 978         vnode_t *vp;
 979         nfs_client_id4 *cl_id4 = &(cp->rc_nfs_client);
 980 
 981         /* allow 2 extra bytes for '/' & NUL */
 982         dirlen = strlen(dss_path) + strlen(NFS4_DSS_STATE_LEAF) + 2;
 983         dir = kmem_alloc(dirlen, KM_SLEEP);
 984         (void) sprintf(dir, "%s/%s", dss_path, NFS4_DSS_STATE_LEAF);
 985 
 986         ss_pn = rfs4_ss_pnalloc(dir, leaf);
 987         /* rfs4_ss_pnalloc takes its own copy */
 988         kmem_free(dir, dirlen);
 989         if (ss_pn == NULL)
 990                 return;
 991 
 992         if (vn_open(ss_pn->pn, UIO_SYSSPACE, FCREAT|FWRITE, 0600, &vp,
 993             CRCREAT, 0)) {
 994                 rfs4_ss_pnfree(ss_pn);
 995                 return;
 996         }
 997 
 998         /*
 999          * We need to record leaf - i.e. the filename - so that we know
1000          * what to remove, in the future. However, the dir part of cp->ss_pn
1001          * should never be referenced directly, since it's potentially only
1002          * one of several paths with this leaf in it.
1003          */
1004         if (cp->rc_ss_pn != NULL) {
1005                 if (strcmp(cp->rc_ss_pn->leaf, leaf) == 0) {
1006                         /* we've already recorded *this* leaf */
1007                         rfs4_ss_pnfree(ss_pn);
1008                 } else {
1009                         /* replace with this leaf */
1010                         rfs4_ss_pnfree(cp->rc_ss_pn);
1011                         cp->rc_ss_pn = ss_pn;
1012                 }
1013         } else {
1014                 cp->rc_ss_pn = ss_pn;
1015         }
1016 
1017         /*
1018          * Build a scatter list that points to the nfs_client_id4
1019          */
1020         iov[0].iov_base = (caddr_t)&file_vers;
1021         iov[0].iov_len = sizeof (int);
1022         iov[1].iov_base = (caddr_t)&(cl_id4->verifier);
1023         iov[1].iov_len = NFS4_VERIFIER_SIZE;
1024         iov[2].iov_base = (caddr_t)&(cl_id4->id_len);
1025         iov[2].iov_len = sizeof (uint_t);
1026         iov[3].iov_base = (caddr_t)cl_id4->id_val;
1027         iov[3].iov_len = cl_id4->id_len;
1028 
1029         uio.uio_iov = iov;
1030         uio.uio_iovcnt = 4;
1031         uio.uio_loffset = 0;
1032         uio.uio_segflg = UIO_SYSSPACE;
1033         uio.uio_llimit = (rlim64_t)MAXOFFSET_T;
1034         uio.uio_resid = cl_id4->id_len + sizeof (int) +
1035             NFS4_VERIFIER_SIZE + sizeof (uint_t);
1036 
1037         ioflag = uio.uio_fmode = (FWRITE|FSYNC);
1038         uio.uio_extflg = UIO_COPY_DEFAULT;
1039 
1040         (void) VOP_RWLOCK(vp, V_WRITELOCK_TRUE, NULL);
1041         /* write the full client id to the file. */
1042         (void) VOP_WRITE(vp, &uio, ioflag, CRED(), NULL);
1043         VOP_RWUNLOCK(vp, V_WRITELOCK_TRUE, NULL);
1044 
1045         (void) VOP_CLOSE(vp, FWRITE, 1, (offset_t)0, CRED(), NULL);
1046         VN_RELE(vp);
1047 }
1048 
1049 /*
1050  * DSS: distributed stable storage.
1051  * Unpack the list of paths passed by nfsd.
1052  * Use nvlist_alloc(9F) to manage the data.
1053  * The caller is responsible for allocating and freeing the buffer.
1054  */
1055 int
1056 rfs4_dss_setpaths(char *buf, size_t buflen)
1057 {
1058         int error;
1059 
1060         /*
1061          * If this is a "warm start", i.e. we previously had DSS paths,
1062          * preserve the old paths.
1063          */
1064         if (rfs4_dss_paths != NULL) {
1065                 /*
1066                  * Before we lose the ptr, destroy the nvlist and pathnames
1067                  * array from the warm start before this one.
1068                  */
1069                 if (rfs4_dss_oldpaths)
1070                         nvlist_free(rfs4_dss_oldpaths);
1071                 rfs4_dss_oldpaths = rfs4_dss_paths;
1072         }
1073 
1074         /* unpack the buffer into a searchable nvlist */
1075         error = nvlist_unpack(buf, buflen, &rfs4_dss_paths, KM_SLEEP);
1076         if (error)
1077                 return (error);
1078 
1079         /*
1080          * Search the nvlist for the pathnames nvpair (which is the only nvpair
1081          * in the list, and record its location.
1082          */
1083         error = nvlist_lookup_string_array(rfs4_dss_paths, NFS4_DSS_NVPAIR_NAME,
1084             &rfs4_dss_newpaths, &rfs4_dss_numnewpaths);
1085         return (error);
1086 }
1087 
1088 /*
1089  * Ultimately the nfssys() call NFS4_CLR_STATE endsup here
1090  * to find and mark the client for forced expire.
1091  */
1092 static void
1093 rfs4_client_scrub(rfs4_entry_t ent, void *arg)
1094 {
1095         rfs4_client_t *cp = (rfs4_client_t *)ent;
1096         struct nfs4clrst_args *clr = arg;
1097         struct sockaddr_in6 *ent_sin6;
1098         struct in6_addr  clr_in6;
1099         struct sockaddr_in  *ent_sin;
1100         struct in_addr   clr_in;
1101 
1102         if (clr->addr_type != cp->rc_addr.ss_family) {
1103                 return;
1104         }
1105 
1106         switch (clr->addr_type) {
1107 
1108         case AF_INET6:
1109                 /* copyin the address from user space */
1110                 if (copyin(clr->ap, &clr_in6, sizeof (clr_in6))) {
1111                         break;
1112                 }
1113 
1114                 ent_sin6 = (struct sockaddr_in6 *)&cp->rc_addr;
1115 
1116                 /*
1117                  * now compare, and if equivalent mark entry
1118                  * for forced expiration
1119                  */
1120                 if (IN6_ARE_ADDR_EQUAL(&ent_sin6->sin6_addr, &clr_in6)) {
1121                         cp->rc_forced_expire = 1;
1122                 }
1123                 break;
1124 
1125         case AF_INET:
1126                 /* copyin the address from user space */
1127                 if (copyin(clr->ap, &clr_in, sizeof (clr_in))) {
1128                         break;
1129                 }
1130 
1131                 ent_sin = (struct sockaddr_in *)&cp->rc_addr;
1132 
1133                 /*
1134                  * now compare, and if equivalent mark entry
1135                  * for forced expiration
1136                  */
1137                 if (ent_sin->sin_addr.s_addr == clr_in.s_addr) {
1138                         cp->rc_forced_expire = 1;
1139                 }
1140                 break;
1141 
1142         default:
1143                 /* force this assert to fail */
1144                 ASSERT(clr->addr_type != clr->addr_type);
1145         }
1146 }
1147 
1148 /*
1149  * This is called from nfssys() in order to clear server state
1150  * for the specified client IP Address.
1151  */
1152 void
1153 rfs4_clear_client_state(struct nfs4clrst_args *clr)
1154 {
1155         (void) rfs4_dbe_walk(rfs4_client_tab, rfs4_client_scrub, clr);
1156 }
1157 
1158 /*
1159  * Used to initialize the NFSv4 server's state or database.  All of
1160  * the tables are created and timers are set. Only called when NFSv4
1161  * service is provided.
1162  */
1163 void
1164 rfs4_state_init()
1165 {
1166         int start_grace;
1167         extern boolean_t rfs4_cpr_callb(void *, int);
1168         char *dss_path = NFS4_DSS_VAR_DIR;
1169         time_t start_time;
1170 
1171         mutex_enter(&rfs4_state_lock);
1172 
1173         /*
1174          * If the server state database has already been initialized,
1175          * skip it
1176          */
1177         if (rfs4_server_state != NULL) {
1178                 mutex_exit(&rfs4_state_lock);
1179                 return;
1180         }
1181 
1182         rw_init(&rfs4_findclient_lock, NULL, RW_DEFAULT, NULL);
1183 
1184         /*
1185          * Set the boot time.  If the server
1186          * has been restarted quickly and has had the opportunity to
1187          * service clients, then the start_time needs to be bumped
1188          * regardless.  A small window but it exists...
1189          */
1190         start_time = gethrestime_sec();
1191         if (rfs4_start_time < start_time)
1192                 rfs4_start_time = start_time;
1193         else
1194                 rfs4_start_time++;
1195 
1196         /* DSS: distributed stable storage: initialise served paths list */
1197         rfs4_dss_pathlist = NULL;
1198 
1199         /*
1200          * Create the first server instance, or a new one if the server has
1201          * been restarted; see above comments on rfs4_start_time. Don't
1202          * start its grace period; that will be done later, to maximise the
1203          * clients' recovery window.
1204          */
1205         start_grace = 0;
1206         rfs4_servinst_create(start_grace, 1, &dss_path);
1207 
1208         /* reset the "first NFSv4 request" status */
1209         rfs4_seen_first_compound = 0;
1210 
1211         /*
1212          * Add a CPR callback so that we can update client
1213          * access times to extend the lease after a suspend
1214          * and resume (using the same class as rpcmod/connmgr)
1215          */
1216         cpr_id = callb_add(rfs4_cpr_callb, 0, CB_CL_CPR_RPC, "rfs4");
1217 
1218         /* set the various cache timers for table creation */
1219         if (rfs4_client_cache_time == 0)
1220                 rfs4_client_cache_time = CLIENT_CACHE_TIME;
1221         if (rfs4_openowner_cache_time == 0)
1222                 rfs4_openowner_cache_time = OPENOWNER_CACHE_TIME;
1223         if (rfs4_state_cache_time == 0)
1224                 rfs4_state_cache_time = STATE_CACHE_TIME;
1225         if (rfs4_lo_state_cache_time == 0)
1226                 rfs4_lo_state_cache_time = LO_STATE_CACHE_TIME;
1227         if (rfs4_lockowner_cache_time == 0)
1228                 rfs4_lockowner_cache_time = LOCKOWNER_CACHE_TIME;
1229         if (rfs4_file_cache_time == 0)
1230                 rfs4_file_cache_time = FILE_CACHE_TIME;
1231         if (rfs4_deleg_state_cache_time == 0)
1232                 rfs4_deleg_state_cache_time = DELEG_STATE_CACHE_TIME;
1233 
1234         /* Create the overall database to hold all server state */
1235         rfs4_server_state = rfs4_database_create(rfs4_database_debug);
1236 
1237         /* Now create the individual tables */
1238         rfs4_client_cache_time *= rfs4_lease_time;
1239         rfs4_client_tab = rfs4_table_create(rfs4_server_state,
1240             "Client",
1241             rfs4_client_cache_time,
1242             2,
1243             rfs4_client_create,
1244             rfs4_client_destroy,
1245             rfs4_client_expiry,
1246             sizeof (rfs4_client_t),
1247             TABSIZE,
1248             MAXTABSZ/8, 100);
1249         rfs4_nfsclnt_idx = rfs4_index_create(rfs4_client_tab,
1250             "nfs_client_id4", nfsclnt_hash,
1251             nfsclnt_compare, nfsclnt_mkkey,
1252             TRUE);
1253         rfs4_clientid_idx = rfs4_index_create(rfs4_client_tab,
1254             "client_id", clientid_hash,
1255             clientid_compare, clientid_mkkey,
1256             FALSE);
1257 
1258         rfs4_clntip_cache_time = 86400 * 365;   /* about a year */
1259         rfs4_clntip_tab = rfs4_table_create(rfs4_server_state,
1260             "ClntIP",
1261             rfs4_clntip_cache_time,
1262             1,
1263             rfs4_clntip_create,
1264             rfs4_clntip_destroy,
1265             rfs4_clntip_expiry,
1266             sizeof (rfs4_clntip_t),
1267             TABSIZE,
1268             MAXTABSZ, 100);
1269         rfs4_clntip_idx = rfs4_index_create(rfs4_clntip_tab,
1270             "client_ip", clntip_hash,
1271             clntip_compare, clntip_mkkey,
1272             TRUE);
1273 
1274         rfs4_openowner_cache_time *= rfs4_lease_time;
1275         rfs4_openowner_tab = rfs4_table_create(rfs4_server_state,
1276             "OpenOwner",
1277             rfs4_openowner_cache_time,
1278             1,
1279             rfs4_openowner_create,
1280             rfs4_openowner_destroy,
1281             rfs4_openowner_expiry,
1282             sizeof (rfs4_openowner_t),
1283             TABSIZE,
1284             MAXTABSZ, 100);
1285         rfs4_openowner_idx = rfs4_index_create(rfs4_openowner_tab,
1286             "open_owner4", openowner_hash,
1287             openowner_compare,
1288             openowner_mkkey, TRUE);
1289 
1290         rfs4_state_cache_time *= rfs4_lease_time;
1291         rfs4_state_tab = rfs4_table_create(rfs4_server_state,
1292             "OpenStateID",
1293             rfs4_state_cache_time,
1294             3,
1295             rfs4_state_create,
1296             rfs4_state_destroy,
1297             rfs4_state_expiry,
1298             sizeof (rfs4_state_t),
1299             TABSIZE,
1300             MAXTABSZ, 100);
1301 
1302         rfs4_state_owner_file_idx = rfs4_index_create(rfs4_state_tab,
1303             "Openowner-File",
1304             state_owner_file_hash,
1305             state_owner_file_compare,
1306             state_owner_file_mkkey, TRUE);
1307 
1308         rfs4_state_idx = rfs4_index_create(rfs4_state_tab,
1309             "State-id", state_hash,
1310             state_compare, state_mkkey, FALSE);
1311 
1312         rfs4_state_file_idx = rfs4_index_create(rfs4_state_tab,
1313             "File", state_file_hash,
1314             state_file_compare, state_file_mkkey,
1315             FALSE);
1316 
1317         rfs4_lo_state_cache_time *= rfs4_lease_time;
1318         rfs4_lo_state_tab = rfs4_table_create(rfs4_server_state,
1319             "LockStateID",
1320             rfs4_lo_state_cache_time,
1321             2,
1322             rfs4_lo_state_create,
1323             rfs4_lo_state_destroy,
1324             rfs4_lo_state_expiry,
1325             sizeof (rfs4_lo_state_t),
1326             TABSIZE,
1327             MAXTABSZ, 100);
1328 
1329         rfs4_lo_state_owner_idx = rfs4_index_create(rfs4_lo_state_tab,
1330             "lockownerxstate",
1331             lo_state_lo_hash,
1332             lo_state_lo_compare,
1333             lo_state_lo_mkkey, TRUE);
1334 
1335         rfs4_lo_state_idx = rfs4_index_create(rfs4_lo_state_tab,
1336             "State-id",
1337             lo_state_hash, lo_state_compare,
1338             lo_state_mkkey, FALSE);
1339 
1340         rfs4_lockowner_cache_time *= rfs4_lease_time;
1341 
1342         rfs4_lockowner_tab = rfs4_table_create(rfs4_server_state,
1343             "Lockowner",
1344             rfs4_lockowner_cache_time,
1345             2,
1346             rfs4_lockowner_create,
1347             rfs4_lockowner_destroy,
1348             rfs4_lockowner_expiry,
1349             sizeof (rfs4_lockowner_t),
1350             TABSIZE,
1351             MAXTABSZ, 100);
1352 
1353         rfs4_lockowner_idx = rfs4_index_create(rfs4_lockowner_tab,
1354             "lock_owner4", lockowner_hash,
1355             lockowner_compare,
1356             lockowner_mkkey, TRUE);
1357 
1358         rfs4_lockowner_pid_idx = rfs4_index_create(rfs4_lockowner_tab,
1359             "pid", pid_hash,
1360             pid_compare, pid_mkkey,
1361             FALSE);
1362 
1363         rfs4_file_cache_time *= rfs4_lease_time;
1364         rfs4_file_tab = rfs4_table_create(rfs4_server_state,
1365             "File",
1366             rfs4_file_cache_time,
1367             1,
1368             rfs4_file_create,
1369             rfs4_file_destroy,
1370             NULL,
1371             sizeof (rfs4_file_t),
1372             TABSIZE,
1373             MAXTABSZ, -1);
1374 
1375         rfs4_file_idx = rfs4_index_create(rfs4_file_tab,
1376             "Filehandle", file_hash,
1377             file_compare, file_mkkey, TRUE);
1378 
1379         rfs4_deleg_state_cache_time *= rfs4_lease_time;
1380         rfs4_deleg_state_tab = rfs4_table_create(rfs4_server_state,
1381             "DelegStateID",
1382             rfs4_deleg_state_cache_time,
1383             2,
1384             rfs4_deleg_state_create,
1385             rfs4_deleg_state_destroy,
1386             rfs4_deleg_state_expiry,
1387             sizeof (rfs4_deleg_state_t),
1388             TABSIZE,
1389             MAXTABSZ, 100);
1390         rfs4_deleg_idx = rfs4_index_create(rfs4_deleg_state_tab,
1391             "DelegByFileClient",
1392             deleg_hash,
1393             deleg_compare,
1394             deleg_mkkey, TRUE);
1395 
1396         rfs4_deleg_state_idx = rfs4_index_create(rfs4_deleg_state_tab,
1397             "DelegState",
1398             deleg_state_hash,
1399             deleg_state_compare,
1400             deleg_state_mkkey, FALSE);
1401 
1402         /*
1403          * Init the stable storage.
1404          */
1405         rfs4_ss_init();
1406 
1407         rfs4_client_clrst = rfs4_clear_client_state;
1408 
1409         mutex_exit(&rfs4_state_lock);
1410 }
1411 
1412 
1413 /*
1414  * Used at server shutdown to cleanup all of the NFSv4 server's structures
1415  * and other state.
1416  */
1417 void
1418 rfs4_state_fini()
1419 {
1420         rfs4_database_t *dbp;
1421 
1422         mutex_enter(&rfs4_state_lock);
1423 
1424         if (rfs4_server_state == NULL) {
1425                 mutex_exit(&rfs4_state_lock);
1426                 return;
1427         }
1428 
1429         rfs4_client_clrst = NULL;
1430 
1431         rfs4_set_deleg_policy(SRV_NEVER_DELEGATE);
1432         dbp = rfs4_server_state;
1433         rfs4_server_state = NULL;
1434 
1435         /*
1436          * Cleanup the CPR callback.
1437          */
1438         if (cpr_id)
1439                 (void) callb_delete(cpr_id);
1440 
1441         rw_destroy(&rfs4_findclient_lock);
1442 
1443         /* First stop all of the reaper threads in the database */
1444         rfs4_database_shutdown(dbp);
1445         /* clean up any dangling stable storage structures */
1446         rfs4_ss_fini();
1447         /* Now actually destroy/release the database and its tables */
1448         rfs4_database_destroy(dbp);
1449 
1450         /* Reset the cache timers for next time */
1451         rfs4_client_cache_time = 0;
1452         rfs4_openowner_cache_time = 0;
1453         rfs4_state_cache_time = 0;
1454         rfs4_lo_state_cache_time = 0;
1455         rfs4_lockowner_cache_time = 0;
1456         rfs4_file_cache_time = 0;
1457         rfs4_deleg_state_cache_time = 0;
1458 
1459         mutex_exit(&rfs4_state_lock);
1460 
1461         /* destroy server instances and current instance ptr */
1462         rfs4_servinst_destroy_all();
1463 
1464         /* reset the "first NFSv4 request" status */
1465         rfs4_seen_first_compound = 0;
1466 
1467         /* DSS: distributed stable storage */
1468         if (rfs4_dss_oldpaths)
1469                 nvlist_free(rfs4_dss_oldpaths);
1470         if (rfs4_dss_paths)
1471                 nvlist_free(rfs4_dss_paths);
1472         rfs4_dss_paths = rfs4_dss_oldpaths = NULL;
1473 }
1474 
1475 typedef union {
1476         struct {
1477                 uint32_t start_time;
1478                 uint32_t c_id;
1479         } impl_id;
1480         clientid4 id4;
1481 } cid;
1482 
1483 static int foreign_stateid(stateid_t *id);
1484 static int foreign_clientid(cid *cidp);
1485 static void embed_nodeid(cid *cidp);
1486 
1487 typedef union {
1488         struct {
1489                 uint32_t c_id;
1490                 uint32_t gen_num;
1491         } cv_impl;
1492         verifier4       confirm_verf;
1493 } scid_confirm_verf;
1494 
1495 static uint32_t
1496 clientid_hash(void *key)
1497 {
1498         cid *idp = key;
1499 
1500         return (idp->impl_id.c_id);
1501 }
1502 
1503 static bool_t
1504 clientid_compare(rfs4_entry_t entry, void *key)
1505 {
1506         rfs4_client_t *cp = (rfs4_client_t *)entry;
1507         clientid4 *idp = key;
1508 
1509         return (*idp == cp->rc_clientid);
1510 }
1511 
1512 static void *
1513 clientid_mkkey(rfs4_entry_t entry)
1514 {
1515         rfs4_client_t *cp = (rfs4_client_t *)entry;
1516 
1517         return (&cp->rc_clientid);
1518 }
1519 
1520 static uint32_t
1521 nfsclnt_hash(void *key)
1522 {
1523         nfs_client_id4 *client = key;
1524         int i;
1525         uint32_t hash = 0;
1526 
1527         for (i = 0; i < client->id_len; i++) {
1528                 hash <<= 1;
1529                 hash += (uint_t)client->id_val[i];
1530         }
1531         return (hash);
1532 }
1533 
1534 
1535 static bool_t
1536 nfsclnt_compare(rfs4_entry_t entry, void *key)
1537 {
1538         rfs4_client_t *cp = (rfs4_client_t *)entry;
1539         nfs_client_id4 *nfs_client = key;
1540 
1541         if (cp->rc_nfs_client.id_len != nfs_client->id_len)
1542                 return (FALSE);
1543 
1544         return (bcmp(cp->rc_nfs_client.id_val, nfs_client->id_val,
1545             nfs_client->id_len) == 0);
1546 }
1547 
1548 static void *
1549 nfsclnt_mkkey(rfs4_entry_t entry)
1550 {
1551         rfs4_client_t *cp = (rfs4_client_t *)entry;
1552 
1553         return (&cp->rc_nfs_client);
1554 }
1555 
1556 static bool_t
1557 rfs4_client_expiry(rfs4_entry_t u_entry)
1558 {
1559         rfs4_client_t *cp = (rfs4_client_t *)u_entry;
1560         bool_t cp_expired;
1561 
1562         if (rfs4_dbe_is_invalid(cp->rc_dbe)) {
1563                 cp->rc_ss_remove = 1;
1564                 return (TRUE);
1565         }
1566         /*
1567          * If the sysadmin has used clear_locks for this
1568          * entry then forced_expire will be set and we
1569          * want this entry to be reaped. Or the entry
1570          * has exceeded its lease period.
1571          */
1572         cp_expired = (cp->rc_forced_expire ||
1573             (gethrestime_sec() - cp->rc_last_access
1574             > rfs4_lease_time));
1575 
1576         if (!cp->rc_ss_remove && cp_expired)
1577                 cp->rc_ss_remove = 1;
1578         return (cp_expired);
1579 }
1580 
1581 /*
1582  * Remove the leaf file from all distributed stable storage paths.
1583  */
1584 static void
1585 rfs4_dss_remove_cpleaf(rfs4_client_t *cp)
1586 {
1587         rfs4_servinst_t *sip;
1588         char *leaf = cp->rc_ss_pn->leaf;
1589 
1590         /*
1591          * since the state files are written to all DSS
1592          * paths we must remove this leaf file instance
1593          * from all server instances.
1594          */
1595 
1596         mutex_enter(&rfs4_servinst_lock);
1597         for (sip = rfs4_cur_servinst; sip != NULL; sip = sip->prev) {
1598                 /* remove the leaf file associated with this server instance */
1599                 rfs4_dss_remove_leaf(sip, NFS4_DSS_STATE_LEAF, leaf);
1600         }
1601         mutex_exit(&rfs4_servinst_lock);
1602 }
1603 
1604 static void
1605 rfs4_dss_remove_leaf(rfs4_servinst_t *sip, char *dir_leaf, char *leaf)
1606 {
1607         int i, npaths = sip->dss_npaths;
1608 
1609         for (i = 0; i < npaths; i++) {
1610                 rfs4_dss_path_t *dss_path = sip->dss_paths[i];
1611                 char *path, *dir;
1612                 size_t pathlen;
1613 
1614                 /* the HA-NFSv4 path might have been failed-over away from us */
1615                 if (dss_path == NULL)
1616                         continue;
1617 
1618                 dir = dss_path->path;
1619 
1620                 /* allow 3 extra bytes for two '/' & a NUL */
1621                 pathlen = strlen(dir) + strlen(dir_leaf) + strlen(leaf) + 3;
1622                 path = kmem_alloc(pathlen, KM_SLEEP);
1623                 (void) sprintf(path, "%s/%s/%s", dir, dir_leaf, leaf);
1624 
1625                 (void) vn_remove(path, UIO_SYSSPACE, RMFILE);
1626 
1627                 kmem_free(path, pathlen);
1628         }
1629 }
1630 
1631 static void
1632 rfs4_client_destroy(rfs4_entry_t u_entry)
1633 {
1634         rfs4_client_t *cp = (rfs4_client_t *)u_entry;
1635 
1636         mutex_destroy(cp->rc_cbinfo.cb_lock);
1637         cv_destroy(cp->rc_cbinfo.cb_cv);
1638         cv_destroy(cp->rc_cbinfo.cb_cv_nullcaller);
1639         list_destroy(&cp->rc_openownerlist);
1640 
1641         /* free callback info */
1642         rfs4_cbinfo_free(&cp->rc_cbinfo);
1643 
1644         if (cp->rc_cp_confirmed)
1645                 rfs4_client_rele(cp->rc_cp_confirmed);
1646 
1647         if (cp->rc_ss_pn) {
1648                 /* check if the stable storage files need to be removed */
1649                 if (cp->rc_ss_remove)
1650                         rfs4_dss_remove_cpleaf(cp);
1651                 rfs4_ss_pnfree(cp->rc_ss_pn);
1652         }
1653 
1654         /* Free the client supplied client id */
1655         kmem_free(cp->rc_nfs_client.id_val, cp->rc_nfs_client.id_len);
1656 
1657         if (cp->rc_sysidt != LM_NOSYSID)
1658                 lm_free_sysidt(cp->rc_sysidt);
1659 }
1660 
1661 static bool_t
1662 rfs4_client_create(rfs4_entry_t u_entry, void *arg)
1663 {
1664         rfs4_client_t *cp = (rfs4_client_t *)u_entry;
1665         nfs_client_id4 *client = (nfs_client_id4 *)arg;
1666         struct sockaddr *ca;
1667         cid *cidp;
1668         scid_confirm_verf *scvp;
1669 
1670         /* Get a clientid to give to the client */
1671         cidp = (cid *)&cp->rc_clientid;
1672         cidp->impl_id.start_time = rfs4_start_time;
1673         cidp->impl_id.c_id = (uint32_t)rfs4_dbe_getid(cp->rc_dbe);
1674 
1675         /* If we are booted as a cluster node, embed our nodeid */
1676         if (cluster_bootflags & CLUSTER_BOOTED)
1677                 embed_nodeid(cidp);
1678 
1679         /* Allocate and copy client's client id value */
1680         cp->rc_nfs_client.id_val = kmem_alloc(client->id_len, KM_SLEEP);
1681         cp->rc_nfs_client.id_len = client->id_len;
1682         bcopy(client->id_val, cp->rc_nfs_client.id_val, client->id_len);
1683         cp->rc_nfs_client.verifier = client->verifier;
1684 
1685         /* Copy client's IP address */
1686         ca = client->cl_addr;
1687         if (ca->sa_family == AF_INET)
1688                 bcopy(ca, &cp->rc_addr, sizeof (struct sockaddr_in));
1689         else if (ca->sa_family == AF_INET6)
1690                 bcopy(ca, &cp->rc_addr, sizeof (struct sockaddr_in6));
1691         cp->rc_nfs_client.cl_addr = (struct sockaddr *)&cp->rc_addr;
1692 
1693         /* Init the value for the SETCLIENTID_CONFIRM verifier */
1694         scvp = (scid_confirm_verf *)&cp->rc_confirm_verf;
1695         scvp->cv_impl.c_id = cidp->impl_id.c_id;
1696         scvp->cv_impl.gen_num = 0;
1697 
1698         /* An F_UNLKSYS has been done for this client */
1699         cp->rc_unlksys_completed = FALSE;
1700 
1701         /* We need the client to ack us */
1702         cp->rc_need_confirm = TRUE;
1703         cp->rc_cp_confirmed = NULL;
1704 
1705         /* TRUE all the time until the callback path actually fails */
1706         cp->rc_cbinfo.cb_notified_of_cb_path_down = TRUE;
1707 
1708         /* Initialize the access time to now */
1709         cp->rc_last_access = gethrestime_sec();
1710 
1711         cp->rc_cr_set = NULL;
1712 
1713         cp->rc_sysidt = LM_NOSYSID;
1714 
1715         list_create(&cp->rc_openownerlist, sizeof (rfs4_openowner_t),
1716             offsetof(rfs4_openowner_t, ro_node));
1717 
1718         /* set up the callback control structure */
1719         cp->rc_cbinfo.cb_state = CB_UNINIT;
1720         mutex_init(cp->rc_cbinfo.cb_lock, NULL, MUTEX_DEFAULT, NULL);
1721         cv_init(cp->rc_cbinfo.cb_cv, NULL, CV_DEFAULT, NULL);
1722         cv_init(cp->rc_cbinfo.cb_cv_nullcaller, NULL, CV_DEFAULT, NULL);
1723 
1724         /*
1725          * Associate the client_t with the current server instance.
1726          * The hold is solely to satisfy the calling requirement of
1727          * rfs4_servinst_assign(). In this case it's not strictly necessary.
1728          */
1729         rfs4_dbe_hold(cp->rc_dbe);
1730         rfs4_servinst_assign(cp, rfs4_cur_servinst);
1731         rfs4_dbe_rele(cp->rc_dbe);
1732 
1733         return (TRUE);
1734 }
1735 
1736 /*
1737  * Caller wants to generate/update the setclientid_confirm verifier
1738  * associated with a client.  This is done during the SETCLIENTID
1739  * processing.
1740  */
1741 void
1742 rfs4_client_scv_next(rfs4_client_t *cp)
1743 {
1744         scid_confirm_verf *scvp;
1745 
1746         /* Init the value for the SETCLIENTID_CONFIRM verifier */
1747         scvp = (scid_confirm_verf *)&cp->rc_confirm_verf;
1748         scvp->cv_impl.gen_num++;
1749 }
1750 
1751 void
1752 rfs4_client_rele(rfs4_client_t *cp)
1753 {
1754         rfs4_dbe_rele(cp->rc_dbe);
1755 }
1756 
1757 rfs4_client_t *
1758 rfs4_findclient(nfs_client_id4 *client, bool_t *create, rfs4_client_t *oldcp)
1759 {
1760         rfs4_client_t *cp;
1761 
1762 
1763         if (oldcp) {
1764                 rw_enter(&rfs4_findclient_lock, RW_WRITER);
1765                 rfs4_dbe_hide(oldcp->rc_dbe);
1766         } else {
1767                 rw_enter(&rfs4_findclient_lock, RW_READER);
1768         }
1769 
1770         cp = (rfs4_client_t *)rfs4_dbsearch(rfs4_nfsclnt_idx, client,
1771             create, (void *)client, RFS4_DBS_VALID);
1772 
1773         if (oldcp)
1774                 rfs4_dbe_unhide(oldcp->rc_dbe);
1775 
1776         rw_exit(&rfs4_findclient_lock);
1777 
1778         return (cp);
1779 }
1780 
1781 rfs4_client_t *
1782 rfs4_findclient_by_id(clientid4 clientid, bool_t find_unconfirmed)
1783 {
1784         rfs4_client_t *cp;
1785         bool_t create = FALSE;
1786         cid *cidp = (cid *)&clientid;
1787 
1788         /* If we're a cluster and the nodeid isn't right, short-circuit */
1789         if (cluster_bootflags & CLUSTER_BOOTED && foreign_clientid(cidp))
1790                 return (NULL);
1791 
1792         rw_enter(&rfs4_findclient_lock, RW_READER);
1793 
1794         cp = (rfs4_client_t *)rfs4_dbsearch(rfs4_clientid_idx, &clientid,
1795             &create, NULL, RFS4_DBS_VALID);
1796 
1797         rw_exit(&rfs4_findclient_lock);
1798 
1799         if (cp && cp->rc_need_confirm && find_unconfirmed == FALSE) {
1800                 rfs4_client_rele(cp);
1801                 return (NULL);
1802         } else {
1803                 return (cp);
1804         }
1805 }
1806 
1807 static uint32_t
1808 clntip_hash(void *key)
1809 {
1810         struct sockaddr *addr = key;
1811         int i, len = 0;
1812         uint32_t hash = 0;
1813         char *ptr;
1814 
1815         if (addr->sa_family == AF_INET) {
1816                 struct sockaddr_in *a = (struct sockaddr_in *)addr;
1817                 len = sizeof (struct in_addr);
1818                 ptr = (char *)&a->sin_addr;
1819         } else if (addr->sa_family == AF_INET6) {
1820                 struct sockaddr_in6 *a = (struct sockaddr_in6 *)addr;
1821                 len = sizeof (struct in6_addr);
1822                 ptr = (char *)&a->sin6_addr;
1823         } else
1824                 return (0);
1825 
1826         for (i = 0; i < len; i++) {
1827                 hash <<= 1;
1828                 hash += (uint_t)ptr[i];
1829         }
1830         return (hash);
1831 }
1832 
1833 static bool_t
1834 clntip_compare(rfs4_entry_t entry, void *key)
1835 {
1836         rfs4_clntip_t *cp = (rfs4_clntip_t *)entry;
1837         struct sockaddr *addr = key;
1838         int len = 0;
1839         char *p1, *p2;
1840 
1841         if (addr->sa_family == AF_INET) {
1842                 struct sockaddr_in *a1 = (struct sockaddr_in *)&cp->ri_addr;
1843                 struct sockaddr_in *a2 = (struct sockaddr_in *)addr;
1844                 len = sizeof (struct in_addr);
1845                 p1 = (char *)&a1->sin_addr;
1846                 p2 = (char *)&a2->sin_addr;
1847         } else if (addr->sa_family == AF_INET6) {
1848                 struct sockaddr_in6 *a1 = (struct sockaddr_in6 *)&cp->ri_addr;
1849                 struct sockaddr_in6 *a2 = (struct sockaddr_in6 *)addr;
1850                 len = sizeof (struct in6_addr);
1851                 p1 = (char *)&a1->sin6_addr;
1852                 p2 = (char *)&a2->sin6_addr;
1853         } else
1854                 return (0);
1855 
1856         return (bcmp(p1, p2, len) == 0);
1857 }
1858 
1859 static void *
1860 clntip_mkkey(rfs4_entry_t entry)
1861 {
1862         rfs4_clntip_t *cp = (rfs4_clntip_t *)entry;
1863 
1864         return (&cp->ri_addr);
1865 }
1866 
1867 static bool_t
1868 rfs4_clntip_expiry(rfs4_entry_t u_entry)
1869 {
1870         rfs4_clntip_t *cp = (rfs4_clntip_t *)u_entry;
1871 
1872         if (rfs4_dbe_is_invalid(cp->ri_dbe))
1873                 return (TRUE);
1874         return (FALSE);
1875 }
1876 
1877 /* ARGSUSED */
1878 static void
1879 rfs4_clntip_destroy(rfs4_entry_t u_entry)
1880 {
1881 }
1882 
1883 static bool_t
1884 rfs4_clntip_create(rfs4_entry_t u_entry, void *arg)
1885 {
1886         rfs4_clntip_t *cp = (rfs4_clntip_t *)u_entry;
1887         struct sockaddr *ca = (struct sockaddr *)arg;
1888 
1889         /* Copy client's IP address */
1890         if (ca->sa_family == AF_INET)
1891                 bcopy(ca, &cp->ri_addr, sizeof (struct sockaddr_in));
1892         else if (ca->sa_family == AF_INET6)
1893                 bcopy(ca, &cp->ri_addr, sizeof (struct sockaddr_in6));
1894         else
1895                 return (FALSE);
1896         cp->ri_no_referrals = 1;
1897 
1898         return (TRUE);
1899 }
1900 
1901 rfs4_clntip_t *
1902 rfs4_find_clntip(struct sockaddr *addr, bool_t *create)
1903 {
1904         rfs4_clntip_t *cp;
1905 
1906         rw_enter(&rfs4_findclient_lock, RW_READER);
1907 
1908         cp = (rfs4_clntip_t *)rfs4_dbsearch(rfs4_clntip_idx, addr,
1909             create, addr, RFS4_DBS_VALID);
1910 
1911         rw_exit(&rfs4_findclient_lock);
1912 
1913         return (cp);
1914 }
1915 
1916 void
1917 rfs4_invalidate_clntip(struct sockaddr *addr)
1918 {
1919         rfs4_clntip_t *cp;
1920         bool_t create = FALSE;
1921 
1922         rw_enter(&rfs4_findclient_lock, RW_READER);
1923 
1924         cp = (rfs4_clntip_t *)rfs4_dbsearch(rfs4_clntip_idx, addr,
1925             &create, NULL, RFS4_DBS_VALID);
1926         if (cp == NULL) {
1927                 rw_exit(&rfs4_findclient_lock);
1928                 return;
1929         }
1930         rfs4_dbe_invalidate(cp->ri_dbe);
1931         rfs4_dbe_rele(cp->ri_dbe);
1932 
1933         rw_exit(&rfs4_findclient_lock);
1934 }
1935 
1936 bool_t
1937 rfs4_lease_expired(rfs4_client_t *cp)
1938 {
1939         bool_t rc;
1940 
1941         rfs4_dbe_lock(cp->rc_dbe);
1942 
1943         /*
1944          * If the admin has executed clear_locks for this
1945          * client id, force expire will be set, so no need
1946          * to calculate anything because it's "outa here".
1947          */
1948         if (cp->rc_forced_expire) {
1949                 rc = TRUE;
1950         } else {
1951                 rc = (gethrestime_sec() - cp->rc_last_access > rfs4_lease_time);
1952         }
1953 
1954         /*
1955          * If the lease has expired we will also want
1956          * to remove any stable storage state data. So
1957          * mark the client id accordingly.
1958          */
1959         if (!cp->rc_ss_remove)
1960                 cp->rc_ss_remove = (rc == TRUE);
1961 
1962         rfs4_dbe_unlock(cp->rc_dbe);
1963 
1964         return (rc);
1965 }
1966 
1967 void
1968 rfs4_update_lease(rfs4_client_t *cp)
1969 {
1970         rfs4_dbe_lock(cp->rc_dbe);
1971         if (!cp->rc_forced_expire)
1972                 cp->rc_last_access = gethrestime_sec();
1973         rfs4_dbe_unlock(cp->rc_dbe);
1974 }
1975 
1976 
1977 static bool_t
1978 EQOPENOWNER(open_owner4 *a, open_owner4 *b)
1979 {
1980         bool_t rc;
1981 
1982         if (a->clientid != b->clientid)
1983                 return (FALSE);
1984 
1985         if (a->owner_len != b->owner_len)
1986                 return (FALSE);
1987 
1988         rc = (bcmp(a->owner_val, b->owner_val, a->owner_len) == 0);
1989 
1990         return (rc);
1991 }
1992 
1993 static uint_t
1994 openowner_hash(void *key)
1995 {
1996         int i;
1997         open_owner4 *openowner = key;
1998         uint_t hash = 0;
1999 
2000         for (i = 0; i < openowner->owner_len; i++) {
2001                 hash <<= 4;
2002                 hash += (uint_t)openowner->owner_val[i];
2003         }
2004         hash += (uint_t)openowner->clientid;
2005         hash |= (openowner->clientid >> 32);
2006 
2007         return (hash);
2008 }
2009 
2010 static bool_t
2011 openowner_compare(rfs4_entry_t u_entry, void *key)
2012 {
2013         rfs4_openowner_t *oo = (rfs4_openowner_t *)u_entry;
2014         open_owner4 *arg = key;
2015 
2016         return (EQOPENOWNER(&oo->ro_owner, arg));
2017 }
2018 
2019 void *
2020 openowner_mkkey(rfs4_entry_t u_entry)
2021 {
2022         rfs4_openowner_t *oo = (rfs4_openowner_t *)u_entry;
2023 
2024         return (&oo->ro_owner);
2025 }
2026 
2027 /* ARGSUSED */
2028 static bool_t
2029 rfs4_openowner_expiry(rfs4_entry_t u_entry)
2030 {
2031         /* openstateid held us and did all needed delay */
2032         return (TRUE);
2033 }
2034 
2035 static void
2036 rfs4_openowner_destroy(rfs4_entry_t u_entry)
2037 {
2038         rfs4_openowner_t *oo = (rfs4_openowner_t *)u_entry;
2039 
2040         /* Remove open owner from client's lists of open owners */
2041         rfs4_dbe_lock(oo->ro_client->rc_dbe);
2042         list_remove(&oo->ro_client->rc_openownerlist, oo);
2043         rfs4_dbe_unlock(oo->ro_client->rc_dbe);
2044 
2045         /* One less reference to the client */
2046         rfs4_client_rele(oo->ro_client);
2047         oo->ro_client = NULL;
2048 
2049         /* Free the last reply for this lock owner */
2050         rfs4_free_reply(&oo->ro_reply);
2051 
2052         if (oo->ro_reply_fh.nfs_fh4_val) {
2053                 kmem_free(oo->ro_reply_fh.nfs_fh4_val,
2054                     oo->ro_reply_fh.nfs_fh4_len);
2055                 oo->ro_reply_fh.nfs_fh4_val = NULL;
2056                 oo->ro_reply_fh.nfs_fh4_len = 0;
2057         }
2058 
2059         rfs4_sw_destroy(&oo->ro_sw);
2060         list_destroy(&oo->ro_statelist);
2061 
2062         /* Free the lock owner id */
2063         kmem_free(oo->ro_owner.owner_val, oo->ro_owner.owner_len);
2064 }
2065 
2066 void
2067 rfs4_openowner_rele(rfs4_openowner_t *oo)
2068 {
2069         rfs4_dbe_rele(oo->ro_dbe);
2070 }
2071 
2072 static bool_t
2073 rfs4_openowner_create(rfs4_entry_t u_entry, void *arg)
2074 {
2075         rfs4_openowner_t *oo = (rfs4_openowner_t *)u_entry;
2076         rfs4_openowner_t *argp = (rfs4_openowner_t *)arg;
2077         open_owner4 *openowner = &argp->ro_owner;
2078         seqid4 seqid = argp->ro_open_seqid;
2079         rfs4_client_t *cp;
2080         bool_t create = FALSE;
2081 
2082         rw_enter(&rfs4_findclient_lock, RW_READER);
2083 
2084         cp = (rfs4_client_t *)rfs4_dbsearch(rfs4_clientid_idx,
2085             &openowner->clientid,
2086             &create, NULL, RFS4_DBS_VALID);
2087 
2088         rw_exit(&rfs4_findclient_lock);
2089 
2090         if (cp == NULL)
2091                 return (FALSE);
2092 
2093         oo->ro_reply_fh.nfs_fh4_len = 0;
2094         oo->ro_reply_fh.nfs_fh4_val = NULL;
2095 
2096         oo->ro_owner.clientid = openowner->clientid;
2097         oo->ro_owner.owner_val =
2098             kmem_alloc(openowner->owner_len, KM_SLEEP);
2099 
2100         bcopy(openowner->owner_val,
2101             oo->ro_owner.owner_val, openowner->owner_len);
2102 
2103         oo->ro_owner.owner_len = openowner->owner_len;
2104 
2105         oo->ro_need_confirm = TRUE;
2106 
2107         rfs4_sw_init(&oo->ro_sw);
2108 
2109         oo->ro_open_seqid = seqid;
2110         bzero(&oo->ro_reply, sizeof (nfs_resop4));
2111         oo->ro_client = cp;
2112         oo->ro_cr_set = NULL;
2113 
2114         list_create(&oo->ro_statelist, sizeof (rfs4_state_t),
2115             offsetof(rfs4_state_t, rs_node));
2116 
2117         /* Insert openowner into client's open owner list */
2118         rfs4_dbe_lock(cp->rc_dbe);
2119         list_insert_tail(&cp->rc_openownerlist, oo);
2120         rfs4_dbe_unlock(cp->rc_dbe);
2121 
2122         return (TRUE);
2123 }
2124 
2125 rfs4_openowner_t *
2126 rfs4_findopenowner(open_owner4 *openowner, bool_t *create, seqid4 seqid)
2127 {
2128         rfs4_openowner_t *oo;
2129         rfs4_openowner_t arg;
2130 
2131         arg.ro_owner = *openowner;
2132         arg.ro_open_seqid = seqid;
2133         oo = (rfs4_openowner_t *)rfs4_dbsearch(rfs4_openowner_idx, openowner,
2134             create, &arg, RFS4_DBS_VALID);
2135 
2136         return (oo);
2137 }
2138 
2139 void
2140 rfs4_update_open_sequence(rfs4_openowner_t *oo)
2141 {
2142 
2143         rfs4_dbe_lock(oo->ro_dbe);
2144 
2145         oo->ro_open_seqid++;
2146 
2147         rfs4_dbe_unlock(oo->ro_dbe);
2148 }
2149 
2150 void
2151 rfs4_update_open_resp(rfs4_openowner_t *oo, nfs_resop4 *resp, nfs_fh4 *fh)
2152 {
2153 
2154         rfs4_dbe_lock(oo->ro_dbe);
2155 
2156         rfs4_free_reply(&oo->ro_reply);
2157 
2158         rfs4_copy_reply(&oo->ro_reply, resp);
2159 
2160         /* Save the filehandle if provided and free if not used */
2161         if (resp->nfs_resop4_u.opopen.status == NFS4_OK &&
2162             fh && fh->nfs_fh4_len) {
2163                 if (oo->ro_reply_fh.nfs_fh4_val == NULL)
2164                         oo->ro_reply_fh.nfs_fh4_val =
2165                             kmem_alloc(fh->nfs_fh4_len, KM_SLEEP);
2166                 nfs_fh4_copy(fh, &oo->ro_reply_fh);
2167         } else {
2168                 if (oo->ro_reply_fh.nfs_fh4_val) {
2169                         kmem_free(oo->ro_reply_fh.nfs_fh4_val,
2170                             oo->ro_reply_fh.nfs_fh4_len);
2171                         oo->ro_reply_fh.nfs_fh4_val = NULL;
2172                         oo->ro_reply_fh.nfs_fh4_len = 0;
2173                 }
2174         }
2175 
2176         rfs4_dbe_unlock(oo->ro_dbe);
2177 }
2178 
2179 static bool_t
2180 lockowner_compare(rfs4_entry_t u_entry, void *key)
2181 {
2182         rfs4_lockowner_t *lo = (rfs4_lockowner_t *)u_entry;
2183         lock_owner4 *b = (lock_owner4 *)key;
2184 
2185         if (lo->rl_owner.clientid != b->clientid)
2186                 return (FALSE);
2187 
2188         if (lo->rl_owner.owner_len != b->owner_len)
2189                 return (FALSE);
2190 
2191         return (bcmp(lo->rl_owner.owner_val, b->owner_val,
2192             lo->rl_owner.owner_len) == 0);
2193 }
2194 
2195 void *
2196 lockowner_mkkey(rfs4_entry_t u_entry)
2197 {
2198         rfs4_lockowner_t *lo = (rfs4_lockowner_t *)u_entry;
2199 
2200         return (&lo->rl_owner);
2201 }
2202 
2203 static uint32_t
2204 lockowner_hash(void *key)
2205 {
2206         int i;
2207         lock_owner4 *lockowner = key;
2208         uint_t hash = 0;
2209 
2210         for (i = 0; i < lockowner->owner_len; i++) {
2211                 hash <<= 4;
2212                 hash += (uint_t)lockowner->owner_val[i];
2213         }
2214         hash += (uint_t)lockowner->clientid;
2215         hash |= (lockowner->clientid >> 32);
2216 
2217         return (hash);
2218 }
2219 
2220 static uint32_t
2221 pid_hash(void *key)
2222 {
2223         return ((uint32_t)(uintptr_t)key);
2224 }
2225 
2226 static void *
2227 pid_mkkey(rfs4_entry_t u_entry)
2228 {
2229         rfs4_lockowner_t *lo = (rfs4_lockowner_t *)u_entry;
2230 
2231         return ((void *)(uintptr_t)lo->rl_pid);
2232 }
2233 
2234 static bool_t
2235 pid_compare(rfs4_entry_t u_entry, void *key)
2236 {
2237         rfs4_lockowner_t *lo = (rfs4_lockowner_t *)u_entry;
2238 
2239         return (lo->rl_pid == (pid_t)(uintptr_t)key);
2240 }
2241 
2242 static void
2243 rfs4_lockowner_destroy(rfs4_entry_t u_entry)
2244 {
2245         rfs4_lockowner_t *lo = (rfs4_lockowner_t *)u_entry;
2246 
2247         /* Free the lock owner id */
2248         kmem_free(lo->rl_owner.owner_val, lo->rl_owner.owner_len);
2249         rfs4_client_rele(lo->rl_client);
2250 }
2251 
2252 void
2253 rfs4_lockowner_rele(rfs4_lockowner_t *lo)
2254 {
2255         rfs4_dbe_rele(lo->rl_dbe);
2256 }
2257 
2258 /* ARGSUSED */
2259 static bool_t
2260 rfs4_lockowner_expiry(rfs4_entry_t u_entry)
2261 {
2262         /*
2263          * Since expiry is called with no other references on
2264          * this struct, go ahead and have it removed.
2265          */
2266         return (TRUE);
2267 }
2268 
2269 static bool_t
2270 rfs4_lockowner_create(rfs4_entry_t u_entry, void *arg)
2271 {
2272         rfs4_lockowner_t *lo = (rfs4_lockowner_t *)u_entry;
2273         lock_owner4 *lockowner = (lock_owner4 *)arg;
2274         rfs4_client_t *cp;
2275         bool_t create = FALSE;
2276 
2277         rw_enter(&rfs4_findclient_lock, RW_READER);
2278 
2279         cp = (rfs4_client_t *)rfs4_dbsearch(rfs4_clientid_idx,
2280             &lockowner->clientid,
2281             &create, NULL, RFS4_DBS_VALID);
2282 
2283         rw_exit(&rfs4_findclient_lock);
2284 
2285         if (cp == NULL)
2286                 return (FALSE);
2287 
2288         /* Reference client */
2289         lo->rl_client = cp;
2290         lo->rl_owner.clientid = lockowner->clientid;
2291         lo->rl_owner.owner_val = kmem_alloc(lockowner->owner_len, KM_SLEEP);
2292         bcopy(lockowner->owner_val, lo->rl_owner.owner_val,
2293             lockowner->owner_len);
2294         lo->rl_owner.owner_len = lockowner->owner_len;
2295         lo->rl_pid = rfs4_dbe_getid(lo->rl_dbe);
2296 
2297         return (TRUE);
2298 }
2299 
2300 rfs4_lockowner_t *
2301 rfs4_findlockowner(lock_owner4 *lockowner, bool_t *create)
2302 {
2303         rfs4_lockowner_t *lo;
2304 
2305         lo = (rfs4_lockowner_t *)rfs4_dbsearch(rfs4_lockowner_idx, lockowner,
2306             create, lockowner, RFS4_DBS_VALID);
2307 
2308         return (lo);
2309 }
2310 
2311 rfs4_lockowner_t *
2312 rfs4_findlockowner_by_pid(pid_t pid)
2313 {
2314         rfs4_lockowner_t *lo;
2315         bool_t create = FALSE;
2316 
2317         lo = (rfs4_lockowner_t *)rfs4_dbsearch(rfs4_lockowner_pid_idx,
2318             (void *)(uintptr_t)pid, &create, NULL, RFS4_DBS_VALID);
2319 
2320         return (lo);
2321 }
2322 
2323 
2324 static uint32_t
2325 file_hash(void *key)
2326 {
2327         return (ADDRHASH(key));
2328 }
2329 
2330 static void *
2331 file_mkkey(rfs4_entry_t u_entry)
2332 {
2333         rfs4_file_t *fp = (rfs4_file_t *)u_entry;
2334 
2335         return (fp->rf_vp);
2336 }
2337 
2338 static bool_t
2339 file_compare(rfs4_entry_t u_entry, void *key)
2340 {
2341         rfs4_file_t *fp = (rfs4_file_t *)u_entry;
2342 
2343         return (fp->rf_vp == (vnode_t *)key);
2344 }
2345 
2346 static void
2347 rfs4_file_destroy(rfs4_entry_t u_entry)
2348 {
2349         rfs4_file_t *fp = (rfs4_file_t *)u_entry;
2350 
2351         list_destroy(&fp->rf_delegstatelist);
2352 
2353         if (fp->rf_filehandle.nfs_fh4_val)
2354                 kmem_free(fp->rf_filehandle.nfs_fh4_val,
2355                     fp->rf_filehandle.nfs_fh4_len);
2356         cv_destroy(fp->rf_dinfo.rd_recall_cv);
2357         if (fp->rf_vp) {
2358                 vnode_t *vp = fp->rf_vp;
2359 
2360                 mutex_enter(&vp->v_vsd_lock);
2361                 (void) vsd_set(vp, nfs4_srv_vkey, NULL);
2362                 mutex_exit(&vp->v_vsd_lock);
2363                 VN_RELE(vp);
2364                 fp->rf_vp = NULL;
2365         }
2366         rw_destroy(&fp->rf_file_rwlock);
2367 }
2368 
2369 /*
2370  * Used to unlock the underlying dbe struct only
2371  */
2372 void
2373 rfs4_file_rele(rfs4_file_t *fp)
2374 {
2375         rfs4_dbe_rele(fp->rf_dbe);
2376 }
2377 
2378 typedef struct {
2379     vnode_t *vp;
2380     nfs_fh4 *fh;
2381 } rfs4_fcreate_arg;
2382 
2383 static bool_t
2384 rfs4_file_create(rfs4_entry_t u_entry, void *arg)
2385 {
2386         rfs4_file_t *fp = (rfs4_file_t *)u_entry;
2387         rfs4_fcreate_arg *ap = (rfs4_fcreate_arg *)arg;
2388         vnode_t *vp = ap->vp;
2389         nfs_fh4 *fh = ap->fh;
2390 
2391         VN_HOLD(vp);
2392 
2393         fp->rf_filehandle.nfs_fh4_len = 0;
2394         fp->rf_filehandle.nfs_fh4_val = NULL;
2395         ASSERT(fh && fh->nfs_fh4_len);
2396         if (fh && fh->nfs_fh4_len) {
2397                 fp->rf_filehandle.nfs_fh4_val =
2398                     kmem_alloc(fh->nfs_fh4_len, KM_SLEEP);
2399                 nfs_fh4_copy(fh, &fp->rf_filehandle);
2400         }
2401         fp->rf_vp = vp;
2402 
2403         list_create(&fp->rf_delegstatelist, sizeof (rfs4_deleg_state_t),
2404             offsetof(rfs4_deleg_state_t, rds_node));
2405 
2406         fp->rf_share_deny = fp->rf_share_access = fp->rf_access_read = 0;
2407         fp->rf_access_write = fp->rf_deny_read = fp->rf_deny_write = 0;
2408 
2409         mutex_init(fp->rf_dinfo.rd_recall_lock, NULL, MUTEX_DEFAULT, NULL);
2410         cv_init(fp->rf_dinfo.rd_recall_cv, NULL, CV_DEFAULT, NULL);
2411 
2412         fp->rf_dinfo.rd_dtype = OPEN_DELEGATE_NONE;
2413 
2414         rw_init(&fp->rf_file_rwlock, NULL, RW_DEFAULT, NULL);
2415 
2416         mutex_enter(&vp->v_vsd_lock);
2417         VERIFY(vsd_set(vp, nfs4_srv_vkey, (void *)fp) == 0);
2418         mutex_exit(&vp->v_vsd_lock);
2419 
2420         return (TRUE);
2421 }
2422 
2423 rfs4_file_t *
2424 rfs4_findfile(vnode_t *vp, nfs_fh4 *fh, bool_t *create)
2425 {
2426         rfs4_file_t *fp;
2427         rfs4_fcreate_arg arg;
2428 
2429         arg.vp = vp;
2430         arg.fh = fh;
2431 
2432         if (*create == TRUE)
2433                 fp = (rfs4_file_t *)rfs4_dbsearch(rfs4_file_idx, vp, create,
2434                     &arg, RFS4_DBS_VALID);
2435         else {
2436                 mutex_enter(&vp->v_vsd_lock);
2437                 fp = (rfs4_file_t *)vsd_get(vp, nfs4_srv_vkey);
2438                 if (fp) {
2439                         rfs4_dbe_lock(fp->rf_dbe);
2440                         if (rfs4_dbe_is_invalid(fp->rf_dbe) ||
2441                             (rfs4_dbe_refcnt(fp->rf_dbe) == 0)) {
2442                                 rfs4_dbe_unlock(fp->rf_dbe);
2443                                 fp = NULL;
2444                         } else {
2445                                 rfs4_dbe_hold(fp->rf_dbe);
2446                                 rfs4_dbe_unlock(fp->rf_dbe);
2447                         }
2448                 }
2449                 mutex_exit(&vp->v_vsd_lock);
2450         }
2451         return (fp);
2452 }
2453 
2454 /*
2455  * Find a file in the db and once it is located, take the rw lock.
2456  * Need to check the vnode pointer and if it does not exist (it was
2457  * removed between the db location and check) redo the find.  This
2458  * assumes that a file struct that has a NULL vnode pointer is marked
2459  * at 'invalid' and will not be found in the db the second time
2460  * around.
2461  */
2462 rfs4_file_t *
2463 rfs4_findfile_withlock(vnode_t *vp, nfs_fh4 *fh, bool_t *create)
2464 {
2465         rfs4_file_t *fp;
2466         rfs4_fcreate_arg arg;
2467         bool_t screate = *create;
2468 
2469         if (screate == FALSE) {
2470                 mutex_enter(&vp->v_vsd_lock);
2471                 fp = (rfs4_file_t *)vsd_get(vp, nfs4_srv_vkey);
2472                 if (fp) {
2473                         rfs4_dbe_lock(fp->rf_dbe);
2474                         if (rfs4_dbe_is_invalid(fp->rf_dbe) ||
2475                             (rfs4_dbe_refcnt(fp->rf_dbe) == 0)) {
2476                                 rfs4_dbe_unlock(fp->rf_dbe);
2477                                 mutex_exit(&vp->v_vsd_lock);
2478                                 fp = NULL;
2479                         } else {
2480                                 rfs4_dbe_hold(fp->rf_dbe);
2481                                 rfs4_dbe_unlock(fp->rf_dbe);
2482                                 mutex_exit(&vp->v_vsd_lock);
2483                                 rw_enter(&fp->rf_file_rwlock, RW_WRITER);
2484                                 if (fp->rf_vp == NULL) {
2485                                         rw_exit(&fp->rf_file_rwlock);
2486                                         rfs4_file_rele(fp);
2487                                         fp = NULL;
2488                                 }
2489                         }
2490                 } else {
2491                         mutex_exit(&vp->v_vsd_lock);
2492                 }
2493         } else {
2494 retry:
2495                 arg.vp = vp;
2496                 arg.fh = fh;
2497 
2498                 fp = (rfs4_file_t *)rfs4_dbsearch(rfs4_file_idx, vp, create,
2499                     &arg, RFS4_DBS_VALID);
2500                 if (fp != NULL) {
2501                         rw_enter(&fp->rf_file_rwlock, RW_WRITER);
2502                         if (fp->rf_vp == NULL) {
2503                                 rw_exit(&fp->rf_file_rwlock);
2504                                 rfs4_file_rele(fp);
2505                                 *create = screate;
2506                                 goto retry;
2507                         }
2508                 }
2509         }
2510 
2511         return (fp);
2512 }
2513 
2514 static uint32_t
2515 lo_state_hash(void *key)
2516 {
2517         stateid_t *id = key;
2518 
2519         return (id->bits.ident+id->bits.pid);
2520 }
2521 
2522 static bool_t
2523 lo_state_compare(rfs4_entry_t u_entry, void *key)
2524 {
2525         rfs4_lo_state_t *lsp = (rfs4_lo_state_t *)u_entry;
2526         stateid_t *id = key;
2527         bool_t rc;
2528 
2529         rc = (lsp->rls_lockid.bits.boottime == id->bits.boottime &&
2530             lsp->rls_lockid.bits.type == id->bits.type &&
2531             lsp->rls_lockid.bits.ident == id->bits.ident &&
2532             lsp->rls_lockid.bits.pid == id->bits.pid);
2533 
2534         return (rc);
2535 }
2536 
2537 static void *
2538 lo_state_mkkey(rfs4_entry_t u_entry)
2539 {
2540         rfs4_lo_state_t *lsp = (rfs4_lo_state_t *)u_entry;
2541 
2542         return (&lsp->rls_lockid);
2543 }
2544 
2545 static bool_t
2546 rfs4_lo_state_expiry(rfs4_entry_t u_entry)
2547 {
2548         rfs4_lo_state_t *lsp = (rfs4_lo_state_t *)u_entry;
2549 
2550         if (rfs4_dbe_is_invalid(lsp->rls_dbe))
2551                 return (TRUE);
2552         if (lsp->rls_state->rs_closed)
2553                 return (TRUE);
2554         return ((gethrestime_sec() -
2555             lsp->rls_state->rs_owner->ro_client->rc_last_access
2556             > rfs4_lease_time));
2557 }
2558 
2559 static void
2560 rfs4_lo_state_destroy(rfs4_entry_t u_entry)
2561 {
2562         rfs4_lo_state_t *lsp = (rfs4_lo_state_t *)u_entry;
2563 
2564         rfs4_dbe_lock(lsp->rls_state->rs_dbe);
2565         list_remove(&lsp->rls_state->rs_lostatelist, lsp);
2566         rfs4_dbe_unlock(lsp->rls_state->rs_dbe);
2567 
2568         rfs4_sw_destroy(&lsp->rls_sw);
2569 
2570         /* Make sure to release the file locks */
2571         if (lsp->rls_locks_cleaned == FALSE) {
2572                 lsp->rls_locks_cleaned = TRUE;
2573                 if (lsp->rls_locker->rl_client->rc_sysidt != LM_NOSYSID) {
2574                         /* Is the PxFS kernel module loaded? */
2575                         if (lm_remove_file_locks != NULL) {
2576                                 int new_sysid;
2577 
2578                                 /* Encode the cluster nodeid in new sysid */
2579                                 new_sysid =
2580                                     lsp->rls_locker->rl_client->rc_sysidt;
2581                                 lm_set_nlmid_flk(&new_sysid);
2582 
2583                                 /*
2584                                  * This PxFS routine removes file locks for a
2585                                  * client over all nodes of a cluster.
2586                                  */
2587                                 DTRACE_PROBE1(nfss_i_clust_rm_lck,
2588                                     int, new_sysid);
2589                                 (*lm_remove_file_locks)(new_sysid);
2590                         } else {
2591                                 (void) cleanlocks(
2592                                     lsp->rls_state->rs_finfo->rf_vp,
2593                                     lsp->rls_locker->rl_pid,
2594                                     lsp->rls_locker->rl_client->rc_sysidt);
2595                         }
2596                 }
2597         }
2598 
2599         /* Free the last reply for this state */
2600         rfs4_free_reply(&lsp->rls_reply);
2601 
2602         rfs4_lockowner_rele(lsp->rls_locker);
2603         lsp->rls_locker = NULL;
2604 
2605         rfs4_state_rele_nounlock(lsp->rls_state);
2606         lsp->rls_state = NULL;
2607 }
2608 
2609 static bool_t
2610 rfs4_lo_state_create(rfs4_entry_t u_entry, void *arg)
2611 {
2612         rfs4_lo_state_t *lsp = (rfs4_lo_state_t *)u_entry;
2613         rfs4_lo_state_t *argp = (rfs4_lo_state_t *)arg;
2614         rfs4_lockowner_t *lo = argp->rls_locker;
2615         rfs4_state_t *sp = argp->rls_state;
2616 
2617         lsp->rls_state = sp;
2618 
2619         lsp->rls_lockid = sp->rs_stateid;
2620         lsp->rls_lockid.bits.type = LOCKID;
2621         lsp->rls_lockid.bits.chgseq = 0;
2622         lsp->rls_lockid.bits.pid = lo->rl_pid;
2623 
2624         lsp->rls_locks_cleaned = FALSE;
2625         lsp->rls_lock_completed = FALSE;
2626 
2627         rfs4_sw_init(&lsp->rls_sw);
2628 
2629         /* Attached the supplied lock owner */
2630         rfs4_dbe_hold(lo->rl_dbe);
2631         lsp->rls_locker = lo;
2632 
2633         rfs4_dbe_lock(sp->rs_dbe);
2634         list_insert_tail(&sp->rs_lostatelist, lsp);
2635         rfs4_dbe_hold(sp->rs_dbe);
2636         rfs4_dbe_unlock(sp->rs_dbe);
2637 
2638         return (TRUE);
2639 }
2640 
2641 void
2642 rfs4_lo_state_rele(rfs4_lo_state_t *lsp, bool_t unlock_fp)
2643 {
2644         if (unlock_fp == TRUE)
2645                 rw_exit(&lsp->rls_state->rs_finfo->rf_file_rwlock);
2646         rfs4_dbe_rele(lsp->rls_dbe);
2647 }
2648 
2649 static rfs4_lo_state_t *
2650 rfs4_findlo_state(stateid_t *id, bool_t lock_fp)
2651 {
2652         rfs4_lo_state_t *lsp;
2653         bool_t create = FALSE;
2654 
2655         lsp = (rfs4_lo_state_t *)rfs4_dbsearch(rfs4_lo_state_idx, id,
2656             &create, NULL, RFS4_DBS_VALID);
2657         if (lock_fp == TRUE && lsp != NULL)
2658                 rw_enter(&lsp->rls_state->rs_finfo->rf_file_rwlock, RW_READER);
2659 
2660         return (lsp);
2661 }
2662 
2663 
2664 static uint32_t
2665 lo_state_lo_hash(void *key)
2666 {
2667         rfs4_lo_state_t *lsp = key;
2668 
2669         return (ADDRHASH(lsp->rls_locker) ^ ADDRHASH(lsp->rls_state));
2670 }
2671 
2672 static bool_t
2673 lo_state_lo_compare(rfs4_entry_t u_entry, void *key)
2674 {
2675         rfs4_lo_state_t *lsp = (rfs4_lo_state_t *)u_entry;
2676         rfs4_lo_state_t *keyp = key;
2677 
2678         return (keyp->rls_locker == lsp->rls_locker &&
2679             keyp->rls_state == lsp->rls_state);
2680 }
2681 
2682 static void *
2683 lo_state_lo_mkkey(rfs4_entry_t u_entry)
2684 {
2685         return (u_entry);
2686 }
2687 
2688 rfs4_lo_state_t *
2689 rfs4_findlo_state_by_owner(rfs4_lockowner_t *lo, rfs4_state_t *sp,
2690     bool_t *create)
2691 {
2692         rfs4_lo_state_t *lsp;
2693         rfs4_lo_state_t arg;
2694 
2695         arg.rls_locker = lo;
2696         arg.rls_state = sp;
2697 
2698         lsp = (rfs4_lo_state_t *)rfs4_dbsearch(rfs4_lo_state_owner_idx, &arg,
2699             create, &arg, RFS4_DBS_VALID);
2700 
2701         return (lsp);
2702 }
2703 
2704 static stateid_t
2705 get_stateid(id_t eid)
2706 {
2707         stateid_t id;
2708 
2709         id.bits.boottime = rfs4_start_time;
2710         id.bits.ident = eid;
2711         id.bits.chgseq = 0;
2712         id.bits.type = 0;
2713         id.bits.pid = 0;
2714 
2715         /*
2716          * If we are booted as a cluster node, embed our nodeid.
2717          * We've already done sanity checks in rfs4_client_create() so no
2718          * need to repeat them here.
2719          */
2720         id.bits.clnodeid = (cluster_bootflags & CLUSTER_BOOTED) ?
2721             clconf_get_nodeid() : 0;
2722 
2723         return (id);
2724 }
2725 
2726 /*
2727  * For use only when booted as a cluster node.
2728  * Returns TRUE if the embedded nodeid indicates that this stateid was
2729  * generated on another node.
2730  */
2731 static int
2732 foreign_stateid(stateid_t *id)
2733 {
2734         ASSERT(cluster_bootflags & CLUSTER_BOOTED);
2735         return (id->bits.clnodeid != (uint32_t)clconf_get_nodeid());
2736 }
2737 
2738 /*
2739  * For use only when booted as a cluster node.
2740  * Returns TRUE if the embedded nodeid indicates that this clientid was
2741  * generated on another node.
2742  */
2743 static int
2744 foreign_clientid(cid *cidp)
2745 {
2746         ASSERT(cluster_bootflags & CLUSTER_BOOTED);
2747         return (cidp->impl_id.c_id >> CLUSTER_NODEID_SHIFT !=
2748             (uint32_t)clconf_get_nodeid());
2749 }
2750 
2751 /*
2752  * For use only when booted as a cluster node.
2753  * Embed our cluster nodeid into the clientid.
2754  */
2755 static void
2756 embed_nodeid(cid *cidp)
2757 {
2758         int clnodeid;
2759         /*
2760          * Currently, our state tables are small enough that their
2761          * ids will leave enough bits free for the nodeid. If the
2762          * tables become larger, we mustn't overwrite the id.
2763          * Equally, we only have room for so many bits of nodeid, so
2764          * must check that too.
2765          */
2766         ASSERT(cluster_bootflags & CLUSTER_BOOTED);
2767         ASSERT(cidp->impl_id.c_id >> CLUSTER_NODEID_SHIFT == 0);
2768         clnodeid = clconf_get_nodeid();
2769         ASSERT(clnodeid <= CLUSTER_MAX_NODEID);
2770         ASSERT(clnodeid != NODEID_UNKNOWN);
2771         cidp->impl_id.c_id |= (clnodeid << CLUSTER_NODEID_SHIFT);
2772 }
2773 
2774 static uint32_t
2775 state_hash(void *key)
2776 {
2777         stateid_t *ip = (stateid_t *)key;
2778 
2779         return (ip->bits.ident);
2780 }
2781 
2782 static bool_t
2783 state_compare(rfs4_entry_t u_entry, void *key)
2784 {
2785         rfs4_state_t *sp = (rfs4_state_t *)u_entry;
2786         stateid_t *id = (stateid_t *)key;
2787         bool_t rc;
2788 
2789         rc = (sp->rs_stateid.bits.boottime == id->bits.boottime &&
2790             sp->rs_stateid.bits.ident == id->bits.ident);
2791 
2792         return (rc);
2793 }
2794 
2795 static void *
2796 state_mkkey(rfs4_entry_t u_entry)
2797 {
2798         rfs4_state_t *sp = (rfs4_state_t *)u_entry;
2799 
2800         return (&sp->rs_stateid);
2801 }
2802 
2803 static void
2804 rfs4_state_destroy(rfs4_entry_t u_entry)
2805 {
2806         rfs4_state_t *sp = (rfs4_state_t *)u_entry;
2807 
2808         /* remove from openowner list */
2809         rfs4_dbe_lock(sp->rs_owner->ro_dbe);
2810         list_remove(&sp->rs_owner->ro_statelist, sp);
2811         rfs4_dbe_unlock(sp->rs_owner->ro_dbe);
2812 
2813         list_destroy(&sp->rs_lostatelist);
2814 
2815         /* release any share locks for this stateid if it's still open */
2816         if (!sp->rs_closed) {
2817                 rfs4_dbe_lock(sp->rs_dbe);
2818                 (void) rfs4_unshare(sp);
2819                 rfs4_dbe_unlock(sp->rs_dbe);
2820         }
2821 
2822         /* Were done with the file */
2823         rfs4_file_rele(sp->rs_finfo);
2824         sp->rs_finfo = NULL;
2825 
2826         /* And now with the openowner */
2827         rfs4_openowner_rele(sp->rs_owner);
2828         sp->rs_owner = NULL;
2829 }
2830 
2831 static void
2832 rfs4_state_rele_nounlock(rfs4_state_t *sp)
2833 {
2834         rfs4_dbe_rele(sp->rs_dbe);
2835 }
2836 
2837 void
2838 rfs4_state_rele(rfs4_state_t *sp)
2839 {
2840         rw_exit(&sp->rs_finfo->rf_file_rwlock);
2841         rfs4_dbe_rele(sp->rs_dbe);
2842 }
2843 
2844 static uint32_t
2845 deleg_hash(void *key)
2846 {
2847         rfs4_deleg_state_t *dsp = (rfs4_deleg_state_t *)key;
2848 
2849         return (ADDRHASH(dsp->rds_client) ^ ADDRHASH(dsp->rds_finfo));
2850 }
2851 
2852 static bool_t
2853 deleg_compare(rfs4_entry_t u_entry, void *key)
2854 {
2855         rfs4_deleg_state_t *dsp = (rfs4_deleg_state_t *)u_entry;
2856         rfs4_deleg_state_t *kdsp = (rfs4_deleg_state_t *)key;
2857 
2858         return (dsp->rds_client == kdsp->rds_client &&
2859             dsp->rds_finfo == kdsp->rds_finfo);
2860 }
2861 
2862 static void *
2863 deleg_mkkey(rfs4_entry_t u_entry)
2864 {
2865         return (u_entry);
2866 }
2867 
2868 static uint32_t
2869 deleg_state_hash(void *key)
2870 {
2871         stateid_t *ip = (stateid_t *)key;
2872 
2873         return (ip->bits.ident);
2874 }
2875 
2876 static bool_t
2877 deleg_state_compare(rfs4_entry_t u_entry, void *key)
2878 {
2879         rfs4_deleg_state_t *dsp = (rfs4_deleg_state_t *)u_entry;
2880         stateid_t *id = (stateid_t *)key;
2881         bool_t rc;
2882 
2883         if (id->bits.type != DELEGID)
2884                 return (FALSE);
2885 
2886         rc = (dsp->rds_delegid.bits.boottime == id->bits.boottime &&
2887             dsp->rds_delegid.bits.ident == id->bits.ident);
2888 
2889         return (rc);
2890 }
2891 
2892 static void *
2893 deleg_state_mkkey(rfs4_entry_t u_entry)
2894 {
2895         rfs4_deleg_state_t *dsp = (rfs4_deleg_state_t *)u_entry;
2896 
2897         return (&dsp->rds_delegid);
2898 }
2899 
2900 static bool_t
2901 rfs4_deleg_state_expiry(rfs4_entry_t u_entry)
2902 {
2903         rfs4_deleg_state_t *dsp = (rfs4_deleg_state_t *)u_entry;
2904 
2905         if (rfs4_dbe_is_invalid(dsp->rds_dbe))
2906                 return (TRUE);
2907 
2908         if (dsp->rds_dtype == OPEN_DELEGATE_NONE)
2909                 return (TRUE);
2910 
2911         if ((gethrestime_sec() - dsp->rds_client->rc_last_access
2912             > rfs4_lease_time)) {
2913                 rfs4_dbe_invalidate(dsp->rds_dbe);
2914                 return (TRUE);
2915         }
2916 
2917         return (FALSE);
2918 }
2919 
2920 static bool_t
2921 rfs4_deleg_state_create(rfs4_entry_t u_entry, void *argp)
2922 {
2923         rfs4_deleg_state_t *dsp = (rfs4_deleg_state_t *)u_entry;
2924         rfs4_file_t *fp = ((rfs4_deleg_state_t *)argp)->rds_finfo;
2925         rfs4_client_t *cp = ((rfs4_deleg_state_t *)argp)->rds_client;
2926 
2927         rfs4_dbe_hold(fp->rf_dbe);
2928         rfs4_dbe_hold(cp->rc_dbe);
2929 
2930         dsp->rds_delegid = get_stateid(rfs4_dbe_getid(dsp->rds_dbe));
2931         dsp->rds_delegid.bits.type = DELEGID;
2932         dsp->rds_finfo = fp;
2933         dsp->rds_client = cp;
2934         dsp->rds_dtype = OPEN_DELEGATE_NONE;
2935 
2936         dsp->rds_time_granted = gethrestime_sec();   /* observability */
2937         dsp->rds_time_revoked = 0;
2938 
2939         list_link_init(&dsp->rds_node);
2940 
2941         return (TRUE);
2942 }
2943 
2944 static void
2945 rfs4_deleg_state_destroy(rfs4_entry_t u_entry)
2946 {
2947         rfs4_deleg_state_t *dsp = (rfs4_deleg_state_t *)u_entry;
2948 
2949         /* return delegation if necessary */
2950         rfs4_return_deleg(dsp, FALSE);
2951 
2952         /* Were done with the file */
2953         rfs4_file_rele(dsp->rds_finfo);
2954         dsp->rds_finfo = NULL;
2955 
2956         /* And now with the openowner */
2957         rfs4_client_rele(dsp->rds_client);
2958         dsp->rds_client = NULL;
2959 }
2960 
2961 rfs4_deleg_state_t *
2962 rfs4_finddeleg(rfs4_state_t *sp, bool_t *create)
2963 {
2964         rfs4_deleg_state_t ds, *dsp;
2965 
2966         ds.rds_client = sp->rs_owner->ro_client;
2967         ds.rds_finfo = sp->rs_finfo;
2968 
2969         dsp = (rfs4_deleg_state_t *)rfs4_dbsearch(rfs4_deleg_idx, &ds,
2970             create, &ds, RFS4_DBS_VALID);
2971 
2972         return (dsp);
2973 }
2974 
2975 rfs4_deleg_state_t *
2976 rfs4_finddelegstate(stateid_t *id)
2977 {
2978         rfs4_deleg_state_t *dsp;
2979         bool_t create = FALSE;
2980 
2981         dsp = (rfs4_deleg_state_t *)rfs4_dbsearch(rfs4_deleg_state_idx, id,
2982             &create, NULL, RFS4_DBS_VALID);
2983 
2984         return (dsp);
2985 }
2986 
2987 void
2988 rfs4_deleg_state_rele(rfs4_deleg_state_t *dsp)
2989 {
2990         rfs4_dbe_rele(dsp->rds_dbe);
2991 }
2992 
2993 void
2994 rfs4_update_lock_sequence(rfs4_lo_state_t *lsp)
2995 {
2996 
2997         rfs4_dbe_lock(lsp->rls_dbe);
2998 
2999         /*
3000          * If we are skipping sequence id checking, this means that
3001          * this is the first lock request and therefore the sequence
3002          * id does not need to be updated.  This only happens on the
3003          * first lock request for a lockowner
3004          */
3005         if (!lsp->rls_skip_seqid_check)
3006                 lsp->rls_seqid++;
3007 
3008         rfs4_dbe_unlock(lsp->rls_dbe);
3009 }
3010 
3011 void
3012 rfs4_update_lock_resp(rfs4_lo_state_t *lsp, nfs_resop4 *resp)
3013 {
3014 
3015         rfs4_dbe_lock(lsp->rls_dbe);
3016 
3017         rfs4_free_reply(&lsp->rls_reply);
3018 
3019         rfs4_copy_reply(&lsp->rls_reply, resp);
3020 
3021         rfs4_dbe_unlock(lsp->rls_dbe);
3022 }
3023 
3024 void
3025 rfs4_free_opens(rfs4_openowner_t *oo, bool_t invalidate,
3026     bool_t close_of_client)
3027 {
3028         rfs4_state_t *sp;
3029 
3030         rfs4_dbe_lock(oo->ro_dbe);
3031 
3032         for (sp = list_head(&oo->ro_statelist); sp != NULL;
3033             sp = list_next(&oo->ro_statelist, sp)) {
3034                 rfs4_state_close(sp, FALSE, close_of_client, CRED());
3035                 if (invalidate == TRUE)
3036                         rfs4_dbe_invalidate(sp->rs_dbe);
3037         }
3038 
3039         rfs4_dbe_invalidate(oo->ro_dbe);
3040         rfs4_dbe_unlock(oo->ro_dbe);
3041 }
3042 
3043 static uint32_t
3044 state_owner_file_hash(void *key)
3045 {
3046         rfs4_state_t *sp = key;
3047 
3048         return (ADDRHASH(sp->rs_owner) ^ ADDRHASH(sp->rs_finfo));
3049 }
3050 
3051 static bool_t
3052 state_owner_file_compare(rfs4_entry_t u_entry, void *key)
3053 {
3054         rfs4_state_t *sp = (rfs4_state_t *)u_entry;
3055         rfs4_state_t *arg = key;
3056 
3057         if (sp->rs_closed == TRUE)
3058                 return (FALSE);
3059 
3060         return (arg->rs_owner == sp->rs_owner && arg->rs_finfo == sp->rs_finfo);
3061 }
3062 
3063 static void *
3064 state_owner_file_mkkey(rfs4_entry_t u_entry)
3065 {
3066         return (u_entry);
3067 }
3068 
3069 static uint32_t
3070 state_file_hash(void *key)
3071 {
3072         return (ADDRHASH(key));
3073 }
3074 
3075 static bool_t
3076 state_file_compare(rfs4_entry_t u_entry, void *key)
3077 {
3078         rfs4_state_t *sp = (rfs4_state_t *)u_entry;
3079         rfs4_file_t *fp = key;
3080 
3081         if (sp->rs_closed == TRUE)
3082                 return (FALSE);
3083 
3084         return (fp == sp->rs_finfo);
3085 }
3086 
3087 static void *
3088 state_file_mkkey(rfs4_entry_t u_entry)
3089 {
3090         rfs4_state_t *sp = (rfs4_state_t *)u_entry;
3091 
3092         return (sp->rs_finfo);
3093 }
3094 
3095 rfs4_state_t *
3096 rfs4_findstate_by_owner_file(rfs4_openowner_t *oo, rfs4_file_t *fp,
3097         bool_t *create)
3098 {
3099         rfs4_state_t *sp;
3100         rfs4_state_t key;
3101 
3102         key.rs_owner = oo;
3103         key.rs_finfo = fp;
3104 
3105         sp = (rfs4_state_t *)rfs4_dbsearch(rfs4_state_owner_file_idx, &key,
3106             create, &key, RFS4_DBS_VALID);
3107 
3108         return (sp);
3109 }
3110 
3111 /* This returns ANY state struct that refers to this file */
3112 static rfs4_state_t *
3113 rfs4_findstate_by_file(rfs4_file_t *fp)
3114 {
3115         bool_t create = FALSE;
3116 
3117         return ((rfs4_state_t *)rfs4_dbsearch(rfs4_state_file_idx, fp,
3118             &create, fp, RFS4_DBS_VALID));
3119 }
3120 
3121 static bool_t
3122 rfs4_state_expiry(rfs4_entry_t u_entry)
3123 {
3124         rfs4_state_t *sp = (rfs4_state_t *)u_entry;
3125 
3126         if (rfs4_dbe_is_invalid(sp->rs_dbe))
3127                 return (TRUE);
3128 
3129         if (sp->rs_closed == TRUE &&
3130             ((gethrestime_sec() - rfs4_dbe_get_timerele(sp->rs_dbe))
3131             > rfs4_lease_time))
3132                 return (TRUE);
3133 
3134         return ((gethrestime_sec() - sp->rs_owner->ro_client->rc_last_access
3135             > rfs4_lease_time));
3136 }
3137 
3138 static bool_t
3139 rfs4_state_create(rfs4_entry_t u_entry, void *argp)
3140 {
3141         rfs4_state_t *sp = (rfs4_state_t *)u_entry;
3142         rfs4_file_t *fp = ((rfs4_state_t *)argp)->rs_finfo;
3143         rfs4_openowner_t *oo = ((rfs4_state_t *)argp)->rs_owner;
3144 
3145         rfs4_dbe_hold(fp->rf_dbe);
3146         rfs4_dbe_hold(oo->ro_dbe);
3147         sp->rs_stateid = get_stateid(rfs4_dbe_getid(sp->rs_dbe));
3148         sp->rs_stateid.bits.type = OPENID;
3149         sp->rs_owner = oo;
3150         sp->rs_finfo = fp;
3151 
3152         list_create(&sp->rs_lostatelist, sizeof (rfs4_lo_state_t),
3153             offsetof(rfs4_lo_state_t, rls_node));
3154 
3155         /* Insert state on per open owner's list */
3156         rfs4_dbe_lock(oo->ro_dbe);
3157         list_insert_tail(&oo->ro_statelist, sp);
3158         rfs4_dbe_unlock(oo->ro_dbe);
3159 
3160         return (TRUE);
3161 }
3162 
3163 static rfs4_state_t *
3164 rfs4_findstate(stateid_t *id, rfs4_dbsearch_type_t find_invalid, bool_t lock_fp)
3165 {
3166         rfs4_state_t *sp;
3167         bool_t create = FALSE;
3168 
3169         sp = (rfs4_state_t *)rfs4_dbsearch(rfs4_state_idx, id,
3170             &create, NULL, find_invalid);
3171         if (lock_fp == TRUE && sp != NULL)
3172                 rw_enter(&sp->rs_finfo->rf_file_rwlock, RW_READER);
3173 
3174         return (sp);
3175 }
3176 
3177 void
3178 rfs4_state_close(rfs4_state_t *sp, bool_t lock_held, bool_t close_of_client,
3179     cred_t *cr)
3180 {
3181         /* Remove the associated lo_state owners */
3182         if (!lock_held)
3183                 rfs4_dbe_lock(sp->rs_dbe);
3184 
3185         /*
3186          * If refcnt == 0, the dbe is about to be destroyed.
3187          * lock state will be released by the reaper thread.
3188          */
3189 
3190         if (rfs4_dbe_refcnt(sp->rs_dbe) > 0) {
3191                 if (sp->rs_closed == FALSE) {
3192                         rfs4_release_share_lock_state(sp, cr, close_of_client);
3193                         sp->rs_closed = TRUE;
3194                 }
3195         }
3196 
3197         if (!lock_held)
3198                 rfs4_dbe_unlock(sp->rs_dbe);
3199 }
3200 
3201 /*
3202  * Remove all state associated with the given client.
3203  */
3204 void
3205 rfs4_client_state_remove(rfs4_client_t *cp)
3206 {
3207         rfs4_openowner_t *oo;
3208 
3209         rfs4_dbe_lock(cp->rc_dbe);
3210 
3211         for (oo = list_head(&cp->rc_openownerlist); oo != NULL;
3212             oo = list_next(&cp->rc_openownerlist, oo)) {
3213                 rfs4_free_opens(oo, TRUE, TRUE);
3214         }
3215 
3216         rfs4_dbe_unlock(cp->rc_dbe);
3217 }
3218 
3219 void
3220 rfs4_client_close(rfs4_client_t *cp)
3221 {
3222         /* Mark client as going away. */
3223         rfs4_dbe_lock(cp->rc_dbe);
3224         rfs4_dbe_invalidate(cp->rc_dbe);
3225         rfs4_dbe_unlock(cp->rc_dbe);
3226 
3227         rfs4_client_state_remove(cp);
3228 
3229         /* Release the client */
3230         rfs4_client_rele(cp);
3231 }
3232 
3233 nfsstat4
3234 rfs4_check_clientid(clientid4 *cp, int setclid_confirm)
3235 {
3236         cid *cidp = (cid *) cp;
3237 
3238         /*
3239          * If we are booted as a cluster node, check the embedded nodeid.
3240          * If it indicates that this clientid was generated on another node,
3241          * inform the client accordingly.
3242          */
3243         if (cluster_bootflags & CLUSTER_BOOTED && foreign_clientid(cidp))
3244                 return (NFS4ERR_STALE_CLIENTID);
3245 
3246         /*
3247          * If the server start time matches the time provided
3248          * by the client (via the clientid) and this is NOT a
3249          * setclientid_confirm then return EXPIRED.
3250          */
3251         if (!setclid_confirm && cidp->impl_id.start_time == rfs4_start_time)
3252                 return (NFS4ERR_EXPIRED);
3253 
3254         return (NFS4ERR_STALE_CLIENTID);
3255 }
3256 
3257 /*
3258  * This is used when a stateid has not been found amongst the
3259  * current server's state.  Check the stateid to see if it
3260  * was from this server instantiation or not.
3261  */
3262 static nfsstat4
3263 what_stateid_error(stateid_t *id, stateid_type_t type)
3264 {
3265         /* If we are booted as a cluster node, was stateid locally generated? */
3266         if ((cluster_bootflags & CLUSTER_BOOTED) && foreign_stateid(id))
3267                 return (NFS4ERR_STALE_STATEID);
3268 
3269         /* If types don't match then no use checking further */
3270         if (type != id->bits.type)
3271                 return (NFS4ERR_BAD_STATEID);
3272 
3273         /* From a different server instantiation, return STALE */
3274         if (id->bits.boottime != rfs4_start_time)
3275                 return (NFS4ERR_STALE_STATEID);
3276 
3277         /*
3278          * From this server but the state is most likely beyond lease
3279          * timeout: return NFS4ERR_EXPIRED.  However, there is the
3280          * case of a delegation stateid.  For delegations, there is a
3281          * case where the state can be removed without the client's
3282          * knowledge/consent: revocation.  In the case of delegation
3283          * revocation, the delegation state will be removed and will
3284          * not be found.  If the client does something like a
3285          * DELEGRETURN or even a READ/WRITE with a delegatoin stateid
3286          * that has been revoked, the server should return BAD_STATEID
3287          * instead of the more common EXPIRED error.
3288          */
3289         if (id->bits.boottime == rfs4_start_time) {
3290                 if (type == DELEGID)
3291                         return (NFS4ERR_BAD_STATEID);
3292                 else
3293                         return (NFS4ERR_EXPIRED);
3294         }
3295 
3296         return (NFS4ERR_BAD_STATEID);
3297 }
3298 
3299 /*
3300  * Used later on to find the various state structs.  When called from
3301  * rfs4_check_stateid()->rfs4_get_all_state(), no file struct lock is
3302  * taken (it is not needed) and helps on the read/write path with
3303  * respect to performance.
3304  */
3305 static nfsstat4
3306 rfs4_get_state_lockit(stateid4 *stateid, rfs4_state_t **spp,
3307     rfs4_dbsearch_type_t find_invalid, bool_t lock_fp)
3308 {
3309         stateid_t *id = (stateid_t *)stateid;
3310         rfs4_state_t *sp;
3311 
3312         *spp = NULL;
3313 
3314         /* If we are booted as a cluster node, was stateid locally generated? */
3315         if ((cluster_bootflags & CLUSTER_BOOTED) && foreign_stateid(id))
3316                 return (NFS4ERR_STALE_STATEID);
3317 
3318         sp = rfs4_findstate(id, find_invalid, lock_fp);
3319         if (sp == NULL) {
3320                 return (what_stateid_error(id, OPENID));
3321         }
3322 
3323         if (rfs4_lease_expired(sp->rs_owner->ro_client)) {
3324                 if (lock_fp == TRUE)
3325                         rfs4_state_rele(sp);
3326                 else
3327                         rfs4_state_rele_nounlock(sp);
3328                 return (NFS4ERR_EXPIRED);
3329         }
3330 
3331         *spp = sp;
3332 
3333         return (NFS4_OK);
3334 }
3335 
3336 nfsstat4
3337 rfs4_get_state(stateid4 *stateid, rfs4_state_t **spp,
3338     rfs4_dbsearch_type_t find_invalid)
3339 {
3340         return (rfs4_get_state_lockit(stateid, spp, find_invalid, TRUE));
3341 }
3342 
3343 int
3344 rfs4_check_stateid_seqid(rfs4_state_t *sp, stateid4 *stateid)
3345 {
3346         stateid_t *id = (stateid_t *)stateid;
3347 
3348         if (rfs4_lease_expired(sp->rs_owner->ro_client))
3349                 return (NFS4_CHECK_STATEID_EXPIRED);
3350 
3351         /* Stateid is some time in the future - that's bad */
3352         if (sp->rs_stateid.bits.chgseq < id->bits.chgseq)
3353                 return (NFS4_CHECK_STATEID_BAD);
3354 
3355         if (sp->rs_stateid.bits.chgseq == id->bits.chgseq + 1)
3356                 return (NFS4_CHECK_STATEID_REPLAY);
3357 
3358         /* Stateid is some time in the past - that's old */
3359         if (sp->rs_stateid.bits.chgseq > id->bits.chgseq)
3360                 return (NFS4_CHECK_STATEID_OLD);
3361 
3362         /* Caller needs to know about confirmation before closure */
3363         if (sp->rs_owner->ro_need_confirm)
3364                 return (NFS4_CHECK_STATEID_UNCONFIRMED);
3365 
3366         if (sp->rs_closed == TRUE)
3367                 return (NFS4_CHECK_STATEID_CLOSED);
3368 
3369         return (NFS4_CHECK_STATEID_OKAY);
3370 }
3371 
3372 int
3373 rfs4_check_lo_stateid_seqid(rfs4_lo_state_t *lsp, stateid4 *stateid)
3374 {
3375         stateid_t *id = (stateid_t *)stateid;
3376 
3377         if (rfs4_lease_expired(lsp->rls_state->rs_owner->ro_client))
3378                 return (NFS4_CHECK_STATEID_EXPIRED);
3379 
3380         /* Stateid is some time in the future - that's bad */
3381         if (lsp->rls_lockid.bits.chgseq < id->bits.chgseq)
3382                 return (NFS4_CHECK_STATEID_BAD);
3383 
3384         if (lsp->rls_lockid.bits.chgseq == id->bits.chgseq + 1)
3385                 return (NFS4_CHECK_STATEID_REPLAY);
3386 
3387         /* Stateid is some time in the past - that's old */
3388         if (lsp->rls_lockid.bits.chgseq > id->bits.chgseq)
3389                 return (NFS4_CHECK_STATEID_OLD);
3390 
3391         if (lsp->rls_state->rs_closed == TRUE)
3392                 return (NFS4_CHECK_STATEID_CLOSED);
3393 
3394         return (NFS4_CHECK_STATEID_OKAY);
3395 }
3396 
3397 nfsstat4
3398 rfs4_get_deleg_state(stateid4 *stateid, rfs4_deleg_state_t **dspp)
3399 {
3400         stateid_t *id = (stateid_t *)stateid;
3401         rfs4_deleg_state_t *dsp;
3402 
3403         *dspp = NULL;
3404 
3405         /* If we are booted as a cluster node, was stateid locally generated? */
3406         if ((cluster_bootflags & CLUSTER_BOOTED) && foreign_stateid(id))
3407                 return (NFS4ERR_STALE_STATEID);
3408 
3409         dsp = rfs4_finddelegstate(id);
3410         if (dsp == NULL) {
3411                 return (what_stateid_error(id, DELEGID));
3412         }
3413 
3414         if (rfs4_lease_expired(dsp->rds_client)) {
3415                 rfs4_deleg_state_rele(dsp);
3416                 return (NFS4ERR_EXPIRED);
3417         }
3418 
3419         *dspp = dsp;
3420 
3421         return (NFS4_OK);
3422 }
3423 
3424 nfsstat4
3425 rfs4_get_lo_state(stateid4 *stateid, rfs4_lo_state_t **lspp, bool_t lock_fp)
3426 {
3427         stateid_t *id = (stateid_t *)stateid;
3428         rfs4_lo_state_t *lsp;
3429 
3430         *lspp = NULL;
3431 
3432         /* If we are booted as a cluster node, was stateid locally generated? */
3433         if ((cluster_bootflags & CLUSTER_BOOTED) && foreign_stateid(id))
3434                 return (NFS4ERR_STALE_STATEID);
3435 
3436         lsp = rfs4_findlo_state(id, lock_fp);
3437         if (lsp == NULL) {
3438                 return (what_stateid_error(id, LOCKID));
3439         }
3440 
3441         if (rfs4_lease_expired(lsp->rls_state->rs_owner->ro_client)) {
3442                 rfs4_lo_state_rele(lsp, lock_fp);
3443                 return (NFS4ERR_EXPIRED);
3444         }
3445 
3446         *lspp = lsp;
3447 
3448         return (NFS4_OK);
3449 }
3450 
3451 static nfsstat4
3452 rfs4_get_all_state(stateid4 *sid, rfs4_state_t **spp,
3453     rfs4_deleg_state_t **dspp, rfs4_lo_state_t **lspp)
3454 {
3455         rfs4_state_t *sp = NULL;
3456         rfs4_deleg_state_t *dsp = NULL;
3457         rfs4_lo_state_t *lsp = NULL;
3458         stateid_t *id;
3459         nfsstat4 status;
3460 
3461         *spp = NULL; *dspp = NULL; *lspp = NULL;
3462 
3463         id = (stateid_t *)sid;
3464         switch (id->bits.type) {
3465         case OPENID:
3466                 status = rfs4_get_state_lockit(sid, &sp, FALSE, FALSE);
3467                 break;
3468         case DELEGID:
3469                 status = rfs4_get_deleg_state(sid, &dsp);
3470                 break;
3471         case LOCKID:
3472                 status = rfs4_get_lo_state(sid, &lsp, FALSE);
3473                 if (status == NFS4_OK) {
3474                         sp = lsp->rls_state;
3475                         rfs4_dbe_hold(sp->rs_dbe);
3476                 }
3477                 break;
3478         default:
3479                 status = NFS4ERR_BAD_STATEID;
3480         }
3481 
3482         if (status == NFS4_OK) {
3483                 *spp = sp;
3484                 *dspp = dsp;
3485                 *lspp = lsp;
3486         }
3487 
3488         return (status);
3489 }
3490 
3491 /*
3492  * Given the I/O mode (FREAD or FWRITE), this checks whether the
3493  * rfs4_state_t struct has access to do this operation and if so
3494  * return NFS4_OK; otherwise the proper NFSv4 error is returned.
3495  */
3496 nfsstat4
3497 rfs4_state_has_access(rfs4_state_t *sp, int mode, vnode_t *vp)
3498 {
3499         nfsstat4 stat = NFS4_OK;
3500         rfs4_file_t *fp;
3501         bool_t create = FALSE;
3502 
3503         rfs4_dbe_lock(sp->rs_dbe);
3504         if (mode == FWRITE) {
3505                 if (!(sp->rs_share_access & OPEN4_SHARE_ACCESS_WRITE)) {
3506                         stat = NFS4ERR_OPENMODE;
3507                 }
3508         } else if (mode == FREAD) {
3509                 if (!(sp->rs_share_access & OPEN4_SHARE_ACCESS_READ)) {
3510                         /*
3511                          * If we have OPENed the file with DENYing access
3512                          * to both READ and WRITE then no one else could
3513                          * have OPENed the file, hence no conflicting READ
3514                          * deny.  This check is merely an optimization.
3515                          */
3516                         if (sp->rs_share_deny == OPEN4_SHARE_DENY_BOTH)
3517                                 goto out;
3518 
3519                         /* Check against file struct's DENY mode */
3520                         fp = rfs4_findfile(vp, NULL, &create);
3521                         if (fp != NULL) {
3522                                 int deny_read = 0;
3523                                 rfs4_dbe_lock(fp->rf_dbe);
3524                                 /*
3525                                  * Check if any other open owner has the file
3526                                  * OPENed with deny READ.
3527                                  */
3528                                 if (sp->rs_share_deny & OPEN4_SHARE_DENY_READ)
3529                                         deny_read = 1;
3530                                 ASSERT(fp->rf_deny_read >= deny_read);
3531                                 if (fp->rf_deny_read > deny_read)
3532                                         stat = NFS4ERR_OPENMODE;
3533                                 rfs4_dbe_unlock(fp->rf_dbe);
3534                                 rfs4_file_rele(fp);
3535                         }
3536                 }
3537         } else {
3538                 /* Illegal I/O mode */
3539                 stat = NFS4ERR_INVAL;
3540         }
3541 out:
3542         rfs4_dbe_unlock(sp->rs_dbe);
3543         return (stat);
3544 }
3545 
3546 /*
3547  * Given the I/O mode (FREAD or FWRITE), the vnode, the stateid and whether
3548  * the file is being truncated, return NFS4_OK if allowed or appropriate
3549  * V4 error if not. Note NFS4ERR_DELAY will be returned and a recall on
3550  * the associated file will be done if the I/O is not consistent with any
3551  * delegation in effect on the file. Should be holding VOP_RWLOCK, either
3552  * as reader or writer as appropriate. rfs4_op_open will acquire the
3553  * VOP_RWLOCK as writer when setting up delegation. If the stateid is bad
3554  * this routine will return NFS4ERR_BAD_STATEID. In addition, through the
3555  * deleg parameter, we will return whether a write delegation is held by
3556  * the client associated with this stateid.
3557  * If the server instance associated with the relevant client is in its
3558  * grace period, return NFS4ERR_GRACE.
3559  */
3560 
3561 nfsstat4
3562 rfs4_check_stateid(int mode, vnode_t *vp,
3563     stateid4 *stateid, bool_t trunc, bool_t *deleg,
3564     bool_t do_access, caller_context_t *ct)
3565 {
3566         rfs4_file_t *fp;
3567         bool_t create = FALSE;
3568         rfs4_state_t *sp;
3569         rfs4_deleg_state_t *dsp;
3570         rfs4_lo_state_t *lsp;
3571         stateid_t *id = (stateid_t *)stateid;
3572         nfsstat4 stat = NFS4_OK;
3573 
3574         if (ct != NULL) {
3575                 ct->cc_sysid = 0;
3576                 ct->cc_pid = 0;
3577                 ct->cc_caller_id = nfs4_srv_caller_id;
3578                 ct->cc_flags = CC_DONTBLOCK;
3579         }
3580 
3581         if (ISSPECIAL(stateid)) {
3582                 fp = rfs4_findfile(vp, NULL, &create);
3583                 if (fp == NULL)
3584                         return (NFS4_OK);
3585                 if (fp->rf_dinfo.rd_dtype == OPEN_DELEGATE_NONE) {
3586                         rfs4_file_rele(fp);
3587                         return (NFS4_OK);
3588                 }
3589                 if (mode == FWRITE ||
3590                     fp->rf_dinfo.rd_dtype == OPEN_DELEGATE_WRITE) {
3591                         rfs4_recall_deleg(fp, trunc, NULL);
3592                         rfs4_file_rele(fp);
3593                         return (NFS4ERR_DELAY);
3594                 }
3595                 rfs4_file_rele(fp);
3596                 return (NFS4_OK);
3597         } else {
3598                 stat = rfs4_get_all_state(stateid, &sp, &dsp, &lsp);
3599                 if (stat != NFS4_OK)
3600                         return (stat);
3601                 if (lsp != NULL) {
3602                         /* Is associated server instance in its grace period? */
3603                         if (rfs4_clnt_in_grace(lsp->rls_locker->rl_client)) {
3604                                 rfs4_lo_state_rele(lsp, FALSE);
3605                                 if (sp != NULL)
3606                                         rfs4_state_rele_nounlock(sp);
3607                                 return (NFS4ERR_GRACE);
3608                         }
3609                         if (id->bits.type == LOCKID) {
3610                                 /* Seqid in the future? - that's bad */
3611                                 if (lsp->rls_lockid.bits.chgseq <
3612                                     id->bits.chgseq) {
3613                                         rfs4_lo_state_rele(lsp, FALSE);
3614                                         if (sp != NULL)
3615                                                 rfs4_state_rele_nounlock(sp);
3616                                         return (NFS4ERR_BAD_STATEID);
3617                                 }
3618                                 /* Seqid in the past? - that's old */
3619                                 if (lsp->rls_lockid.bits.chgseq >
3620                                     id->bits.chgseq) {
3621                                         rfs4_lo_state_rele(lsp, FALSE);
3622                                         if (sp != NULL)
3623                                                 rfs4_state_rele_nounlock(sp);
3624                                         return (NFS4ERR_OLD_STATEID);
3625                                 }
3626                                 /* Ensure specified filehandle matches */
3627                                 if (lsp->rls_state->rs_finfo->rf_vp != vp) {
3628                                         rfs4_lo_state_rele(lsp, FALSE);
3629                                         if (sp != NULL)
3630                                                 rfs4_state_rele_nounlock(sp);
3631                                         return (NFS4ERR_BAD_STATEID);
3632                                 }
3633                         }
3634                         if (ct != NULL) {
3635                                 ct->cc_sysid =
3636                                     lsp->rls_locker->rl_client->rc_sysidt;
3637                                 ct->cc_pid = lsp->rls_locker->rl_pid;
3638                         }
3639                         rfs4_lo_state_rele(lsp, FALSE);
3640                 }
3641 
3642                 /* Stateid provided was an "open" stateid */
3643                 if (sp != NULL) {
3644                         /* Is associated server instance in its grace period? */
3645                         if (rfs4_clnt_in_grace(sp->rs_owner->ro_client)) {
3646                                 rfs4_state_rele_nounlock(sp);
3647                                 return (NFS4ERR_GRACE);
3648                         }
3649                         if (id->bits.type == OPENID) {
3650                                 /* Seqid in the future? - that's bad */
3651                                 if (sp->rs_stateid.bits.chgseq <
3652                                     id->bits.chgseq) {
3653                                         rfs4_state_rele_nounlock(sp);
3654                                         return (NFS4ERR_BAD_STATEID);
3655                                 }
3656                                 /* Seqid in the past - that's old */
3657                                 if (sp->rs_stateid.bits.chgseq >
3658                                     id->bits.chgseq) {
3659                                         rfs4_state_rele_nounlock(sp);
3660                                         return (NFS4ERR_OLD_STATEID);
3661                                 }
3662                         }
3663                         /* Ensure specified filehandle matches */
3664                         if (sp->rs_finfo->rf_vp != vp) {
3665                                 rfs4_state_rele_nounlock(sp);
3666                                 return (NFS4ERR_BAD_STATEID);
3667                         }
3668 
3669                         if (sp->rs_owner->ro_need_confirm) {
3670                                 rfs4_state_rele_nounlock(sp);
3671                                 return (NFS4ERR_BAD_STATEID);
3672                         }
3673 
3674                         if (sp->rs_closed == TRUE) {
3675                                 rfs4_state_rele_nounlock(sp);
3676                                 return (NFS4ERR_OLD_STATEID);
3677                         }
3678 
3679                         if (do_access)
3680                                 stat = rfs4_state_has_access(sp, mode, vp);
3681                         else
3682                                 stat = NFS4_OK;
3683 
3684                         /*
3685                          * Return whether this state has write
3686                          * delegation if desired
3687                          */
3688                         if (deleg && (sp->rs_finfo->rf_dinfo.rd_dtype ==
3689                             OPEN_DELEGATE_WRITE))
3690                                 *deleg = TRUE;
3691 
3692                         /*
3693                          * We got a valid stateid, so we update the
3694                          * lease on the client. Ideally we would like
3695                          * to do this after the calling op succeeds,
3696                          * but for now this will be good
3697                          * enough. Callers of this routine are
3698                          * currently insulated from the state stuff.
3699                          */
3700                         rfs4_update_lease(sp->rs_owner->ro_client);
3701 
3702                         /*
3703                          * If a delegation is present on this file and
3704                          * this is a WRITE, then update the lastwrite
3705                          * time to indicate that activity is present.
3706                          */
3707                         if (sp->rs_finfo->rf_dinfo.rd_dtype ==
3708                             OPEN_DELEGATE_WRITE &&
3709                             mode == FWRITE) {
3710                                 sp->rs_finfo->rf_dinfo.rd_time_lastwrite =
3711                                     gethrestime_sec();
3712                         }
3713 
3714                         rfs4_state_rele_nounlock(sp);
3715 
3716                         return (stat);
3717                 }
3718 
3719                 if (dsp != NULL) {
3720                         /* Is associated server instance in its grace period? */
3721                         if (rfs4_clnt_in_grace(dsp->rds_client)) {
3722                                 rfs4_deleg_state_rele(dsp);
3723                                 return (NFS4ERR_GRACE);
3724                         }
3725                         if (dsp->rds_delegid.bits.chgseq != id->bits.chgseq) {
3726                                 rfs4_deleg_state_rele(dsp);
3727                                 return (NFS4ERR_BAD_STATEID);
3728                         }
3729 
3730                         /* Ensure specified filehandle matches */
3731                         if (dsp->rds_finfo->rf_vp != vp) {
3732                                 rfs4_deleg_state_rele(dsp);
3733                                 return (NFS4ERR_BAD_STATEID);
3734                         }
3735                         /*
3736                          * Return whether this state has write
3737                          * delegation if desired
3738                          */
3739                         if (deleg && (dsp->rds_finfo->rf_dinfo.rd_dtype ==
3740                             OPEN_DELEGATE_WRITE))
3741                                 *deleg = TRUE;
3742 
3743                         rfs4_update_lease(dsp->rds_client);
3744 
3745                         /*
3746                          * If a delegation is present on this file and
3747                          * this is a WRITE, then update the lastwrite
3748                          * time to indicate that activity is present.
3749                          */
3750                         if (dsp->rds_finfo->rf_dinfo.rd_dtype ==
3751                             OPEN_DELEGATE_WRITE && mode == FWRITE) {
3752                                 dsp->rds_finfo->rf_dinfo.rd_time_lastwrite =
3753                                     gethrestime_sec();
3754                         }
3755 
3756                         /*
3757                          * XXX - what happens if this is a WRITE and the
3758                          * delegation type of for READ.
3759                          */
3760                         rfs4_deleg_state_rele(dsp);
3761 
3762                         return (stat);
3763                 }
3764                 /*
3765                  * If we got this far, something bad happened
3766                  */
3767                 return (NFS4ERR_BAD_STATEID);
3768         }
3769 }
3770 
3771 
3772 /*
3773  * This is a special function in that for the file struct provided the
3774  * server wants to remove/close all current state associated with the
3775  * file.  The prime use of this would be with OP_REMOVE to force the
3776  * release of state and particularly of file locks.
3777  *
3778  * There is an assumption that there is no delegations outstanding on
3779  * this file at this point.  The caller should have waited for those
3780  * to be returned or revoked.
3781  */
3782 void
3783 rfs4_close_all_state(rfs4_file_t *fp)
3784 {
3785         rfs4_state_t *sp;
3786 
3787         rfs4_dbe_lock(fp->rf_dbe);
3788 
3789 #ifdef DEBUG
3790         /* only applies when server is handing out delegations */
3791         if (rfs4_deleg_policy != SRV_NEVER_DELEGATE)
3792                 ASSERT(fp->rf_dinfo.rd_hold_grant > 0);
3793 #endif
3794 
3795         /* No delegations for this file */
3796         ASSERT(list_is_empty(&fp->rf_delegstatelist));
3797 
3798         /* Make sure that it can not be found */
3799         rfs4_dbe_invalidate(fp->rf_dbe);
3800 
3801         if (fp->rf_vp == NULL) {
3802                 rfs4_dbe_unlock(fp->rf_dbe);
3803                 return;
3804         }
3805         rfs4_dbe_unlock(fp->rf_dbe);
3806 
3807         /*
3808          * Hold as writer to prevent other server threads from
3809          * processing requests related to the file while all state is
3810          * being removed.
3811          */
3812         rw_enter(&fp->rf_file_rwlock, RW_WRITER);
3813 
3814         /* Remove ALL state from the file */
3815         while (sp = rfs4_findstate_by_file(fp)) {
3816                 rfs4_state_close(sp, FALSE, FALSE, CRED());
3817                 rfs4_state_rele_nounlock(sp);
3818         }
3819 
3820         /*
3821          * This is only safe since there are no further references to
3822          * the file.
3823          */
3824         rfs4_dbe_lock(fp->rf_dbe);
3825         if (fp->rf_vp) {
3826                 vnode_t *vp = fp->rf_vp;
3827 
3828                 mutex_enter(&vp->v_vsd_lock);
3829                 (void) vsd_set(vp, nfs4_srv_vkey, NULL);
3830                 mutex_exit(&vp->v_vsd_lock);
3831                 VN_RELE(vp);
3832                 fp->rf_vp = NULL;
3833         }
3834         rfs4_dbe_unlock(fp->rf_dbe);
3835 
3836         /* Finally let other references to proceed */
3837         rw_exit(&fp->rf_file_rwlock);
3838 }
3839 
3840 /*
3841  * This function is used as a target for the rfs4_dbe_walk() call
3842  * below.  The purpose of this function is to see if the
3843  * lockowner_state refers to a file that resides within the exportinfo
3844  * export.  If so, then remove the lock_owner state (file locks and
3845  * share "locks") for this object since the intent is the server is
3846  * unexporting the specified directory.  Be sure to invalidate the
3847  * object after the state has been released
3848  */
3849 static void
3850 rfs4_lo_state_walk_callout(rfs4_entry_t u_entry, void *e)
3851 {
3852         rfs4_lo_state_t *lsp = (rfs4_lo_state_t *)u_entry;
3853         struct exportinfo *exi = (struct exportinfo *)e;
3854         nfs_fh4_fmt_t   fhfmt4, *exi_fhp, *finfo_fhp;
3855         fhandle_t *efhp;
3856 
3857         efhp = (fhandle_t *)&exi->exi_fh;
3858         exi_fhp = (nfs_fh4_fmt_t *)&fhfmt4;
3859 
3860         FH_TO_FMT4(efhp, exi_fhp);
3861 
3862         finfo_fhp = (nfs_fh4_fmt_t *)lsp->rls_state->rs_finfo->
3863             rf_filehandle.nfs_fh4_val;
3864 
3865         if (EQFSID(&finfo_fhp->fh4_fsid, &exi_fhp->fh4_fsid) &&
3866             bcmp(&finfo_fhp->fh4_xdata, &exi_fhp->fh4_xdata,
3867             exi_fhp->fh4_xlen) == 0) {
3868                 rfs4_state_close(lsp->rls_state, FALSE, FALSE, CRED());
3869                 rfs4_dbe_invalidate(lsp->rls_dbe);
3870                 rfs4_dbe_invalidate(lsp->rls_state->rs_dbe);
3871         }
3872 }
3873 
3874 /*
3875  * This function is used as a target for the rfs4_dbe_walk() call
3876  * below.  The purpose of this function is to see if the state refers
3877  * to a file that resides within the exportinfo export.  If so, then
3878  * remove the open state for this object since the intent is the
3879  * server is unexporting the specified directory.  The main result for
3880  * this type of entry is to invalidate it such it will not be found in
3881  * the future.
3882  */
3883 static void
3884 rfs4_state_walk_callout(rfs4_entry_t u_entry, void *e)
3885 {
3886         rfs4_state_t *sp = (rfs4_state_t *)u_entry;
3887         struct exportinfo *exi = (struct exportinfo *)e;
3888         nfs_fh4_fmt_t   fhfmt4, *exi_fhp, *finfo_fhp;
3889         fhandle_t *efhp;
3890 
3891         efhp = (fhandle_t *)&exi->exi_fh;
3892         exi_fhp = (nfs_fh4_fmt_t *)&fhfmt4;
3893 
3894         FH_TO_FMT4(efhp, exi_fhp);
3895 
3896         finfo_fhp =
3897             (nfs_fh4_fmt_t *)sp->rs_finfo->rf_filehandle.nfs_fh4_val;
3898 
3899         if (EQFSID(&finfo_fhp->fh4_fsid, &exi_fhp->fh4_fsid) &&
3900             bcmp(&finfo_fhp->fh4_xdata, &exi_fhp->fh4_xdata,
3901             exi_fhp->fh4_xlen) == 0) {
3902                 rfs4_state_close(sp, TRUE, FALSE, CRED());
3903                 rfs4_dbe_invalidate(sp->rs_dbe);
3904         }
3905 }
3906 
3907 /*
3908  * This function is used as a target for the rfs4_dbe_walk() call
3909  * below.  The purpose of this function is to see if the state refers
3910  * to a file that resides within the exportinfo export.  If so, then
3911  * remove the deleg state for this object since the intent is the
3912  * server is unexporting the specified directory.  The main result for
3913  * this type of entry is to invalidate it such it will not be found in
3914  * the future.
3915  */
3916 static void
3917 rfs4_deleg_state_walk_callout(rfs4_entry_t u_entry, void *e)
3918 {
3919         rfs4_deleg_state_t *dsp = (rfs4_deleg_state_t *)u_entry;
3920         struct exportinfo *exi = (struct exportinfo *)e;
3921         nfs_fh4_fmt_t   fhfmt4, *exi_fhp, *finfo_fhp;
3922         fhandle_t *efhp;
3923 
3924         efhp = (fhandle_t *)&exi->exi_fh;
3925         exi_fhp = (nfs_fh4_fmt_t *)&fhfmt4;
3926 
3927         FH_TO_FMT4(efhp, exi_fhp);
3928 
3929         finfo_fhp =
3930             (nfs_fh4_fmt_t *)dsp->rds_finfo->rf_filehandle.nfs_fh4_val;
3931 
3932         if (EQFSID(&finfo_fhp->fh4_fsid, &exi_fhp->fh4_fsid) &&
3933             bcmp(&finfo_fhp->fh4_xdata, &exi_fhp->fh4_xdata,
3934             exi_fhp->fh4_xlen) == 0) {
3935                 rfs4_dbe_invalidate(dsp->rds_dbe);
3936         }
3937 }
3938 
3939 /*
3940  * This function is used as a target for the rfs4_dbe_walk() call
3941  * below.  The purpose of this function is to see if the state refers
3942  * to a file that resides within the exportinfo export.  If so, then
3943  * release vnode hold for this object since the intent is the server
3944  * is unexporting the specified directory.  Invalidation will prevent
3945  * this struct from being found in the future.
3946  */
3947 static void
3948 rfs4_file_walk_callout(rfs4_entry_t u_entry, void *e)
3949 {
3950         rfs4_file_t *fp = (rfs4_file_t *)u_entry;
3951         struct exportinfo *exi = (struct exportinfo *)e;
3952         nfs_fh4_fmt_t   fhfmt4, *exi_fhp, *finfo_fhp;
3953         fhandle_t *efhp;
3954 
3955         efhp = (fhandle_t *)&exi->exi_fh;
3956         exi_fhp = (nfs_fh4_fmt_t *)&fhfmt4;
3957 
3958         FH_TO_FMT4(efhp, exi_fhp);
3959 
3960         finfo_fhp = (nfs_fh4_fmt_t *)fp->rf_filehandle.nfs_fh4_val;
3961 
3962         if (EQFSID(&finfo_fhp->fh4_fsid, &exi_fhp->fh4_fsid) &&
3963             bcmp(&finfo_fhp->fh4_xdata, &exi_fhp->fh4_xdata,
3964             exi_fhp->fh4_xlen) == 0) {
3965                 if (fp->rf_vp) {
3966                         vnode_t *vp = fp->rf_vp;
3967 
3968                         /*
3969                          * don't leak monitors and remove the reference
3970                          * put on the vnode when the delegation was granted.
3971                          */
3972                         if (fp->rf_dinfo.rd_dtype == OPEN_DELEGATE_READ) {
3973                                 (void) fem_uninstall(vp, deleg_rdops,
3974                                     (void *)fp);
3975                                 vn_open_downgrade(vp, FREAD);
3976                         } else if (fp->rf_dinfo.rd_dtype ==
3977                             OPEN_DELEGATE_WRITE) {
3978                                 (void) fem_uninstall(vp, deleg_wrops,
3979                                     (void *)fp);
3980                                 vn_open_downgrade(vp, FREAD|FWRITE);
3981                         }
3982                         mutex_enter(&vp->v_vsd_lock);
3983                         (void) vsd_set(vp, nfs4_srv_vkey, NULL);
3984                         mutex_exit(&vp->v_vsd_lock);
3985                         VN_RELE(vp);
3986                         fp->rf_vp = NULL;
3987                 }
3988                 rfs4_dbe_invalidate(fp->rf_dbe);
3989         }
3990 }
3991 
3992 /*
3993  * Given a directory that is being unexported, cleanup/release all
3994  * state in the server that refers to objects residing underneath this
3995  * particular export.  The ordering of the release is important.
3996  * Lock_owner, then state and then file.
3997  */
3998 void
3999 rfs4_clean_state_exi(struct exportinfo *exi)
4000 {
4001         mutex_enter(&rfs4_state_lock);
4002 
4003         if (rfs4_server_state == NULL) {
4004                 mutex_exit(&rfs4_state_lock);
4005                 return;
4006         }
4007 
4008         rfs4_dbe_walk(rfs4_lo_state_tab, rfs4_lo_state_walk_callout, exi);
4009         rfs4_dbe_walk(rfs4_state_tab, rfs4_state_walk_callout, exi);
4010         rfs4_dbe_walk(rfs4_deleg_state_tab, rfs4_deleg_state_walk_callout, exi);
4011         rfs4_dbe_walk(rfs4_file_tab, rfs4_file_walk_callout, exi);
4012 
4013         mutex_exit(&rfs4_state_lock);
4014 }