From: Trond Myklebust NFSv4: Convert the lease renewal daemon from being per-mountpoint to being per-server. Instead of running it on top of rpciod, convert it to use keventd. This mean we can use the struct nfs4_client semaphores for ordering purposes. --- fs/nfs/inode.c | 17 +++++-- fs/nfs/nfs4proc.c | 41 +++++++++-------- fs/nfs/nfs4renewd.c | 110 ++++++++++++++++++++++++++++++---------------- fs/nfs/nfs4state.c | 26 ++++++++++ include/linux/nfs_fs.h | 35 ++++++-------- include/linux/nfs_fs_sb.h | 5 +- 6 files changed, 155 insertions(+), 79 deletions(-) diff -puN fs/nfs/inode.c~nfs-19-renewd fs/nfs/inode.c --- 25/fs/nfs/inode.c~nfs-19-renewd 2004-01-14 02:09:49.000000000 -0800 +++ 25-akpm/fs/nfs/inode.c 2004-01-14 02:09:49.000000000 -0800 @@ -163,6 +163,8 @@ nfs_put_super(struct super_block *sb) nfs_idmap_delete(server); #endif /* CONFIG_NFS_V4 */ + nfs4_renewd_prepare_shutdown(server); + if (server->client != NULL) rpc_shutdown_client(server->client); if (server->client_sys != NULL) @@ -1281,6 +1283,8 @@ static struct super_block *nfs_get_sb(st if (!server) return ERR_PTR(-ENOMEM); memset(server, 0, sizeof(struct nfs_server)); + /* Zero out the NFS state stuff */ + init_nfsv4_state(server); root = &server->fh; memcpy(root, &data->root, sizeof(*root)); @@ -1444,13 +1448,15 @@ static int nfs4_fill_super(struct super_ clp->cl_cred = rpcauth_lookupcred(clnt->cl_auth, 0); memcpy(clp->cl_ipaddr, server->ip_addr, sizeof(clp->cl_ipaddr)); } + list_add_tail(&server->nfs4_siblings, &clp->cl_superblocks); clnt = rpc_clone_client(clp->cl_rpcclient); server->nfs4_state = clp; up_write(&clp->cl_sem); + clp = NULL; if (clnt == NULL) { printk(KERN_WARNING "NFS: cannot create RPC client.\n"); - goto out_fail; + goto out_remove_list; } clnt->cl_intr = (server->flags & NFS4_MOUNT_INTR) ? 1 : 0; @@ -1476,13 +1482,16 @@ static int nfs4_fill_super(struct super_ err = nfs_sb_init(sb, authflavour); if (err == 0) return 0; - clp = NULL; rpciod_down(); - destroy_nfsv4_state(server); if (server->idmap != NULL) nfs_idmap_delete(server); out_shutdown: rpc_shutdown_client(server->client); +out_remove_list: + down_write(&server->nfs4_state->cl_sem); + list_del_init(&server->nfs4_siblings); + up_write(&server->nfs4_state->cl_sem); + destroy_nfsv4_state(server); out_fail: if (clp) nfs4_put_client(clp); @@ -1542,6 +1551,8 @@ static struct super_block *nfs4_get_sb(s if (!server) return ERR_PTR(-ENOMEM); memset(server, 0, sizeof(struct nfs_server)); + /* Zero out the NFS state stuff */ + init_nfsv4_state(server); if (data->version != NFS4_MOUNT_VERSION) { printk("nfs warning: mount version %s than kernel\n", diff -puN fs/nfs/nfs4proc.c~nfs-19-renewd fs/nfs/nfs4proc.c --- 25/fs/nfs/nfs4proc.c~nfs-19-renewd 2004-01-14 02:09:49.000000000 -0800 +++ 25-akpm/fs/nfs/nfs4proc.c 2004-01-14 02:09:49.000000000 -0800 @@ -56,8 +56,6 @@ extern struct rpc_procinfo nfs4_procedur extern nfs4_stateid zero_stateid; -static spinlock_t renew_lock = SPIN_LOCK_UNLOCKED; - static void nfs4_setup_compound(struct nfs4_compound *cp, struct nfs4_op *ops, struct nfs_server *server, char *tag) @@ -480,10 +478,11 @@ nfs4_setup_setclientid_confirm(struct nf static void renew_lease(struct nfs_server *server, unsigned long timestamp) { - spin_lock(&renew_lock); - if (time_before(server->last_renewal,timestamp)) - server->last_renewal = timestamp; - spin_unlock(&renew_lock); + struct nfs4_client *clp = server->nfs4_state; + spin_lock(&clp->cl_lock); + if (time_before(clp->cl_last_renewal,timestamp)) + clp->cl_last_renewal = timestamp; + spin_unlock(&clp->cl_lock); } static inline void @@ -748,6 +747,7 @@ nfs4_proc_get_root(struct nfs_server *se struct nfs_fsinfo fsinfo; unsigned char * p; struct qstr q; + unsigned long last_renewed; int status; clp = server->nfs4_state; @@ -773,6 +773,7 @@ nfs4_proc_get_root(struct nfs_server *se */ nfs4_setup_compound(&compound, ops, server, "setclientid"); nfs4_setup_setclientid(&compound, 0, 0); + last_renewed = jiffies; if ((status = nfs4_call_compound(&compound, NULL, 0))) goto out_unlock; @@ -786,20 +787,22 @@ nfs4_proc_get_root(struct nfs_server *se nfs4_setup_putrootfh(&compound); nfs4_setup_getrootattr(&compound, fattr, &fsinfo); nfs4_setup_getfh(&compound, fhandle); + last_renewed = jiffies; if ((status = nfs4_call_compound(&compound, NULL, 0))) goto out_unlock; - clp->cl_state = NFS4CLNT_OK; -no_setclientid: /* * Now that we have instantiated the clientid and determined * the lease time, we can initialize the renew daemon for this * server. * FIXME: we only need one renewd daemon per server. */ - server->lease_time = fsinfo.lease_time * HZ; - if ((status = nfs4_init_renewd(server))) - goto out_unlock; + clp->cl_lease_time = fsinfo.lease_time * HZ; + clp->cl_last_renewal = last_renewed; + nfs4_schedule_state_renewal(clp); + clp->cl_state = NFS4CLNT_OK; + +no_setclientid: up_write(&clp->cl_sem); /* @@ -1642,22 +1645,24 @@ nfs4_proc_commit_setup(struct nfs_write_ static void renew_done(struct rpc_task *task) { - struct nfs_server *server = (struct nfs_server *)task->tk_msg.rpc_resp; + struct nfs4_client *clp = (struct nfs4_client *)task->tk_msg.rpc_argp; unsigned long timestamp = (unsigned long)task->tk_calldata; - renew_lease(server, timestamp); + spin_lock(&clp->cl_lock); + if (time_before(clp->cl_last_renewal,timestamp)) + clp->cl_last_renewal = timestamp; + spin_unlock(&clp->cl_lock); } int -nfs4_proc_async_renew(struct nfs_server *server, struct rpc_cred *cred) +nfs4_proc_async_renew(struct nfs4_client *clp) { struct rpc_message msg = { .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_RENEW], - .rpc_argp = server->nfs4_state, - .rpc_resp = server, - .rpc_cred = cred, + .rpc_argp = clp, + .rpc_cred = clp->cl_cred, }; - return rpc_call_async(server->client, &msg, 0, renew_done, (void *)jiffies); + return rpc_call_async(clp->cl_rpcclient, &msg, 0, renew_done, (void *)jiffies); } /* diff -puN fs/nfs/nfs4renewd.c~nfs-19-renewd fs/nfs/nfs4renewd.c --- 25/fs/nfs/nfs4renewd.c~nfs-19-renewd 2004-01-14 02:09:49.000000000 -0800 +++ 25-akpm/fs/nfs/nfs4renewd.c 2004-01-14 02:09:49.000000000 -0800 @@ -54,53 +54,91 @@ #include #include -static RPC_WAITQ(nfs4_renewd_queue, "nfs4_renewd_queue"); +#define NFSDBG_FACILITY NFSDBG_PROC -static void -renewd(struct rpc_task *task) +void +nfs4_renew_state(void *data) { - struct nfs_server *server = (struct nfs_server *)task->tk_calldata; - unsigned long lease = server->lease_time; - unsigned long last = server->last_renewal; - unsigned long timeout; - - if (!server->nfs4_state) - timeout = (2 * lease) / 3; - else if (jiffies < last + lease/3) - timeout = (2 * lease) / 3 + last - jiffies; - else { + struct nfs4_client *clp = (struct nfs4_client *)data; + long lease, timeout; + unsigned long last, now; + + down_read(&clp->cl_sem); + dprintk("%s: start\n", __FUNCTION__); + /* Are there any active superblocks? */ + if (list_empty(&clp->cl_superblocks)) + goto out; + spin_lock(&clp->cl_lock); + lease = clp->cl_lease_time; + last = clp->cl_last_renewal; + now = jiffies; + timeout = (2 * lease) / 3 + (long)last - (long)now; + /* Are we close to a lease timeout? */ + if (time_after(now, last + lease/3)) { + spin_unlock(&clp->cl_lock); /* Queue an asynchronous RENEW. */ - nfs4_proc_async_renew(server, NULL); + nfs4_proc_async_renew(clp); timeout = (2 * lease) / 3; - } - + spin_lock(&clp->cl_lock); + } else + dprintk("%s: failed to call renewd. Reason: lease not expired \n", + __FUNCTION__); if (timeout < 5 * HZ) /* safeguard */ timeout = 5 * HZ; - task->tk_timeout = timeout; - task->tk_action = renewd; - task->tk_exit = NULL; - rpc_sleep_on(&nfs4_renewd_queue, task, NULL, NULL); - return; + dprintk("%s: requeueing work. Lease period = %ld\n", + __FUNCTION__, (timeout + HZ - 1) / HZ); + cancel_delayed_work(&clp->cl_renewd); + schedule_delayed_work(&clp->cl_renewd, timeout); + spin_unlock(&clp->cl_lock); +out: + up_read(&clp->cl_sem); + dprintk("%s: done\n", __FUNCTION__); +} + +/* Must be called with clp->cl_sem locked for writes */ +void +nfs4_schedule_state_renewal(struct nfs4_client *clp) +{ + long timeout; + + spin_lock(&clp->cl_lock); + timeout = (2 * clp->cl_lease_time) / 3 + (long)clp->cl_last_renewal + - (long)jiffies; + if (timeout < 5 * HZ) + timeout = 5 * HZ; + dprintk("%s: requeueing work. Lease period = %ld\n", + __FUNCTION__, (timeout + HZ - 1) / HZ); + cancel_delayed_work(&clp->cl_renewd); + schedule_delayed_work(&clp->cl_renewd, timeout); + spin_unlock(&clp->cl_lock); } -int -nfs4_init_renewd(struct nfs_server *server) +void +nfs4_renewd_prepare_shutdown(struct nfs_server *server) { - struct rpc_task *task; - int status; + struct nfs4_client *clp = server->nfs4_state; - lock_kernel(); - status = -ENOMEM; - task = rpc_new_task(server->client, NULL, RPC_TASK_ASYNC); - if (!task) - goto out; - task->tk_calldata = server; - task->tk_action = renewd; - status = rpc_execute(task); + if (!clp) + return; + flush_scheduled_work(); + down_write(&clp->cl_sem); + if (!list_empty(&server->nfs4_siblings)) + list_del_init(&server->nfs4_siblings); + up_write(&clp->cl_sem); +} -out: - unlock_kernel(); - return status; +/* Must be called with clp->cl_sem locked for writes */ +void +nfs4_kill_renewd(struct nfs4_client *clp) +{ + down_read(&clp->cl_sem); + if (!list_empty(&clp->cl_superblocks)) { + up_read(&clp->cl_sem); + return; + } + cancel_delayed_work(&clp->cl_renewd); + up_read(&clp->cl_sem); + flush_scheduled_work(); } /* diff -puN fs/nfs/nfs4state.c~nfs-19-renewd fs/nfs/nfs4state.c --- 25/fs/nfs/nfs4state.c~nfs-19-renewd 2004-01-14 02:09:49.000000000 -0800 +++ 25-akpm/fs/nfs/nfs4state.c 2004-01-14 02:09:49.000000000 -0800 @@ -41,6 +41,7 @@ #include #include #include +#include #define OPENOWNER_POOL_SIZE 8 @@ -55,6 +56,28 @@ nfs4_stateid one_stateid = static LIST_HEAD(nfs4_clientid_list); +extern void nfs4_renew_state(void *); + +void +init_nfsv4_state(struct nfs_server *server) +{ + server->nfs4_state = NULL; + INIT_LIST_HEAD(&server->nfs4_siblings); +} + +void +destroy_nfsv4_state(struct nfs_server *server) +{ + if (server->mnt_path) { + kfree(server->mnt_path); + server->mnt_path = NULL; + } + if (server->nfs4_state) { + nfs4_put_client(server->nfs4_state); + server->nfs4_state = NULL; + } +} + /* * nfs4_get_client(): returns an empty client structure * nfs4_put_client(): drops reference to client structure @@ -75,6 +98,8 @@ nfs4_alloc_client(struct in_addr *addr) INIT_LIST_HEAD(&clp->cl_unused); spin_lock_init(&clp->cl_lock); atomic_set(&clp->cl_count, 1); + INIT_WORK(&clp->cl_renewd, nfs4_renew_state, clp); + INIT_LIST_HEAD(&clp->cl_superblocks); clp->cl_state = NFS4CLNT_NEW; } return clp; @@ -130,6 +155,7 @@ nfs4_put_client(struct nfs4_client *clp) return; list_del(&clp->cl_servers); spin_unlock(&state_spinlock); + nfs4_kill_renewd(clp); nfs4_free_client(clp); } diff -puN include/linux/nfs_fs.h~nfs-19-renewd include/linux/nfs_fs.h --- 25/include/linux/nfs_fs.h~nfs-19-renewd 2004-01-14 02:09:49.000000000 -0800 +++ 25-akpm/include/linux/nfs_fs.h 2004-01-14 02:09:49.000000000 -0800 @@ -28,6 +28,7 @@ #include #include #include +#include /* * Enable debugging support for nfs client. @@ -493,6 +494,12 @@ struct nfs4_client { struct rpc_clnt * cl_rpcclient; struct rpc_cred * cl_cred; + struct list_head cl_superblocks; /* List of nfs_server structs */ + + unsigned long cl_lease_time; + unsigned long cl_last_renewal; + struct work_struct cl_renewd; + /* Our own IP address, as a null-terminated string. * This is used to generate the clientid, and the callback address. */ @@ -545,13 +552,17 @@ struct nfs4_state { /* nfs4proc.c */ -extern int nfs4_proc_async_renew(struct nfs_server *server, struct rpc_cred *); +extern int nfs4_proc_async_renew(struct nfs4_client *); extern int nfs4_do_close(struct inode *, struct nfs4_state *); /* nfs4renewd.c */ -extern int nfs4_init_renewd(struct nfs_server *server); +extern void nfs4_schedule_state_renewal(struct nfs4_client *); +extern void nfs4_renewd_prepare_shutdown(struct nfs_server *); +extern void nfs4_kill_renewd(struct nfs4_client *); /* nfs4state.c */ +extern void init_nfsv4_state(struct nfs_server *); +extern void destroy_nfsv4_state(struct nfs_server *); extern struct nfs4_client *nfs4_get_client(struct in_addr *); extern void nfs4_put_client(struct nfs4_client *clp); extern struct nfs4_state_owner * nfs4_get_state_owner(struct nfs_server *, struct rpc_cred *); @@ -560,29 +571,13 @@ extern struct nfs4_state * nfs4_get_open extern void nfs4_put_open_state(struct nfs4_state *); extern void nfs4_increment_seqid(u32 status, struct nfs4_state_owner *sp); - - - - - struct nfs4_mount_data; -static inline void -destroy_nfsv4_state(struct nfs_server *server) -{ - if (server->mnt_path) { - kfree(server->mnt_path); - server->mnt_path = NULL; - } - if (server->nfs4_state) { - nfs4_put_client(server->nfs4_state); - server->nfs4_state = NULL; - } -} #else -#define create_nfsv4_state(server, data) 0 +#define init_nfsv4_state(server) do { } while (0) #define destroy_nfsv4_state(server) do { } while (0) #define nfs4_put_state_owner(inode, owner) do { } while (0) #define nfs4_put_open_state(state) do { } while (0) +#define nfs4_renewd_prepare_shutdown(server) do { } while (0) #endif #endif /* __KERNEL__ */ diff -puN include/linux/nfs_fs_sb.h~nfs-19-renewd include/linux/nfs_fs_sb.h --- 25/include/linux/nfs_fs_sb.h~nfs-19-renewd 2004-01-14 02:09:49.000000000 -0800 +++ 25-akpm/include/linux/nfs_fs_sb.h 2004-01-14 02:09:49.000000000 -0800 @@ -35,8 +35,9 @@ struct nfs_server { char ip_addr[16]; char * mnt_path; struct nfs4_client * nfs4_state; /* all NFSv4 state starts here */ - unsigned long lease_time; /* in jiffies */ - unsigned long last_renewal; /* in jiffies */ + struct list_head nfs4_siblings; /* List of other nfs_server structs + * that share the same clientid + */ void *idmap; #endif }; _