From: Dipankar Sarma fget() shows up on profiles, especially on SMP. Dipankar's patch special-cases the situation wherein there are no sharers of current->files. In this situation we know that no other process can close this file, so it is not necessary to increment the file's refcount. It's ugly as sin, but makes a substantial difference. The test is dd if=/dev/zero of=foo bs=1 count=1M On 4CPU P3 xeon with 1MB L2 cache and 512MB ram: kernel sys time std-dev ------------ -------- ------- UP - vanilla 2.104 0.028 UP - file 1.867 0.019 SMP - vanilla 2.976 0.023 SMP - file 2.719 0.026 25-akpm/fs/file_table.c | 35 ++++++++++++++++++++++++++++++ 25-akpm/fs/read_write.c | 49 +++++++++++++++++++++++++------------------ 25-akpm/include/linux/file.h | 2 + 3 files changed, 66 insertions(+), 20 deletions(-) diff -puN fs/file_table.c~fget-speedup fs/file_table.c --- 25/fs/file_table.c~fget-speedup Fri May 2 13:59:29 2003 +++ 25-akpm/fs/file_table.c Fri May 2 13:59:29 2003 @@ -147,6 +147,13 @@ void fput(struct file * file) __fput(file); } +void fput_light(struct file * file, int fput_needed) +{ + if (unlikely(fput_needed)) + if (atomic_dec_and_test(&file->f_count)) + __fput(file); +} + /* __fput is called from task context when aio completion releases the last * last use of a struct file *. Do not use otherwise. */ @@ -190,6 +197,34 @@ struct file *fget(unsigned int fd) return file; } +/* + * Lightweight file lookup - no refcnt increment if fd table isn't shared. + * You can use this only if it is guranteed that the current task already + * holds a refcnt to that file. That check has to be done at fget() only + * and a flag is returned to be passed to the corresponding fput_light(). + * There must not be a cloning between an fget_light/fput_light pair. + */ +struct file *fget_light(unsigned int fd, int *fput_needed) +{ + struct file *file; + struct files_struct *files = current->files; + + *fput_needed = 0; + if (likely((atomic_read(&files->count) == 1))) { + file = fcheck(fd); + } else { + read_lock(&files->file_lock); + file = fcheck(fd); + if (file) { + get_file(file); + *fput_needed = 1; + } + read_unlock(&files->file_lock); + } + return file; +} + + void put_filp(struct file *file) { if (atomic_dec_and_test(&file->f_count)) { diff -puN fs/read_write.c~fget-speedup fs/read_write.c --- 25/fs/read_write.c~fget-speedup Fri May 2 13:59:29 2003 +++ 25-akpm/fs/read_write.c Fri May 2 13:59:29 2003 @@ -115,9 +115,10 @@ asmlinkage off_t sys_lseek(unsigned int { off_t retval; struct file * file; + int fput_needed; retval = -EBADF; - file = fget(fd); + file = fget_light(fd, &fput_needed); if (!file) goto bad; @@ -128,7 +129,7 @@ asmlinkage off_t sys_lseek(unsigned int if (res != (loff_t)retval) retval = -EOVERFLOW; /* LFS: should only happen on 32 bit platforms */ } - fput(file); + fput_light(file, fput_needed); bad: return retval; } @@ -141,9 +142,10 @@ asmlinkage long sys_llseek(unsigned int int retval; struct file * file; loff_t offset; + int fput_needed; retval = -EBADF; - file = fget(fd); + file = fget_light(fd, &fput_needed); if (!file) goto bad; @@ -161,7 +163,7 @@ asmlinkage long sys_llseek(unsigned int retval = 0; } out_putf: - fput(file); + fput_light(file, fput_needed); bad: return retval; } @@ -251,11 +253,12 @@ asmlinkage ssize_t sys_read(unsigned int { struct file *file; ssize_t ret = -EBADF; + int fput_needed; - file = fget(fd); + file = fget_light(fd, &fput_needed); if (file) { ret = vfs_read(file, buf, count, &file->f_pos); - fput(file); + fput_light(file, fput_needed); } return ret; @@ -265,11 +268,12 @@ asmlinkage ssize_t sys_write(unsigned in { struct file *file; ssize_t ret = -EBADF; + int fput_needed; - file = fget(fd); + file = fget_light(fd, &fput_needed); if (file) { ret = vfs_write(file, buf, count, &file->f_pos); - fput(file); + fput_light(file, fput_needed); } return ret; @@ -280,14 +284,15 @@ asmlinkage ssize_t sys_pread64(unsigned { struct file *file; ssize_t ret = -EBADF; + int fput_needed; if (pos < 0) return -EINVAL; - file = fget(fd); + file = fget_light(fd, &fput_needed); if (file) { ret = vfs_read(file, buf, count, &pos); - fput(file); + fput_light(file, fput_needed); } return ret; @@ -298,14 +303,15 @@ asmlinkage ssize_t sys_pwrite64(unsigned { struct file *file; ssize_t ret = -EBADF; + int fput_needed; if (pos < 0) return -EINVAL; - file = fget(fd); + file = fget_light(fd, &fput_needed); if (file) { ret = vfs_write(file, buf, count, &pos); - fput(file); + fput_light(file, fput_needed); } return ret; @@ -479,11 +485,12 @@ sys_readv(unsigned long fd, const struct { struct file *file; ssize_t ret = -EBADF; + int fput_needed; - file = fget(fd); + file = fget_light(fd, &fput_needed); if (file) { ret = vfs_readv(file, vec, vlen, &file->f_pos); - fput(file); + fput_light(file, fput_needed); } return ret; @@ -494,11 +501,12 @@ sys_writev(unsigned long fd, const struc { struct file *file; ssize_t ret = -EBADF; + int fput_needed; - file = fget(fd); + file = fget_light(fd, &fput_needed); if (file) { ret = vfs_writev(file, vec, vlen, &file->f_pos); - fput(file); + fput_light(file, fput_needed); } return ret; @@ -511,12 +519,13 @@ static ssize_t do_sendfile(int out_fd, i struct inode * in_inode, * out_inode; loff_t pos; ssize_t retval; + int fput_needed_in, fput_needed_out; /* * Get input file, and verify that it is ok.. */ retval = -EBADF; - in_file = fget(in_fd); + in_file = fget_light(in_fd, &fput_needed_in); if (!in_file) goto out; if (!(in_file->f_mode & FMODE_READ)) @@ -539,7 +548,7 @@ static ssize_t do_sendfile(int out_fd, i * Get output file, and verify that it is ok.. */ retval = -EBADF; - out_file = fget(out_fd); + out_file = fget_light(out_fd, &fput_needed_out); if (!out_file) goto fput_in; if (!(out_file->f_mode & FMODE_WRITE)) @@ -579,9 +588,9 @@ static ssize_t do_sendfile(int out_fd, i retval = -EOVERFLOW; fput_out: - fput(out_file); + fput_light(out_file, fput_needed_out); fput_in: - fput(in_file); + fput_light(in_file, fput_needed_in); out: return retval; } diff -puN include/linux/file.h~fget-speedup include/linux/file.h --- 25/include/linux/file.h~fget-speedup Fri May 2 13:59:29 2003 +++ 25-akpm/include/linux/file.h Fri May 2 14:00:12 2003 @@ -35,7 +35,9 @@ struct files_struct { extern void FASTCALL(__fput(struct file *)); extern void FASTCALL(fput(struct file *)); +extern void FASTCALL(fput_light(struct file *, int)); extern struct file * FASTCALL(fget(unsigned int fd)); +extern struct file * FASTCALL(fget_light(unsigned int fd, int *fput_needed)); extern void FASTCALL(set_close_on_exec(unsigned int fd, int flag)); extern void put_filp(struct file *); extern int get_unused_fd(void); _