Patch from Ravikiran G Thirumalai Makes the disk stats on struct gendisk per-cpu. drivers/block/genhd.c | 27 ++++++++----- drivers/block/ll_rw_blk.c | 29 +++++++------- drivers/md/md.c | 4 + fs/partitions/check.c | 8 --- include/linux/genhd.h | 94 ++++++++++++++++++++++++++++++++++++++++++---- 5 files changed, 123 insertions(+), 39 deletions(-) diff -puN drivers/block/genhd.c~per-cpu-disk-stats drivers/block/genhd.c --- 25/drivers/block/genhd.c~per-cpu-disk-stats 2003-02-28 03:54:44.000000000 -0800 +++ 25-akpm/drivers/block/genhd.c 2003-02-28 03:54:44.000000000 -0800 @@ -318,7 +318,7 @@ static inline unsigned jiffies_to_msec(u return (jif / HZ) * 1000 + (jif % HZ) * 1000 / HZ; #endif } -static ssize_t disk_stat_read(struct gendisk * disk, char *page) +static ssize_t disk_stats_read(struct gendisk * disk, char *page) { disk_round_stats(disk); return sprintf(page, @@ -326,14 +326,16 @@ static ssize_t disk_stat_read(struct gen "%8u %8u %8llu %8u " "%8u %8u %8u" "\n", - disk->reads, disk->read_merges, - (unsigned long long)disk->read_sectors, - jiffies_to_msec(disk->read_ticks), - disk->writes, disk->write_merges, - (unsigned long long)disk->write_sectors, - jiffies_to_msec(disk->write_ticks), - disk->in_flight, jiffies_to_msec(disk->io_ticks), - jiffies_to_msec(disk->time_in_queue)); + disk_stat_read(disk, reads), disk_stat_read(disk, read_merges), + (unsigned long long)disk_stat_read(disk, read_sectors), + jiffies_to_msec(disk_stat_read(disk, read_ticks)), + disk_stat_read(disk, writes), + disk_stat_read(disk, write_merges), + (unsigned long long)disk_stat_read(disk, write_sectors), + jiffies_to_msec(disk_stat_read(disk, write_ticks)), + disk_stat_read(disk, in_flight), + jiffies_to_msec(disk_stat_read(disk, io_ticks)), + jiffies_to_msec(disk_stat_read(disk, time_in_queue))); } static struct disk_attribute disk_attr_dev = { .attr = {.name = "dev", .mode = S_IRUGO }, @@ -349,7 +351,7 @@ static struct disk_attribute disk_attr_s }; static struct disk_attribute disk_attr_stat = { .attr = {.name = "stat", .mode = S_IRUGO }, - .show = disk_stat_read + .show = disk_stats_read }; static struct attribute * default_attrs[] = { @@ -365,6 +367,7 @@ static void disk_release(struct kobject struct gendisk *disk = to_disk(kobj); kfree(disk->random); kfree(disk->part); + free_disk_stats(disk); kfree(disk); } @@ -384,6 +387,10 @@ struct gendisk *alloc_disk(int minors) struct gendisk *disk = kmalloc(sizeof(struct gendisk), GFP_KERNEL); if (disk) { memset(disk, 0, sizeof(struct gendisk)); + if (!init_disk_stats(disk)) { + kfree(disk); + return NULL; + } if (minors > 1) { int size = (minors - 1) * sizeof(struct hd_struct); disk->part = kmalloc(size, GFP_KERNEL); diff -puN drivers/block/ll_rw_blk.c~per-cpu-disk-stats drivers/block/ll_rw_blk.c --- 25/drivers/block/ll_rw_blk.c~per-cpu-disk-stats 2003-02-28 03:54:44.000000000 -0800 +++ 25-akpm/drivers/block/ll_rw_blk.c 2003-02-28 03:54:44.000000000 -0800 @@ -1458,17 +1458,17 @@ void drive_stat_acct(struct request *rq, return; if (rw == READ) { - rq->rq_disk->read_sectors += nr_sectors; + disk_stat_add(rq->rq_disk, read_sectors, nr_sectors); if (!new_io) - rq->rq_disk->read_merges++; + disk_stat_inc(rq->rq_disk, read_merges); } else if (rw == WRITE) { - rq->rq_disk->write_sectors += nr_sectors; + disk_stat_add(rq->rq_disk, write_sectors, nr_sectors); if (!new_io) - rq->rq_disk->write_merges++; + disk_stat_inc(rq->rq_disk, write_merges); } if (new_io) { disk_round_stats(rq->rq_disk); - rq->rq_disk->in_flight++; + disk_stat_inc(rq->rq_disk, in_flight); } } @@ -1508,11 +1508,12 @@ void disk_round_stats(struct gendisk *di { unsigned long now = jiffies; - disk->time_in_queue += disk->in_flight * (now - disk->stamp); + disk_stat_add(disk, time_in_queue, + disk_stat_read(disk, in_flight) * (now - disk->stamp)); disk->stamp = now; - if (disk->in_flight) - disk->io_ticks += (now - disk->stamp_idle); + if (disk_stat_read(disk, in_flight)) + disk_stat_add(disk, io_ticks, (now - disk->stamp_idle)); disk->stamp_idle = now; } @@ -1628,7 +1629,7 @@ static int attempt_merge(request_queue_t if (req->rq_disk) { disk_round_stats(req->rq_disk); - req->rq_disk->in_flight--; + disk_stat_dec(req->rq_disk, in_flight); } __blk_put_request(q, next); @@ -2180,16 +2181,16 @@ void end_that_request_last(struct reques unsigned long duration = jiffies - req->start_time; switch (rq_data_dir(req)) { case WRITE: - disk->writes++; - disk->write_ticks += duration; + disk_stat_inc(disk, writes); + disk_stat_add(disk, write_ticks, duration); break; case READ: - disk->reads++; - disk->read_ticks += duration; + disk_stat_inc(disk, reads); + disk_stat_add(disk, read_ticks, duration); break; } disk_round_stats(disk); - disk->in_flight--; + disk_stat_dec(disk, in_flight); } __blk_put_request(req->q, req); } diff -puN drivers/md/md.c~per-cpu-disk-stats drivers/md/md.c --- 25/drivers/md/md.c~per-cpu-disk-stats 2003-02-28 03:54:44.000000000 -0800 +++ 25-akpm/drivers/md/md.c 2003-02-28 03:54:44.000000000 -0800 @@ -2803,7 +2803,9 @@ static int is_mddev_idle(mddev_t *mddev) idle = 1; ITERATE_RDEV(mddev,rdev,tmp) { struct gendisk *disk = rdev->bdev->bd_contains->bd_disk; - curr_events = disk->read_sectors + disk->write_sectors - disk->sync_io; + curr_events = disk_stat_read(disk, read_sectors) + + disk_stat_read(disk, write_sectors) - + disk->sync_io; if ((curr_events - rdev->last_events) > 32) { rdev->last_events = curr_events; idle = 0; diff -puN fs/partitions/check.c~per-cpu-disk-stats fs/partitions/check.c --- 25/fs/partitions/check.c~per-cpu-disk-stats 2003-02-28 03:54:44.000000000 -0800 +++ 25-akpm/fs/partitions/check.c 2003-02-28 03:54:44.000000000 -0800 @@ -503,13 +503,7 @@ void del_gendisk(struct gendisk *disk) disk->capacity = 0; disk->flags &= ~GENHD_FL_UP; unlink_gendisk(disk); - disk->reads = disk->writes = 0; - disk->read_sectors = disk->write_sectors = 0; - disk->read_merges = disk->write_merges = 0; - disk->read_ticks = disk->write_ticks = 0; - disk->in_flight = 0; - disk->io_ticks = 0; - disk->time_in_queue = 0; + disk_stat_set_all(disk, 0); disk->stamp = disk->stamp_idle = 0; devfs_remove_partitions(disk); if (disk->driverfs_dev) { diff -puN include/linux/genhd.h~per-cpu-disk-stats include/linux/genhd.h --- 25/include/linux/genhd.h~per-cpu-disk-stats 2003-02-28 03:54:44.000000000 -0800 +++ 25-akpm/include/linux/genhd.h 2003-02-28 03:54:44.000000000 -0800 @@ -70,6 +70,16 @@ struct hd_struct { #define GENHD_FL_CD 8 #define GENHD_FL_UP 16 +struct disk_stats { + unsigned read_sectors, write_sectors; + unsigned reads, writes; + unsigned read_merges, write_merges; + unsigned read_ticks, write_ticks; + unsigned io_ticks; + int in_flight; + unsigned time_in_queue; +}; + struct gendisk { int major; /* major number of driver */ int first_minor; @@ -94,16 +104,86 @@ struct gendisk { int policy; unsigned sync_io; /* RAID */ - unsigned read_sectors, write_sectors; - unsigned reads, writes; - unsigned read_merges, write_merges; - unsigned read_ticks, write_ticks; - unsigned io_ticks; - int in_flight; unsigned long stamp, stamp_idle; - unsigned time_in_queue; +#ifdef CONFIG_SMP + struct disk_stats *dkstats; +#else + struct disk_stats dkstats; +#endif }; +/* + * Macros to operate on percpu disk statistics: + * Since writes to disk_stats are serialised through the queue_lock, + * smp_processor_id() should be enough to get to the per_cpu versions + * of statistics counters + */ +#ifdef CONFIG_SMP +#define disk_stat_add(gendiskp, field, addnd) \ + (per_cpu_ptr(gendiskp->dkstats, smp_processor_id())->field += addnd) +#define disk_stat_read(gendiskp, field) \ +({ \ + typeof(gendiskp->dkstats->field) res = 0; \ + int i; \ + for (i=0; i < NR_CPUS; i++) { \ + if (!cpu_possible(i)) \ + continue; \ + res += per_cpu_ptr(gendiskp->dkstats, i)->field; \ + } \ + res; \ +}) + +static inline void disk_stat_set_all(struct gendisk *gendiskp, int value) { + int i; + for (i=0; i < NR_CPUS; i++) { + if (cpu_possible(i)) { + memset(per_cpu_ptr(gendiskp->dkstats, i), value, + sizeof (struct disk_stats)); + } + } +} + +#else +#define disk_stat_add(gendiskp, field, addnd) (gendiskp->dkstats.field += addnd) +#define disk_stat_read(gendiskp, field) (gendiskp->dkstats.field) + +static inline void disk_stat_set_all(struct gendisk *gendiskp, int value) { + memset(&gendiskp->dkstats, value, sizeof (struct disk_stats)); +} +#endif + +#define disk_stat_inc(gendiskp, field) disk_stat_add(gendiskp, field, 1) +#define disk_stat_dec(gendiskp, field) disk_stat_add(gendiskp, field, -1) +#define disk_stat_sub(gendiskp, field, subnd) \ + disk_stat_add(gendiskp, field, -subnd) + + +/* Inlines to alloc and free disk stats in struct gendisk */ +#ifdef CONFIG_SMP +static inline int init_disk_stats(struct gendisk *disk) +{ + disk->dkstats = kmalloc_percpu(sizeof (struct disk_stats), GFP_KERNEL); + if (!disk->dkstats) + return 0; + disk_stat_set_all(disk, 0); + return 1; +} + +static inline void free_disk_stats(struct gendisk *disk) +{ + kfree_percpu(disk->dkstats); +} +#else /* CONFIG_SMP */ +static inline int init_disk_stats(struct gendisk *disk) +{ + return 1; +} + +static inline void free_disk_stats(struct gendisk *disk) +{ +} +#endif /* CONFIG_SMP */ + /* drivers/block/ll_rw_blk.c */ extern void disk_round_stats(struct gendisk *disk); _