From dmberezin at hotmail.com Tue Oct 5 08:21:48 2004 From: dmberezin at hotmail.com (dmberezin at hotmail.com) Date: Tue, 5 Oct 2004 08:21:48 -0700 Subject: [Orca-checkins] r395 - trunk/orca/data_gatherers/orcallator Message-ID: <200410051521.i95FLmvG006769@gw.orcaware.com> Author: dmberezin at hotmail.com Date: Tue Oct 5 08:18:50 2004 New Revision: 395 Modified: trunk/orca/data_gatherers/orcallator/orcallator.se Log: More changes to disk/tape data collection * data_gatherers/orcallator/orcallator.se Add check: define USE_KSTAT_IO only if NO_KSTAT_IO is not defined. This allows disabling KSTAT_IO code with command line option. Separate disk and tape statistics gathering. Add define WATCH_TAPE function measure_tape We can only gather tape performance data with KSTAT_IO code at this time, because of the way live_rules.se checks for disk slices to ignore them. It simply checks if there is an 's' in the long_name, which is true for st devices, so they are ignored. We could fix live_rules.se or combine diskinfo.se and tapeinfo.se together to have correct long_name (rmt/xx) appear when live_rules.se accesses it, but perhaps using KSTAT_IO for this is sufficient. (measure_disk): remove tape data gathering code - move to measure_tape. (io_dev_info_t): add string dev_class to make it easier to identify type of devices in measure_disk/tape functions. (orca_io_info_update): call this function only if USE_KSTAT_IO is defined and at least one of WATCH_DISK/TAPE is defined. Save time if not watching io. Fixed some comments. Modified: trunk/orca/data_gatherers/orcallator/orcallator.se ============================================================================== --- trunk/orca/data_gatherers/orcallator/orcallator.se (original) +++ trunk/orca/data_gatherers/orcallator/orcallator.se Tue Oct 5 08:18:50 2004 @@ -24,7 +24,9 @@ #define MAX_COLUMNS 2048 // Enable kstat io measuring code. +#ifndef NO_KSTAT_IO #define USE_KSTAT_IO 1 +#endif // If WATCH_OS is defined, then measure every part of the operating // system. @@ -37,6 +39,7 @@ #define WATCH_NFS_SERVER 1 #define WATCH_MOUNTS 1 #define WATCH_DISK 1 +#define WATCH_TAPE 1 #define WATCH_DNLC 1 #define WATCH_INODE 1 #define WATCH_RAM 1 @@ -300,17 +303,14 @@ #ifdef USE_KSTAT_IO #include #include -// This code was developed so that the performance of virtual disks -// originating from a Sun A1000 raid controller could be monitored. -// These disks do not show up in the GLOBAL_disk[] io structure of SE. -// -// This extension accesses the sys_kstat.se interface to the kstat IO -// queues to extract info on drives not available in the kstat.se -// kstat$disk interface. Global data shared between function calls. +// The following code collects performance data for disk and/or tape devices +// by accessing kstat interface directly. It was originally developed to +// overcome limitation of GLOBAL_disk[] array populated in live_rules.se. struct io_dev_info_t { // Exposed interface that matches kstat. string long_name; string short_name; + string dev_class; double reads; double kreads; double writes; @@ -337,7 +337,7 @@ uint _rcnt; // Count of elements in run state }; -// Define global for tracking raw disk data. +// Define globals for tracking kstat io data. io_dev_info_t ORCA_io_dev_info[]; int ORCA_io_dev_count=0; int ORCA_max_io_dev_count=0; @@ -362,7 +362,13 @@ struct_fill(nkp[0], ul); if (nkp[0].ks_type == KSTAT_TYPE_IO) { // Look for disk or tape statistics +#if defined (WATCH_DISK) && defined (WATCH_TAPE) if (nkp[0].ks_class == "disk" || nkp[0].ks_class == "tape") { +#elif defined (WATCH_TAPE) + if (nkp[0].ks_class == "tape") { +#else + if (nkp[0].ks_class == "disk") { +#endif // Get data from the kernel for this kstat if (kstat_read(kc, nkp, 0) == -1) { perror("orca_io_info_update:kstat_read error"); @@ -384,7 +390,7 @@ ORCA_max_io_dev_count += 10; ORCA_io_dev_info = renew ORCA_io_dev_info[ORCA_max_io_dev_count]; } - + // Lookup index into GLOBAL_disk/tape_info array if (nkp[0].ks_class == "tape") { index = find_tape_inst(nkp[0].ks_name); } else { @@ -399,8 +405,8 @@ } else { ORCA_io_dev_info[iodev].long_name = nkp[0].ks_name; } - ORCA_io_dev_info[iodev].short_name = nkp[0].ks_name; - + ORCA_io_dev_info[iodev].short_name = nkp[0].ks_name; + ORCA_io_dev_info[iodev].dev_class = nkp[0].ks_class; ORCA_io_dev_info[iodev]._writes = kio.writes; ORCA_io_dev_info[iodev]._nwritten = kio.nwritten; ORCA_io_dev_info[iodev]._wlastupdate = kio.wlastupdate; @@ -691,6 +697,7 @@ fprintf(stderr, " -DWATCH_NFS_SERVER watch NFS server requests\n"); fprintf(stderr, " -DWATCH_MOUNTS watch usage of mount points\n"); fprintf(stderr, " -DWATCH_DISK watch disk read/write usage\n"); + fprintf(stderr, " -DWATCH_TAPE watch tape read/write usage\n"); fprintf(stderr, " -DWATCH_DNLC watch the directory name lookup cache\n"); fprintf(stderr, " -DWATCH_INODE watch the inode cache\n"); fprintf(stderr, " -DWATCH_RAM watch memory usage\n"); @@ -861,7 +868,7 @@ tmp_tcp = tcp$tcp; #endif -#ifdef USE_KSTAT_IO +#if defined (USE_KSTAT_IO) && (defined (WATCH_DISK) || defined (WATCH_TAPE)) orca_io_info_update(); #endif } @@ -901,6 +908,11 @@ measure_disk(); #endif + // Take care of the tapes. +#if defined (WATCH_TAPE) && defined (USE_KSTAT_IO) + measure_tape(); +#endif + // Take care of ram. #ifdef WATCH_RAM measure_ram(); @@ -1399,10 +1411,9 @@ #ifdef WATCH_DISK measure_disk() { - // These two variables are treated as static variables to keep track - // of any changes to the number of disks and tapes on the system. + // This variable is treated as static variable to keep track + // of any changes to the number of disks on the system. int previous_disk_count = -1; - int previous_tape_count = -1; double mean_disk_busy; double peak_disk_busy; @@ -1412,12 +1423,6 @@ double total_disk_writek; int disk_count; - double total_tape_reads; - double total_tape_writes; - double total_tape_readk; - double total_tape_writek; - int tape_count; - int i; mean_disk_busy = 0.0; @@ -1428,62 +1433,43 @@ total_disk_writek = 0.0; disk_count = 0; - total_tape_reads = 0.0; - total_tape_writes = 0.0; - total_tape_readk = 0.0; - total_tape_writek = 0.0; - tape_count = 0; - #ifdef USE_KSTAT_IO for (i=0; i. Check - // [wr]lentime to see if an EMC is using a fake disk for control. - // EMC disks have a fake disk which commands are run over to - // configure the disk array or to get stats from; they are not - // real data transfers. They cause 1000 MB/sec writes to appear - // in the stats. I still get them but not as often with this bit - // of code in. If the I/O which occurred in the last five minutes - // is not greater than 1/100sec then it is not a valid measurement - // anyway. What happens is that we can have a small I/O, say 1024 - // bytes, in a 1/100sec = 1024*100/sec. I am thinking of making - // it wlentime+rlentime>2 since I am still getting fake write - // spikes. + // Comments from Damon Atkins . Check + // [wr]lentime to see if an EMC is using a fake disk for control. + // EMC disks have a fake disk which commands are run over to + // configure the disk array or to get stats from; they are not + // real data transfers. They cause 1000 MB/sec writes to appear + // in the stats. I still get them but not as often with this bit + // of code in. If the I/O which occurred in the last five minutes + // is not greater than 1/100sec then it is not a valid measurement + // anyway. What happens is that we can have a small I/O, say 1024 + // bytes, in a 1/100sec = 1024*100/sec. I am thinking of making + // it wlentime+rlentime>2 since I am still getting fake write + // spikes. #ifdef HAVE_EMC_DISK_CONTROL - if ((pioGLOB_old_wlentime[i] + pioGLOB_old_rlentime[i]) > 1) { + if ((pioGLOB_old_wlentime[i] + pioGLOB_old_rlentime[i]) > 1) { #endif - total_disk_reads += ORCA_io_dev_info[i].reads; - total_disk_writes += ORCA_io_dev_info[i].writes; - total_disk_readk += ORCA_io_dev_info[i].kreads; - total_disk_writek += ORCA_io_dev_info[i].kwrites; - mean_disk_busy += ORCA_io_dev_info[i].run_percent; - if (ORCA_io_dev_info[i].run_percent > peak_disk_busy) { - peak_disk_busy = ORCA_io_dev_info[i].run_percent; - } + total_disk_reads += ORCA_io_dev_info[i].reads; + total_disk_writes += ORCA_io_dev_info[i].writes; + total_disk_readk += ORCA_io_dev_info[i].kreads; + total_disk_writek += ORCA_io_dev_info[i].kwrites; + mean_disk_busy += ORCA_io_dev_info[i].run_percent; + if (ORCA_io_dev_info[i].run_percent > peak_disk_busy) { + peak_disk_busy = ORCA_io_dev_info[i].run_percent; + } #ifdef HAVE_EMC_DISK_CONTROL - } + } #endif + } } #else for (i=0; i Author: dmberezin at hotmail.com Date: Tue Oct 5 13:39:28 2004 New Revision: 396 Modified: trunk/orca/data_gatherers/orcallator/orcallator.se Log: More changes to disk/tape data collection * data_gatherers/orcallator/orcallator.se (orca_io_info_update): fixed average wait and service time calculation. Now compute read_writes correctly. Renamed variables: avg_wait -> avg_wait_time, avg_serv -> avg_serv_time Added variables: avg_run, avg_wait Modified: trunk/orca/data_gatherers/orcallator/orcallator.se ============================================================================== --- trunk/orca/data_gatherers/orcallator/orcallator.se (original) +++ trunk/orca/data_gatherers/orcallator/orcallator.se Tue Oct 5 13:39:28 2004 @@ -308,18 +308,20 @@ // overcome limitation of GLOBAL_disk[] array populated in live_rules.se. struct io_dev_info_t { // Exposed interface that matches kstat. - string long_name; - string short_name; - string dev_class; + string long_name; // Long device name (cXtXdX, /dev/rmtXX) + string short_name; // Short device name (sdXX, stXX) + string dev_class; // Device class (disk, tape) double reads; double kreads; double writes; double kwrites; - double avg_wait; - double avg_serv; - double service; + double avg_wait; // Wait queue length + double avg_run; // Active queue length + double avg_wait_time; // Wait time in ms + double avg_serv_time; // Active service time in ms + double service; // Response time (avg_wait_time+avg_serv_time) double wait_percent; - double run_percent; + double run_percent; // a.k.a utilization // Hidden internal registers to track sys_kstats counters. int _number; // kstat disk # @@ -434,14 +436,17 @@ ORCA_io_dev_info[iodev].writes =(kio.writes-ORCA_io_dev_info[iodev]._writes) /hz_etime; ORCA_io_dev_info[iodev].kwrites=(kio.nwritten-ORCA_io_dev_info[iodev]._nwritten)/big_etime; - read_writes = elapsed_etime * (ORCA_io_dev_info[iodev].reads + ORCA_io_dev_info[iodev].writes) / 1024.0; + ORCA_io_dev_info[iodev].avg_wait=(kio.wlentime-ORCA_io_dev_info[iodev]._wlentime)/elapsed_etime; + ORCA_io_dev_info[iodev].avg_run =(kio.rlentime-ORCA_io_dev_info[iodev]._rlentime)/elapsed_etime; + + read_writes = (ORCA_io_dev_info[iodev].reads + ORCA_io_dev_info[iodev].writes) / 1000.0; if (read_writes > 0) { - ORCA_io_dev_info[iodev].avg_wait = (kio.wlentime - ORCA_io_dev_info[iodev]._wlentime) / read_writes; - ORCA_io_dev_info[iodev].avg_serv = (kio.rlentime - ORCA_io_dev_info[iodev]._rlentime) / read_writes; - ORCA_io_dev_info[iodev].service = ORCA_io_dev_info[iodev].avg_wait + ORCA_io_dev_info[iodev].avg_serv; + ORCA_io_dev_info[iodev].avg_wait_time = ORCA_io_dev_info[iodev].avg_wait / read_writes; + ORCA_io_dev_info[iodev].avg_serv_time = ORCA_io_dev_info[iodev].avg_run / read_writes; + ORCA_io_dev_info[iodev].service = ORCA_io_dev_info[iodev].avg_wait_time + ORCA_io_dev_info[iodev].avg_serv_time; } else { - ORCA_io_dev_info[iodev].avg_wait = 0.0; - ORCA_io_dev_info[iodev].avg_serv = 0.0; + ORCA_io_dev_info[iodev].avg_wait_time = 0.0; + ORCA_io_dev_info[iodev].avg_serv_time = 0.0; ORCA_io_dev_info[iodev].service = 0.0; } From dmberezin at hotmail.com Mon Oct 11 08:07:04 2004 From: dmberezin at hotmail.com (dmberezin at hotmail.com) Date: Mon, 11 Oct 2004 08:07:04 -0700 Subject: [Orca-checkins] r397 - trunk/orca/data_gatherers/orcallator Message-ID: <200410111507.i9BF74XY015129@gw.orcaware.com> Author: dmberezin at hotmail.com Date: Mon Oct 11 08:01:45 2004 New Revision: 397 Modified: trunk/orca/data_gatherers/orcallator/orcallator.se Log: More changes to disk/tape data collection * data_gatherers/orcallator/orcallator.se (orca_io_info_update): minor changes to the function; committing separately to make changes in the next patch easier to see. Replace struct_fill calls with indirection assignments. Modified: trunk/orca/data_gatherers/orcallator/orcallator.se ============================================================================== --- trunk/orca/data_gatherers/orcallator/orcallator.se (original) +++ trunk/orca/data_gatherers/orcallator/orcallator.se Mon Oct 11 08:01:45 2004 @@ -359,100 +359,55 @@ // Initialize kstat control structure kc[0] = kstat_open(); - // Traverse the chain looking for IO events. + // Traverse the chain looking for IO events for disk and tape devices for (ul=kc[0].kc_chain; ul!=0; ul=nkp[0].ks_next) { - struct_fill(nkp[0], ul); - if (nkp[0].ks_type == KSTAT_TYPE_IO) { - // Look for disk or tape statistics + nkp[0] = *((kstat_t *) ul); + if (nkp[0].ks_type == KSTAT_TYPE_IO && #if defined (WATCH_DISK) && defined (WATCH_TAPE) - if (nkp[0].ks_class == "disk" || nkp[0].ks_class == "tape") { + (nkp[0].ks_class == "disk" || nkp[0].ks_class == "tape")) { #elif defined (WATCH_TAPE) - if (nkp[0].ks_class == "tape") { + nkp[0].ks_class == "tape") { #else - if (nkp[0].ks_class == "disk") { + nkp[0].ks_class == "disk") { #endif - // Get data from the kernel for this kstat - if (kstat_read(kc, nkp, 0) == -1) { - perror("orca_io_info_update:kstat_read error"); - exit(1); + // Get data from the kernel for this kstat + if (kstat_read(kc, nkp, 0) == -1) { + perror("orca_io_info_update:kstat_read error"); + exit(1); + } + kio = *((kstat_io_t *) nkp[0].ks_data); + + // Try to locate device in our array + for (iodev=0; iodev < ORCA_io_dev_count; ++iodev) { + if (ORCA_io_dev_info[iodev].short_name == nkp[0].ks_name) { + break; } - struct_fill(kio, nkp[0].ks_data); + } - // Try to locate device in our array - for (iodev=0; iodev < ORCA_io_dev_count; ++iodev) { - if (ORCA_io_dev_info[iodev].short_name == nkp[0].ks_name) { - break; - } + // It must be new. Add it! + if (iodev == ORCA_io_dev_count) { + // Grow the device array if needed + if (ORCA_io_dev_count == ORCA_max_io_dev_count) { + ORCA_max_io_dev_count += 10; + ORCA_io_dev_info = renew ORCA_io_dev_info[ORCA_max_io_dev_count]; } - - // It must be new. Add it! - if (iodev == ORCA_io_dev_count) { - // Grow the device array if needed - if (ORCA_io_dev_count == ORCA_max_io_dev_count) { - ORCA_max_io_dev_count += 10; - ORCA_io_dev_info = renew ORCA_io_dev_info[ORCA_max_io_dev_count]; - } - // Lookup index into GLOBAL_disk/tape_info array + // Lookup index into GLOBAL_disk/tape_info array + if (nkp[0].ks_class == "tape") { + index = find_tape_inst(nkp[0].ks_name); + } else { + index = find_inst(nkp[0].ks_name); + } + if (index != -1) { if (nkp[0].ks_class == "tape") { - index = find_tape_inst(nkp[0].ks_name); + ORCA_io_dev_info[iodev].long_name = GLOBAL_tape_info[index].long_name; } else { - index = find_inst(nkp[0].ks_name); + ORCA_io_dev_info[iodev].long_name = GLOBAL_disk_info[index].long_name; } - if (index != -1) { - if (nkp[0].ks_class == "tape") { - ORCA_io_dev_info[iodev].long_name = GLOBAL_tape_info[index].long_name; - } else { - ORCA_io_dev_info[iodev].long_name = GLOBAL_disk_info[index].long_name; - } - } else { - ORCA_io_dev_info[iodev].long_name = nkp[0].ks_name; - } - ORCA_io_dev_info[iodev].short_name = nkp[0].ks_name; - ORCA_io_dev_info[iodev].dev_class = nkp[0].ks_class; - ORCA_io_dev_info[iodev]._writes = kio.writes; - ORCA_io_dev_info[iodev]._nwritten = kio.nwritten; - ORCA_io_dev_info[iodev]._wlastupdate = kio.wlastupdate; - ORCA_io_dev_info[iodev]._wlentime = kio.wlentime; - ORCA_io_dev_info[iodev]._wtime = kio.wtime; - ORCA_io_dev_info[iodev]._wcnt = kio.wcnt; - ORCA_io_dev_info[iodev]._reads = kio.reads; - ORCA_io_dev_info[iodev]._nread = kio.nread; - ORCA_io_dev_info[iodev]._rlastupdate = kio.rlastupdate; - ORCA_io_dev_info[iodev]._rlentime = kio.rlentime; - ORCA_io_dev_info[iodev]._rtime = kio.rtime; - ORCA_io_dev_info[iodev]._rcnt = kio.rcnt; - ORCA_io_dev_count++; - } - - elapsed_etime = (kio.wlastupdate-ORCA_io_dev_info[iodev]._wlastupdate); - if (elapsed_etime == 0) { - elapsed_etime = NANOSEC; - } - hz_etime = elapsed_etime / NANOSEC; - big_etime = 1024.0 * hz_etime; - - ORCA_io_dev_info[iodev].reads =(kio.reads-ORCA_io_dev_info[iodev]._reads) /hz_etime; - ORCA_io_dev_info[iodev].kreads =(kio.nread-ORCA_io_dev_info[iodev]._nread) /big_etime; - ORCA_io_dev_info[iodev].writes =(kio.writes-ORCA_io_dev_info[iodev]._writes) /hz_etime; - ORCA_io_dev_info[iodev].kwrites=(kio.nwritten-ORCA_io_dev_info[iodev]._nwritten)/big_etime; - - ORCA_io_dev_info[iodev].avg_wait=(kio.wlentime-ORCA_io_dev_info[iodev]._wlentime)/elapsed_etime; - ORCA_io_dev_info[iodev].avg_run =(kio.rlentime-ORCA_io_dev_info[iodev]._rlentime)/elapsed_etime; - - read_writes = (ORCA_io_dev_info[iodev].reads + ORCA_io_dev_info[iodev].writes) / 1000.0; - if (read_writes > 0) { - ORCA_io_dev_info[iodev].avg_wait_time = ORCA_io_dev_info[iodev].avg_wait / read_writes; - ORCA_io_dev_info[iodev].avg_serv_time = ORCA_io_dev_info[iodev].avg_run / read_writes; - ORCA_io_dev_info[iodev].service = ORCA_io_dev_info[iodev].avg_wait_time + ORCA_io_dev_info[iodev].avg_serv_time; } else { - ORCA_io_dev_info[iodev].avg_wait_time = 0.0; - ORCA_io_dev_info[iodev].avg_serv_time = 0.0; - ORCA_io_dev_info[iodev].service = 0.0; + ORCA_io_dev_info[iodev].long_name = nkp[0].ks_name; } - - // Update the counters. - ORCA_io_dev_info[iodev].run_percent = 100.0 * (kio.rtime - ORCA_io_dev_info[iodev]._rtime) / elapsed_etime; - ORCA_io_dev_info[iodev].wait_percent = 100.0 * (kio.wtime - ORCA_io_dev_info[iodev]._wtime) / elapsed_etime; + ORCA_io_dev_info[iodev].short_name = nkp[0].ks_name; + ORCA_io_dev_info[iodev].dev_class = nkp[0].ks_class; ORCA_io_dev_info[iodev]._writes = kio.writes; ORCA_io_dev_info[iodev]._nwritten = kio.nwritten; ORCA_io_dev_info[iodev]._wlastupdate = kio.wlastupdate; @@ -465,7 +420,50 @@ ORCA_io_dev_info[iodev]._rlentime = kio.rlentime; ORCA_io_dev_info[iodev]._rtime = kio.rtime; ORCA_io_dev_info[iodev]._rcnt = kio.rcnt; + ORCA_io_dev_count++; + } + + elapsed_etime = (kio.wlastupdate-ORCA_io_dev_info[iodev]._wlastupdate); + if (elapsed_etime == 0) { + elapsed_etime = NANOSEC; } + hz_etime = elapsed_etime / NANOSEC; + big_etime = 1024.0 * hz_etime; + + ORCA_io_dev_info[iodev].reads =(kio.reads-ORCA_io_dev_info[iodev]._reads) /hz_etime; + ORCA_io_dev_info[iodev].kreads =(kio.nread-ORCA_io_dev_info[iodev]._nread) /big_etime; + ORCA_io_dev_info[iodev].writes =(kio.writes-ORCA_io_dev_info[iodev]._writes) /hz_etime; + ORCA_io_dev_info[iodev].kwrites=(kio.nwritten-ORCA_io_dev_info[iodev]._nwritten)/big_etime; + + ORCA_io_dev_info[iodev].avg_wait=(kio.wlentime-ORCA_io_dev_info[iodev]._wlentime)/elapsed_etime; + ORCA_io_dev_info[iodev].avg_run =(kio.rlentime-ORCA_io_dev_info[iodev]._rlentime)/elapsed_etime; + + read_writes = (ORCA_io_dev_info[iodev].reads + ORCA_io_dev_info[iodev].writes) / 1000.0; + if (read_writes > 0) { + ORCA_io_dev_info[iodev].avg_wait_time = ORCA_io_dev_info[iodev].avg_wait / read_writes; + ORCA_io_dev_info[iodev].avg_serv_time = ORCA_io_dev_info[iodev].avg_run / read_writes; + ORCA_io_dev_info[iodev].service = ORCA_io_dev_info[iodev].avg_wait_time + ORCA_io_dev_info[iodev].avg_serv_time; + } else { + ORCA_io_dev_info[iodev].avg_wait_time = 0.0; + ORCA_io_dev_info[iodev].avg_serv_time = 0.0; + ORCA_io_dev_info[iodev].service = 0.0; + } + + // Update the counters. + ORCA_io_dev_info[iodev].run_percent = 100.0 * (kio.rtime - ORCA_io_dev_info[iodev]._rtime) / elapsed_etime; + ORCA_io_dev_info[iodev].wait_percent = 100.0 * (kio.wtime - ORCA_io_dev_info[iodev]._wtime) / elapsed_etime; + ORCA_io_dev_info[iodev]._writes = kio.writes; + ORCA_io_dev_info[iodev]._nwritten = kio.nwritten; + ORCA_io_dev_info[iodev]._wlastupdate = kio.wlastupdate; + ORCA_io_dev_info[iodev]._wlentime = kio.wlentime; + ORCA_io_dev_info[iodev]._wtime = kio.wtime; + ORCA_io_dev_info[iodev]._wcnt = kio.wcnt; + ORCA_io_dev_info[iodev]._reads = kio.reads; + ORCA_io_dev_info[iodev]._nread = kio.nread; + ORCA_io_dev_info[iodev]._rlastupdate = kio.rlastupdate; + ORCA_io_dev_info[iodev]._rlentime = kio.rlentime; + ORCA_io_dev_info[iodev]._rtime = kio.rtime; + ORCA_io_dev_info[iodev]._rcnt = kio.rcnt; } } kstat_close(kc); @@ -874,7 +872,7 @@ #endif #if defined (USE_KSTAT_IO) && (defined (WATCH_DISK) || defined (WATCH_TAPE)) - orca_io_info_update(); + orca_io_info_update(); #endif } From dmberezin at hotmail.com Mon Oct 11 12:12:35 2004 From: dmberezin at hotmail.com (dmberezin at hotmail.com) Date: Mon, 11 Oct 2004 12:12:35 -0700 Subject: [Orca-checkins] r398 - trunk/orca/data_gatherers/orcallator Message-ID: <200410111912.i9BJCZUw017899@gw.orcaware.com> Author: dmberezin at hotmail.com Date: Mon Oct 11 12:10:53 2004 New Revision: 398 Modified: trunk/orca/data_gatherers/orcallator/orcallator.se Log: Fix for kio.nread bug in SE * data_gatherers/orcallator/orcallator.se (get_new_kstat_data): new function (orca_io_info_update): add code to re-read kstat data if kio.nread appears to be corrupt. SE appears to have a bug - occasionally kio.nread is erroneously set to 0. It looks like a memory management problem somewhere deep in SE's code, since this problem is related to any memory allocation calls elsewhere in the script. For example, a call to "renew" inside kstat traversing loop will cause nread to be 0 in the next iteration of the loop. "Data fixing" code, dealing with this issue, was removed in revision 392. This patch introduces a new function to re-read affected kstat, instead of ignoring bad data. I can add a few more "if" statements to cover the case when re-read data is still bad, but I think this will be an overkill, since we should trust SE to some degree :-). Modified: trunk/orca/data_gatherers/orcallator/orcallator.se ============================================================================== --- trunk/orca/data_gatherers/orcallator/orcallator.se (original) +++ trunk/orca/data_gatherers/orcallator/orcallator.se Mon Oct 11 12:10:53 2004 @@ -339,6 +339,37 @@ uint _rcnt; // Count of elements in run state }; +// SE appears to have a bug - occasionally kio.nread is erroneously set to 0. +// This function is used to re-read data for a given kstat. +ulong get_new_kstat_data(kstat_t okp[1]) { + ulong ul; + kstat_ctl_t kc[1]; + kstat_t nkp[1]; + kstat_t rkp[1]; + + // Return old data if no match found + rkp = okp; + // Initialize kstat control structure + kc[0] = kstat_open(); + // Traverse the chain looking for matching kstat + for (ul=kc[0].kc_chain; ul!=0; ul=nkp[0].ks_next) { + nkp[0] = *((kstat_t *) ul); + if (nkp[0].ks_type == okp[0].ks_type && + nkp[0].ks_class == okp[0].ks_class && + nkp[0].ks_name == okp[0].ks_name && + nkp[0].ks_instance == okp[0].ks_instance ) { + if (kstat_read(kc, nkp, 0) == -1) { + perror("get_new_kstat_data:kstat_read error"); + exit(1); + } + rkp = nkp; + break; + } + } + kstat_close(kc); + return rkp[0].ks_data; +} + // Define globals for tracking kstat io data. io_dev_info_t ORCA_io_dev_info[]; int ORCA_io_dev_count=0; @@ -408,6 +439,17 @@ } ORCA_io_dev_info[iodev].short_name = nkp[0].ks_name; ORCA_io_dev_info[iodev].dev_class = nkp[0].ks_class; + + // Check if kio data is valid, and re-read kstat if it is not. + // At this time, only kio.nread appears to have occasional problems, + // but we check the other three just in case. + // It is possible for these statistics to be 0, in such case + // kio data will remain the same. + if (kio.writes == 0 || kio.nwritten == 0 || + kio.reads == 0 || kio.nread == 0) { + kio = *((kstat_io_t *) get_new_kstat_data(nkp)); + } + ORCA_io_dev_info[iodev]._writes = kio.writes; ORCA_io_dev_info[iodev]._nwritten = kio.nwritten; ORCA_io_dev_info[iodev]._wlastupdate = kio.wlastupdate; @@ -422,6 +464,15 @@ ORCA_io_dev_info[iodev]._rcnt = kio.rcnt; ORCA_io_dev_count++; } + // Check if kio data is valid, and re-read kstat if it is not. + // At this time, only kio.nread appears to have occasional problems, + // but we check the other three just in case. + if (kio.writes < ORCA_io_dev_info[iodev]._writes || + kio.nwritten < ORCA_io_dev_info[iodev]._nwritten || + kio.reads < ORCA_io_dev_info[iodev]._reads || + kio.nread < ORCA_io_dev_info[iodev]._nread) { + kio = *((kstat_io_t *) get_new_kstat_data(nkp)); + } elapsed_etime = (kio.wlastupdate-ORCA_io_dev_info[iodev]._wlastupdate); if (elapsed_etime == 0) { From dmberezin at hotmail.com Fri Oct 22 09:38:53 2004 From: dmberezin at hotmail.com (dmberezin at hotmail.com) Date: Fri, 22 Oct 2004 09:38:53 -0700 Subject: [Orca-checkins] r399 - trunk/orca/data_gatherers/orcallator Message-ID: <200410221638.i9MGcrIB001252@gw.orcaware.com> Author: dmberezin at hotmail.com Date: Fri Oct 22 09:31:12 2004 New Revision: 399 Modified: trunk/orca/data_gatherers/orcallator/orcallator.se Log: Add optional measurements for disk and tape data collection * data_gatherers/orcallator/orcallator.se Add the following defines to allow flexibility in specifying (at the command line) what disk and tape data should be collected WATCH_IO WATCH_DISK_EXTENDED WATCH_DISK_EXTENDED_TOTALS WATCH_TAPE_EXTENDED WATCH_TAPE_EXTENDED_TOTALS WATCH_IO_EXTENDED WATCH_IO_EXTENDED_TOTALS WATCH_IO_ALL Usage: Define WATCH_IO to collect both disk and tape statistics Define WATCH_DISK_EXTENDED_TOTALS to collect more system-wide disk data Define WATCH_DISK_EXTENDED to collect more data for each disk in the system Define WATCH_TAPE_EXTENDED_TOTALS to collect more system-wide tape data Define WATCH_TAPE_EXTENDED to collect more data for each tape device Define WATCH_IO_EXTENDED to define WATCH_DISK_EXTENDED & WATCH_TAPE_EXTENDED Define WATCH_IO_EXTENDED_TOTALS to define both WATCH_DISK_EXTENDED_TOTALS & WATCH_TAPE_EXTENDED_TOTALS Define WATCH_IO_ALL to define WATCH_IO_EXTENDED & WATCH_IO_EXTENDED_TOTALS (measure_disk): add comments, remove HAVE_EMC_DISK_CONTROL code, gather more system-wide statistics, add the following variables total_disk_avg_wait total_disk_avg_run mean_disk_avg_wait_time mean_disk_avg_serv_time mean_disk_service mean_disk_wait_percent (measure_tape): add comments, gather more system-wide statistics, add these variables mean_tape_busy peak_tape_busy total_tape_avg_wait total_tape_avg_run mean_tape_avg_wait_time mean_tape_avg_serv_time mean_tape_service mean_tape_wait_percent Modified: trunk/orca/data_gatherers/orcallator/orcallator.se ============================================================================== --- trunk/orca/data_gatherers/orcallator/orcallator.se (original) +++ trunk/orca/data_gatherers/orcallator/orcallator.se Fri Oct 22 09:31:12 2004 @@ -38,14 +38,33 @@ #define WATCH_NFS_CLIENT 1 #define WATCH_NFS_SERVER 1 #define WATCH_MOUNTS 1 -#define WATCH_DISK 1 -#define WATCH_TAPE 1 +#define WATCH_IO 1 #define WATCH_DNLC 1 #define WATCH_INODE 1 #define WATCH_RAM 1 #define WATCH_PAGES 1 #endif +#ifdef WATCH_IO +#define WATCH_DISK 1 +#define WATCH_TAPE 1 +#endif + +#ifdef WATCH_IO_ALL +#define WATCH_IO_EXTENDED 1 +#define WATCH_IO_EXTENDED_TOTALS 1 +#endif + +#ifdef WATCH_IO_EXTENDED +#define WATCH_DISK_EXTENDED 1 +#define WATCH_TAPE_EXTENDED 1 +#endif + +#ifdef WATCH_IO_EXTENDED_TOTALS +#define WATCH_DISK_EXTENDED_TOTALS 1 +#define WATCH_TAPE_EXTENDED_TOTALS 1 +#endif + // Keep backwards compatibility with WATCH_HTTPD. #ifdef WATCH_HTTPD #define WATCH_WEB 1 @@ -752,6 +771,7 @@ fprintf(stderr, " -DWATCH_MOUNTS watch usage of mount points\n"); fprintf(stderr, " -DWATCH_DISK watch disk read/write usage\n"); fprintf(stderr, " -DWATCH_TAPE watch tape read/write usage\n"); + fprintf(stderr, " -DWATCH_IO watch disk and tape read/write usage\n"); fprintf(stderr, " -DWATCH_DNLC watch the directory name lookup cache\n"); fprintf(stderr, " -DWATCH_INODE watch the inode cache\n"); fprintf(stderr, " -DWATCH_RAM watch memory usage\n"); @@ -1469,14 +1489,21 @@ // of any changes to the number of disks on the system. int previous_disk_count = -1; - double mean_disk_busy; - double peak_disk_busy; - double total_disk_reads; - double total_disk_writes; - double total_disk_readk; - double total_disk_writek; + double mean_disk_busy; // Avg % of time trans. in progress + double peak_disk_busy; // Max % of time trans. in progress + double total_disk_reads; // Total disk reads per second + double total_disk_writes; // Total disk writes per second + double total_disk_readk; // Total kilobytes read per second + double total_disk_writek; // Total kilobytes written per second +#ifdef WATCH_DISK_EXTENDED_TOTALS + double total_disk_avg_wait; // Total avg trans. waiting for service + double total_disk_avg_run; // Total avg trans. actively being serviced + double mean_disk_avg_wait_time; // Avg service time in wait queue (ms) + double mean_disk_avg_serv_time; // Avg service time active trans. (ms) + double mean_disk_service; // Avg service time (ms) + double mean_disk_wait_percent; // Avg % of time there are trans. waiting for service +#endif // WATCH_DISK_EXTENDED_TOTALS int disk_count; - int i; mean_disk_busy = 0.0; @@ -1485,86 +1512,140 @@ total_disk_writes = 0.0; total_disk_readk = 0.0; total_disk_writek = 0.0; +#ifdef WATCH_DISK_EXTENDED_TOTALS + total_disk_avg_wait = 0.0; + total_disk_avg_run = 0.0; + mean_disk_avg_wait_time = 0.0; + mean_disk_avg_serv_time = 0.0; + mean_disk_service = 0.0; + mean_disk_wait_percent = 0.0; +#endif // WATCH_DISK_EXTENDED_TOTALS disk_count = 0; #ifdef USE_KSTAT_IO for (i=0; i. Check - // [wr]lentime to see if an EMC is using a fake disk for control. - // EMC disks have a fake disk which commands are run over to - // configure the disk array or to get stats from; they are not - // real data transfers. They cause 1000 MB/sec writes to appear - // in the stats. I still get them but not as often with this bit - // of code in. If the I/O which occurred in the last five minutes - // is not greater than 1/100sec then it is not a valid measurement - // anyway. What happens is that we can have a small I/O, say 1024 - // bytes, in a 1/100sec = 1024*100/sec. I am thinking of making - // it wlentime+rlentime>2 since I am still getting fake write - // spikes. -#ifdef HAVE_EMC_DISK_CONTROL - if ((pioGLOB_old_wlentime[i] + pioGLOB_old_rlentime[i]) > 1) { -#endif - total_disk_reads += ORCA_io_dev_info[i].reads; - total_disk_writes += ORCA_io_dev_info[i].writes; - total_disk_readk += ORCA_io_dev_info[i].kreads; - total_disk_writek += ORCA_io_dev_info[i].kwrites; - mean_disk_busy += ORCA_io_dev_info[i].run_percent; - if (ORCA_io_dev_info[i].run_percent > peak_disk_busy) { - peak_disk_busy = ORCA_io_dev_info[i].run_percent; - } -#ifdef HAVE_EMC_DISK_CONTROL - } -#endif +#ifdef WATCH_DISK_EXTENDED + // Reads per second + put_output(sprintf("disk_rd/s_%s", ORCA_io_dev_info[i].long_name), + sprintf("%16.5f", ORCA_io_dev_info[i].reads)); + // Writes per second + put_output(sprintf("disk_wr/s_%s", ORCA_io_dev_info[i].long_name), + sprintf("%16.5f", ORCA_io_dev_info[i].writes)); + // Kilobytes read per second + put_output(sprintf("disk_rK/s_%s", ORCA_io_dev_info[i].long_name), + sprintf("%16.5f", ORCA_io_dev_info[i].kreads)); + // Kilobytes written per second + put_output(sprintf("disk_wK/s_%s", ORCA_io_dev_info[i].long_name), + sprintf("%16.5f", ORCA_io_dev_info[i].kwrites)); + // Average number of transactions waiting for service + put_output(sprintf("disk_wait_%s", ORCA_io_dev_info[i].long_name), + sprintf("%16.5f", ORCA_io_dev_info[i].avg_wait)); + // Average number of transactions actively being serviced + put_output(sprintf("disk_actv_%s", ORCA_io_dev_info[i].long_name), + sprintf("%16.5f", ORCA_io_dev_info[i].avg_run)); + // Average service time in wait queue in milliseconds + put_output(sprintf("disk_wsvct_%s", ORCA_io_dev_info[i].long_name), + sprintf("%16.5f", ORCA_io_dev_info[i].avg_wait_time)); + // Average service time active transactions in milliseconds + put_output(sprintf("disk_asvct_%s", ORCA_io_dev_info[i].long_name), + sprintf("%16.5f", ORCA_io_dev_info[i].avg_serv_time)); + // Percent of time there are transactions waiting for service + put_output(sprintf("disk_waitp_%s", ORCA_io_dev_info[i].long_name), + sprintf("%16.5f", ORCA_io_dev_info[i].wait_percent)); +#endif // WATCH_DISK_EXTENDED + + total_disk_reads += ORCA_io_dev_info[i].reads; + total_disk_writes += ORCA_io_dev_info[i].writes; + total_disk_readk += ORCA_io_dev_info[i].kreads; + total_disk_writek += ORCA_io_dev_info[i].kwrites; + mean_disk_busy += ORCA_io_dev_info[i].run_percent; + if (ORCA_io_dev_info[i].run_percent > peak_disk_busy) { + peak_disk_busy = ORCA_io_dev_info[i].run_percent; + } +#ifdef WATCH_DISK_EXTENDED_TOTALS + total_disk_avg_wait += ORCA_io_dev_info[i].avg_wait; + total_disk_avg_run += ORCA_io_dev_info[i].avg_run; + mean_disk_avg_wait_time += ORCA_io_dev_info[i].avg_wait_time; + mean_disk_avg_serv_time += ORCA_io_dev_info[i].avg_serv_time; + mean_disk_service += ORCA_io_dev_info[i].service; + mean_disk_wait_percent += ORCA_io_dev_info[i].wait_percent; +#endif // WATCH_DISK_EXTENDED_TOTALS } } #else for (i=0; i. Check - // [wr]lentime to see if an EMC is using a fake disk for control. - // EMC disks have a fake disk which commands are run over to - // configure the disk array or to get stats from; they are not - // real data transfers. They cause 1000 MB/sec writes to appear - // in the stats. I still get them but not as often with this bit - // of code in. If the I/O which occurred in the last five minutes - // is not greater than 1/100sec then it is not a valid measurement - // anyway. What happens is that we can have a small I/O, say 1024 - // bytes, in a 1/100sec = 1024*100/sec. I am thinking of making - // it wlentime+rlentime>2 since I am still getting fake write - // spikes. -#ifdef HAVE_EMC_DISK_CONTROL - if ((pioGLOB_old_wlentime[i] + pioGLOB_old_rlentime[i]) > 1) { -#endif - total_disk_reads += GLOBAL_disk[i].reads; - total_disk_writes += GLOBAL_disk[i].writes; - total_disk_readk += GLOBAL_disk[i].kreads; - total_disk_writek += GLOBAL_disk[i].kwrites; - mean_disk_busy += GLOBAL_disk[i].run_percent; - if (GLOBAL_disk[i].run_percent > peak_disk_busy) { - peak_disk_busy = GLOBAL_disk[i].run_percent; - } -#ifdef HAVE_EMC_DISK_CONTROL - } -#endif +#ifdef WATCH_DISK_EXTENDED + // Reads per second + put_output(sprintf("disk_rd/s_%s", GLOBAL_disk[i].info.long_name), + sprintf("%16.5f", GLOBAL_disk[i].reads)); + // Writes per second + put_output(sprintf("disk_wr/s_%s", GLOBAL_disk[i].info.long_name), + sprintf("%16.5f", GLOBAL_disk[i].writes)); + // Kilobytes read per second + put_output(sprintf("disk_rK/s_%s", GLOBAL_disk[i].info.long_name), + sprintf("%16.5f", GLOBAL_disk[i].kreads)); + // Kilobytes written per second + put_output(sprintf("disk_wK/s_%s", GLOBAL_disk[i].info.long_name), + sprintf("%16.5f", GLOBAL_disk[i].kwrites)); + // Average number of transactions waiting for service + put_output(sprintf("disk_wait_%s", GLOBAL_disk[i].info.long_name), + sprintf("%16.5f", GLOBAL_disk[i].avg_wait)); + // Average number of transactions actively being serviced + put_output(sprintf("disk_actv_%s", GLOBAL_disk[i].info.long_name), + sprintf("%16.5f", GLOBAL_disk[i].avg_run)); + // Average service time in wait queue in milliseconds + put_output(sprintf("disk_wsvct_%s", GLOBAL_disk[i].info.long_name), + sprintf("%16.5f", GLOBAL_disk[i].avg_wait_time)); + // Average service time active transactions in milliseconds + put_output(sprintf("disk_asvct_%s", GLOBAL_disk[i].info.long_name), + sprintf("%16.5f", GLOBAL_disk[i].avg_serv_time)); + // Percent of time there are transactions waiting for service + put_output(sprintf("disk_waitp_%s", GLOBAL_disk[i].info.long_name), + sprintf("%16.5f", GLOBAL_disk[i].wait_percent)); +#endif // WATCH_DISK_EXTENDED + + total_disk_reads += GLOBAL_disk[i].reads; + total_disk_writes += GLOBAL_disk[i].writes; + total_disk_readk += GLOBAL_disk[i].kreads; + total_disk_writek += GLOBAL_disk[i].kwrites; + mean_disk_busy += GLOBAL_disk[i].run_percent; + if (GLOBAL_disk[i].run_percent > peak_disk_busy) { + peak_disk_busy = GLOBAL_disk[i].run_percent; + } +#ifdef WATCH_DISK_EXTENDED_TOTALS + total_disk_avg_wait += GLOBAL_disk[i].avg_wait; + total_disk_avg_run += GLOBAL_disk[i].avg_run; + mean_disk_avg_wait_time += GLOBAL_disk[i].avg_wait_time; + mean_disk_avg_serv_time += GLOBAL_disk[i].avg_serv_time; + mean_disk_service += GLOBAL_disk[i].service; + mean_disk_wait_percent += GLOBAL_disk[i].wait_percent; +#endif // WATCH_DISK_EXTENDED_TOTALS } #endif if (disk_count != 0) { mean_disk_busy = mean_disk_busy/disk_count; +#ifdef WATCH_DISK_EXTENDED_TOTALS + mean_disk_avg_wait_time = mean_disk_avg_wait_time / disk_count; + mean_disk_avg_serv_time = mean_disk_avg_serv_time / disk_count; + mean_disk_service = mean_disk_service / disk_count; + mean_disk_wait_percent = mean_disk_wait_percent / disk_count; +#endif // WATCH_DISK_EXTENDED_TOTALS } put_output("disk_peak", sprintf("%9.3f", peak_disk_busy)); @@ -1573,6 +1654,14 @@ put_output("disk_wr/s", sprintf("%9.1f", total_disk_writes)); put_output("disk_rK/s", sprintf("%9.1f", total_disk_readk)); put_output("disk_wK/s", sprintf("%9.1f", total_disk_writek)); +#ifdef WATCH_DISK_EXTENDED_TOTALS + put_output("disk_wait", sprintf("%9.3f", total_disk_avg_wait)); + put_output("disk_actv", sprintf("%9.3f", total_disk_avg_run)); + put_output("disk_wsvct", sprintf("%9.3f", mean_disk_avg_wait_time)); + put_output("disk_asvct", sprintf("%9.3f", mean_disk_avg_serv_time)); + put_output("disk_svct", sprintf("%9.3f", mean_disk_service)); + put_output("disk_waitp", sprintf("%9.3f", mean_disk_wait_percent)); +#endif // WATCH_DISK_EXTENDED_TOTALS // If the number of disks has changed, say due to a // add_drv, then print new headers. @@ -1590,36 +1679,121 @@ // of any changes to the number of tapes on the system. int previous_tape_count = -1; - double total_tape_reads; - double total_tape_writes; - double total_tape_readk; - double total_tape_writek; + double mean_tape_busy; // Avg % of time trans. in progress + double peak_tape_busy; // Max % of time trans. in progress + double total_tape_reads; // Total tape reads per second + double total_tape_writes; // Total tape writes per second + double total_tape_readk; // Total kilobytes read per second + double total_tape_writek; // Total kilobytes written per second +#ifdef WATCH_TAPE_EXTENDED_TOTALS + double total_tape_avg_wait; // Total avg trans. waiting for service + double total_tape_avg_run; // Total avg trans. actively being serviced + double mean_tape_avg_wait_time; // Avg service time in wait queue (ms) + double mean_tape_avg_serv_time; // Avg service time active trans. (ms) + double mean_tape_service; // Avg service time (ms) + double mean_tape_wait_percent; // Avg % of time there are trans. waiting for service +#endif // WATCH_TAPE_EXTENDED_TOTALS int tape_count; - int i; + mean_tape_busy = 0.0; + peak_tape_busy = 0.0; total_tape_reads = 0.0; total_tape_writes = 0.0; total_tape_readk = 0.0; total_tape_writek = 0.0; +#ifdef WATCH_TAPE_EXTENDED_TOTALS + total_tape_avg_wait = 0.0; + total_tape_avg_run = 0.0; + mean_tape_avg_wait_time = 0.0; + mean_tape_avg_serv_time = 0.0; + mean_tape_service = 0.0; + mean_tape_wait_percent = 0.0; +#endif // WATCH_TAPE_EXTENDED_TOTALS tape_count = 0; for (i=0; i peak_tape_busy) { + peak_tape_busy = ORCA_io_dev_info[i].run_percent; + } +#ifdef WATCH_TAPE_EXTENDED_TOTALS + total_tape_avg_wait += ORCA_io_dev_info[i].avg_wait; + total_tape_avg_run += ORCA_io_dev_info[i].avg_run; + mean_tape_avg_wait_time += ORCA_io_dev_info[i].avg_wait_time; + mean_tape_avg_serv_time += ORCA_io_dev_info[i].avg_serv_time; + mean_tape_service += ORCA_io_dev_info[i].service; + mean_tape_wait_percent += ORCA_io_dev_info[i].wait_percent; +#endif // WATCH_TAPE_EXTENDED_TOTALS } } + if (tape_count != 0) { + mean_tape_busy = mean_tape_busy/tape_count; +#ifdef WATCH_TAPE_EXTENDED_TOTALS + mean_tape_avg_wait_time = mean_tape_avg_wait_time / tape_count; + mean_tape_avg_serv_time = mean_tape_avg_serv_time / tape_count; + mean_tape_service = mean_tape_service / tape_count; + mean_tape_wait_percent = mean_tape_wait_percent / tape_count; +#endif // WATCH_TAPE_EXTENDED_TOTALS + } + + put_output("tape_peak", sprintf("%9.3f", peak_tape_busy)); + put_output("tape_mean", sprintf("%9.3f", mean_tape_busy)); put_output("tape_rd/s", sprintf("%9.1f", total_tape_reads)); put_output("tape_wr/s", sprintf("%9.1f", total_tape_writes)); put_output("tape_rK/s", sprintf("%9.1f", total_tape_readk)); put_output("tape_wK/s", sprintf("%9.1f", total_tape_writek)); +#ifdef WATCH_TAPE_EXTENDED_TOTALS + put_output("tape_wait", sprintf("%9.3f", total_tape_avg_wait)); + put_output("tape_actv", sprintf("%9.3f", total_tape_avg_run)); + put_output("tape_wsvct", sprintf("%9.3f", mean_tape_avg_wait_time)); + put_output("tape_asvct", sprintf("%9.3f", mean_tape_avg_serv_time)); + put_output("tape_svct", sprintf("%9.3f", mean_tape_service)); + put_output("tape_waitp", sprintf("%9.3f", mean_tape_wait_percent)); +#endif // WATCH_TAPE_EXTENDED_TOTALS // If the number of tapes has changed, say due to a // add_drv, then print new headers.