Issue

• A program has 2 thread, one reads from a mmapped file while the other writes to it. There is a difference in the way these two threads are scheduled in RHEL-5 and RHEL-6.
• Delays can reach up to 1 second or more when there is high I/O load on the server.
• Following program is an example for this:

/*-----------------------------------------------------------------------*/
#include <stdio.h>
#include <err.h>
#include <unistd.h>
#include <pthread.h>
#include <errno.h>
#include <time.h>
#include <sys/time.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>

void* thread_runhigh(void *arg);
void* thread_runlow(void *arg);
static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
char* mem_shared;
int rw_index = 0;
int count=0;
int main()
{
     int res;
     pthread_t thread_high;
     pthread_t thread_mid;
     void *status;
     int fd;
     char *tmp;
     char name[1000];

     sprintf(name, "/tmp/pthreadmmap.share%d", getpid());
     printf ("create file %s\n", name);
     fd = open(name, O_RDWR | O_CREAT,
               S_IRUSR|S_IWUSR| S_IRGRP| S_IROTH);
     tmp = (char*) malloc(300*1000);
     memset(tmp, 0, 300*1000);
     write(fd, tmp, 300*1000);
     mem_shared = mmap(NULL, 300*1000, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);

     res = pthread_create(&thread_high, NULL, thread_runhigh, NULL);
     thread_runlow(NULL);

     pthread_join(thread_high, status);

     munmap(mem_shared, 300*1000);
     close(fd);
     return 0;
}

void* thread_runhigh(void *arg)
{
     struct timeval pre;    
     volatile int a = 0;
     printf("start run high\n");
     while(1)
     {
          a++;    
          struct timeval time;
            time.tv_sec  = 0;
            time.tv_usec = 10000;
            (void)select( 0, NULL, NULL, NULL, &time );
          pthread_mutex_lock(&mutex);
          count++;
          gettimeofday(&pre, NULL);
           char temp[300];
          memcpy(mem_shared, temp, 300);
          rw_index += 300;
          rw_index = rw_index%(300*1000);    
          pthread_mutex_unlock(&mutex);

          struct timeval now;    
          gettimeofday(&now, NULL);
          long dela = now.tv_sec * 1000000 + now.tv_usec - pre.tv_sec*1000000 - pre.tv_usec;
          if ( dela > 1000) {
               printf("high task:(%ld) %d runs\n", dela, count);
          }
     }
}

void* thread_runlow(void *arg)
{

     struct timeval pre;    
     volatile int a = 0;
     printf("start run low in main\n");
     while(1)
     {
          //read
          pthread_mutex_lock(&mutex);
          gettimeofday(&pre, NULL);
     
           char temp[300];
          memcpy(temp , mem_shared, 300);

          struct timeval now;    
          gettimeofday(&now, NULL);
          long dela = now.tv_sec * 1000000 + now.tv_usec - pre.tv_sec*1000000 - pre.tv_usec;
          if ( dela > 1000*20) {
               printf("low task:(%ld)\n", dela);
          }
          pthread_mutex_unlock(&mutex);
     }
}

/*-----------------------------------------------------------------------*/