Thread performing a write does not wake up fast enough on RHEL-6

Solution Verified - Updated -

Issue

  • A program has 2 thread, one reads from a mmapped file while the other writes to it. There is a difference in the way these two threads are scheduled in RHEL-5 and RHEL-6.
  • Delays can reach up to 1 second or more when there is high I/O load on the server.
  • Following program is an example for this:

/*-----------------------------------------------------------------------*/
#include <stdio.h>
#include <err.h>
#include <unistd.h>
#include <pthread.h>
#include <errno.h>
#include <time.h>
#include <sys/time.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>

void* thread_runhigh(void *arg);
void* thread_runlow(void *arg);
static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
char* mem_shared;
int rw_index = 0;
int count=0;
int main()
{
     int res;
     pthread_t thread_high;
     pthread_t thread_mid;
     void *status;
     int fd;
     char *tmp;
     char name[1000];

     sprintf(name, "/tmp/pthreadmmap.share%d", getpid());
     printf ("create file %s\n", name);
     fd = open(name, O_RDWR | O_CREAT,
               S_IRUSR|S_IWUSR| S_IRGRP| S_IROTH);
     tmp = (char*) malloc(300*1000);
     memset(tmp, 0, 300*1000);
     write(fd, tmp, 300*1000);
     mem_shared = mmap(NULL, 300*1000, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);

     res = pthread_create(&thread_high, NULL, thread_runhigh, NULL);
     thread_runlow(NULL);

     pthread_join(thread_high, status);

     munmap(mem_shared, 300*1000);
     close(fd);
     return 0;
}

void* thread_runhigh(void *arg)
{
     struct timeval pre;    
     volatile int a = 0;
     printf("start run high\n");
     while(1)
     {
          a++;    
          struct timeval time;
            time.tv_sec  = 0;
            time.tv_usec = 10000;
            (void)select( 0, NULL, NULL, NULL, &time );
          pthread_mutex_lock(&mutex);
          count++;
          gettimeofday(&pre, NULL);
           char temp[300];
          memcpy(mem_shared, temp, 300);
          rw_index += 300;
          rw_index = rw_index%(300*1000);    
          pthread_mutex_unlock(&mutex);

          struct timeval now;    
          gettimeofday(&now, NULL);
          long dela = now.tv_sec * 1000000 + now.tv_usec - pre.tv_sec*1000000 - pre.tv_usec;
          if ( dela > 1000) {
               printf("high task:(%ld) %d runs\n", dela, count);
          }
     }
}

void* thread_runlow(void *arg)
{

     struct timeval pre;    
     volatile int a = 0;
     printf("start run low in main\n");
     while(1)
     {
          //read
          pthread_mutex_lock(&mutex);
          gettimeofday(&pre, NULL);
     
           char temp[300];
          memcpy(temp , mem_shared, 300);

          struct timeval now;    
          gettimeofday(&now, NULL);
          long dela = now.tv_sec * 1000000 + now.tv_usec - pre.tv_sec*1000000 - pre.tv_usec;
          if ( dela > 1000*20) {
               printf("low task:(%ld)\n", dela);
          }
          pthread_mutex_unlock(&mutex);
     }
}

/*-----------------------------------------------------------------------*/

Environment

  • Red Hat Enterprise Linux 6
  • kernel

Subscriber exclusive content

A Red Hat subscription provides unlimited access to our knowledgebase, tools, and much more.

Current Customers and Partners

Log in for full access

Log In

New to Red Hat?

Learn more about Red Hat subscriptions

Using a Red Hat product through a public cloud?

How to access this content