Charm++ parallel programming system

["Sameer Kumar12" <sameer_km AT in.ibm.com>]

CqsPrioGT blows up in NAMD startup on the pami-linux-ppc64le-smp path on 800 worker threads. We may have to add fine grained locking or atomics to the priority queue path?

Sameer Kumar12---06/12/2017 02:14:00 PM---Hi, In Charm 6.7.1, I see node queue priority is compared with the local scheduler priority queue.

From: Sameer Kumar12/India/IBM
To: "Charm Mailing List" <charm AT cs.illinois.edu>
Date: 06/12/2017 02:14 PM
Subject: Node Priority Queue Broken?

---

Hi,

        In Charm 6.7.1, I see node queue priority is compared with the local scheduler priority queue. This will enable high priority PE messages get executed before the low priority node messages?   Here is the code that compares the priorities

          if (NULL!=(msg=CmiGetNonLocalNodeQ())) return msg;
          if (!CqsEmpty(s->nodeQ)
              && CqsPrioGT(CqsGetPriority(s->schedQ),
                           CqsGetPriority(s->nodeQ))) {                //The CqsPrioGT routine can change the priority queue state but is not protected by the trylock block
            if(CmiTryLock(s->nodeLock) == 0) {
              CqsDequeue(s->nodeQ,(void **)&msg);
              CmiUnlock(s->nodeLock);
              if (msg!=NULL) return msg;
            }
          }

        The CqsPrioGT routine looks like this

int CqsPrioGT(_prio prio1, _prio prio2)
{
#ifndef FASTQ
  unsigned int ints1 = prio1->ints;
  unsigned int ints2 = prio2->ints;
#endif
  unsigned int *data1 = prio1->data;
  unsigned int *data2 = prio2->data;
#ifndef FASTQ
  unsigned int val1;
  unsigned int val2;
#endif
  while (1) {
#ifndef FASTQ
    if (ints1==0) return 0;
    if (ints2==0) return 1;
#else
    if (prio1->ints==0) return 0;
    if (prio2->ints==0) return 1;
#endif
#ifndef FASTQ
    val1 = *data1++;                    //priority queue state altered without a lock
    val2 = *data2++;                    //priority queue state altered without a lock
    if (val1 < val2) return 0;
    if (val1 > val2) return 1;
    ints1--;
    ints2--;
#else
    if(*data1++ < *data2++) return 0;
    if(*data1++ > *data2++) return 1;
    (prio1->ints)--;
    (prio2->ints)--;
#endif
  }
}