Amazon kernel's scheduler does not schedule tasks frequently enough

[glommer]

Running the following test program, we can see how often we can schedule to a new thread

#include <sys/timerfd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <assert.h>
#include <iostream>
#include <chrono>
#include <thread>
#include <math.h>

int testfd;

inline
void pin_this_thread(unsigned cpu_id) {
    cpu_set_t cs;
    CPU_ZERO(&cs);
    CPU_SET(cpu_id, &cs);
    auto r = pthread_setaffinity_np(pthread_self(), sizeof(cs), &cs);
    assert(r == 0);
}

void
test_timer_thread_fn() {
    pin_this_thread(0);
    auto last = std::chrono::steady_clock::now();
    auto count = 10;
    while (count) {
        uint64_t events;
        read(testfd, &events, 8);
        count--;
        auto now = std::chrono::steady_clock::now();
        auto delta = std::chrono::duration_cast<std::chrono::microseconds>(now - last);
        last = now;
        std::cout << delta.count() << "us" << std::endl;
    }
}

int main() {
    pin_this_thread(0);
    std::thread timer(test_timer_thread_fn);
    testfd = timerfd_create(CLOCK_MONOTONIC, TFD_CLOEXEC);
    assert(testfd > 0);
    itimerspec its;
    its.it_value = timespec{0, 250000};
    its.it_interval = timespec{0, 250000};
    auto r = timerfd_settime(testfd, 0, &its, nullptr);
    assert(r == 0);
    for (;;) {
        r = exp(r + 1);
    }
    timer.join();
    return 0;
}

In a properly configured system, we see messages print at roughly every 250usec. With the Amazon kernel, even with the configurations applied from the scylla-conf package, the timer thread runs in much longer intervals.

[tzach]

@glommer is this still relevant?

[glommer]

I have no idea.