I'm on a laptop with 2 connected keyboards (built-in and USB). I'm obtaining these connected keyboards with libudev and using epoll to poll them for input via the evdev interface:

// Compile with $(gcc udev.c -ludev)

#include <stdbool.h>
#include <stdio.h>
#include <string.h>

#include <sys/epoll.h>
#include <sys/poll.h>
#include <sys/types.h>
#include <fcntl.h>
#include <unistd.h>
#include <linux/input.h>
#include <time.h>
#include <libudev.h>

#define BILLION 1000000000L

long timespec_diff(struct timespec *start, struct timespec *end)
{
  return (BILLION * (end->tv_sec - start->tv_sec)) +
         (end->tv_nsec - start->tv_nsec);
}

bool want_to_run = true;

int
main(int argc, char *argv[])
{
  int epoll_fd = epoll_create1(0);

  struct udev *udev_obj = udev_new();
  struct udev_enumerate *udev_enum = udev_enumerate_new(udev_obj);
  udev_enumerate_add_match_subsystem(udev_enum, "input");
  udev_enumerate_scan_devices(udev_enum);

  struct udev_list_entry *udev_entries = udev_enumerate_get_list_entry(udev_enum);
  struct udev_list_entry *udev_entry = NULL;
  udev_list_entry_foreach(udev_entry, udev_entries)
  {
    char const *udev_entry_syspath = udev_list_entry_get_name(udev_entry);
    struct udev_device *device = udev_device_new_from_syspath(udev_obj, 
                                                              udev_entry_syspath);

    char const *dev_prop = \
      udev_device_get_property_value(device, "ID_INPUT_KEYBOARD");
    if (dev_prop != NULL && strcmp(dev_prop, "1") == 0) 
    {
      const char *dev_path = udev_device_get_devnode(device);
      if (dev_path != NULL)
      {
        int dev_fd = open(dev_path, O_RDWR | O_NONBLOCK);

        struct epoll_event event = {};
        event.events = EPOLLIN;
        event.data.fd = dev_fd;
        epoll_ctl(epoll_fd, EPOLL_CTL_ADD, dev_fd, &event);
      }
    }
    udev_device_unref(device);
  }
  udev_enumerate_unref(udev_enum);

  struct timespec prev_timespec = {};
  clock_gettime(CLOCK_MONOTONIC_RAW, &prev_timespec);
  while (want_to_run)
  {
    struct epoll_event epoll_events[5] = {0};
    int num_epoll_events = epoll_wait(epoll_fd, epoll_events, 5, 0);
    for (int epoll_event_i = 0; epoll_event_i < num_epoll_events; ++epoll_event_i)
    {
      int dev_fd = epoll_events[epoll_event_i].data.fd;

      struct input_event dev_events[4] = {0};
      int dev_event_bytes_read = read(dev_fd, dev_events, sizeof(dev_events));

      int num_dev_events = dev_event_bytes_read / sizeof(dev_events[0]); 
      for (int dev_event_i = 0; dev_event_i < num_dev_events; ++dev_event_i)
      {
        int dev_event_type = dev_events[dev_event_i].type;
        int dev_event_code = dev_events[dev_event_i].code;
        int dev_event_value = dev_events[dev_event_i].value;

        bool is_released = (dev_event_type == EV_KEY ? dev_event_value == 0 : false);
        bool is_down = (dev_event_type == EV_KEY ? dev_event_value == 1 : false);
        bool was_down = (dev_event_type == EV_KEY ? dev_event_value == 2 : false);

        bool w = (dev_event_code == KEY_W);
        bool a = (dev_event_code == KEY_A);
        bool s = (dev_event_code == KEY_S);
        bool d = (dev_event_code == KEY_D);
        bool q = (dev_event_code == KEY_Q);
        bool e = (dev_event_code == KEY_E);
        bool up = (dev_event_code == KEY_UP);
        bool down = (dev_event_code == KEY_DOWN);
        bool left = (dev_event_code == KEY_LEFT);
        bool right = (dev_event_code == KEY_RIGHT);
        bool escape = (dev_event_code == KEY_ESC);
        bool space = (dev_event_code == KEY_SPACE);
        bool enter = (dev_event_code == KEY_ENTER);
        bool ctrl = (dev_event_code == KEY_LEFTCTRL);
        if (q) want_to_run = false;
      }
    }

    struct timespec end_timespec = {};
    clock_gettime(CLOCK_MONOTONIC_RAW, &end_timespec);
    printf("ns per frame: %lu\n", timespec_diff(&prev_timespec, &end_timespec)); 
    prev_timespec = end_timespec;
  }

  return 0;
}

Experimenting by entering keys on each keyboard I experience some serious lag/stall in the following circumstances (I encourage you to compile and try yourself):

1. If I start entering keys on one keyboard and then switch to the other, the program stalls briefly.
2. If I simultaneously enter keys on each keyboard the program stalls indefinitely until I stop entering keys.

What is going on here?

I run the program and do not see any stalls when I press keys on multiple connected keyboards.

Interesting...

Thanks for testing it out Mārtiņš.

I think it may be specific to my setup as when I spam keys on each keyboard inside another program, e.g. gnome terminal, firefox, etc. the same stalling and delay happens. If I wail on the keys it's some serious stall (the cpu fan goes off and everything) It only happens for keyboards it seems as moving an external mouse and trackpad together causes no issues.

I'm at a loss as to what is going on or even how to debug this. I've replaced keyboards and still the same issues happen.

I think this may be an issue with xorg, as running in a virtual terminal I don't get any stalling. What is your environment you tested it on?

It was on laptop + usb keyboard. Running ArchLinux with Sway/wayland.

Although I cannot say definitively I believe the source of the issue lies in GNOME not Xorg. See GNOME issue 1 GNOME issue 2

Installing an alternative X environment such as xfce4, sudo apt install xfce4 removes this stalling.