← Back to index

Issues Sending Large Files Over TCP Localhost

Lachlan
#26983

I'm trying to copy a files using TCP. I only intend for this to be over localhost (so MTU is 65536 as given by lo interface from ifconfig). It works for small files (less than 32K it seems), not for large files (it appends ^@ characters to file).

Inspecting with vbindiff, differences in large files appear at 32K/0x8000 mark. I'm not sure why this is.

Example files are generated with:

base64 /dev/urandom | head -c $((1024 * 30)) > test-small.txt
base64 /dev/urandom | head -c $((1024 * 180)) > test-large.txt

Consider source files server.c and client.c. I'm sorry, I know these files appear long and unwieldy, but that is just due to error checking if indentation with socket setup and basic read/write helper functions that can largely be ignored (trying to provide a minimal reproducible example). The relevant parts are only a few lines in the middle of main() for both.

server.c

#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <errno.h>
#include <time.h>
#include <stdbool.h>
#include <ctype.h>
#include <assert.h>
#include <signal.h>
#include <stdarg.h>

#include <arpa/inet.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/prctl.h>
#include <sys/mman.h>
#include <sys/wait.h>
#include <unistd.h>
#include <fcntl.h>


typedef int64_t s64;
typedef uint8_t u8;
typedef uint32_t u32;
typedef uint64_t u64;

void
write_entire_file(char *file_name, void *buf, u64 buf_size)
{
  int file_fd = open(file_name, O_CREAT | O_TRUNC | O_WRONLY, 0666); 
  if (file_fd != -1) 
  {
    u8 *byte_location = (u8 *)buf;
    u32 bytes_to_write = buf_size;

    while (bytes_to_write > 0) 
    {
      int write_res = write(file_fd, byte_location, bytes_to_write); 
      if (write_res != -1) 
      {
        bytes_to_write -= write_res;
        byte_location += write_res;
      }
      else
      {
        fprintf(stderr, "Error: unable to write file %s (%s)\n", file_name, strerror(errno));
        break;
      }
    }

    close(file_fd);
  }
  else
  {
    fprintf(stderr, "Error: unable to open file %s (%s)\n", file_name, strerror(errno));
  }
}

void
readx(int fd, void *buf, size_t count)
{
  int bytes_read = read(fd, buf, count);
  if (bytes_read == -1)
  {
    fprintf(stderr, "Error: read failed (%s)\n", strerror(errno));
    exit(1);
  }
  if (bytes_read != count)
  {
    fprintf(stderr, "Error: read failed to read all bytes in one go(%s)\n", strerror(errno));
    exit(1);
  }
}

void *
mallocx(size_t size)
{
  void *result = NULL;

  result = malloc(size);
  if (result == NULL)
  {
    fprintf(stderr, "Error: malloc failed (%s)\n", strerror(errno));
    exit(1);
  }

  return result;
}

#define MTU 16384

typedef struct 
{
  u32 file_size;
  u32 contents_size;
  char contents[MTU];
} FileMessage;

int
main(int argc, char *argv[])
{
  int server_sock = socket(AF_INET, SOCK_STREAM, 0);
  if (server_sock != -1)
  {
    int opt_val = 1;
    if (setsockopt(server_sock, SOL_SOCKET, SO_REUSEADDR, (void *)&opt_val, 
          sizeof(opt_val)) == -1)
    {
      fprintf(stderr, "Warning: unable to set resuable socket (%s)\n", strerror(errno));
    }

    struct sockaddr_in server_addr = {0};
    server_addr.sin_family = AF_INET;
    server_addr.sin_addr.s_addr = htonl(INADDR_ANY);
    server_addr.sin_port = htons(8000);

    if (bind(server_sock, (struct sockaddr *)&server_addr, sizeof(server_addr)) != -1)
    {
      u32 max_num_connections = 100;
      if (listen(server_sock, max_num_connections) != -1)
      {
        while (true)
        {
          struct sockaddr_in client_addr = {0}; 
          u32 client_size = sizeof(client_addr);
          int client_fd = accept(server_sock, (struct sockaddr *)&client_addr, &client_size);
          if (client_fd != -1)
          {
            FileMessage file_message = {0}; 
            readx(client_fd, &file_message, sizeof(file_message)); 

            u32 byte_counter = 0;
            void *file_mem = mallocx(file_message.file_size);
            u8 *file_cursor = file_mem;
            u32 file_size = file_message.file_size;

            memcpy(file_cursor, file_message.contents, file_message.contents_size);
            byte_counter += file_message.contents_size;

            fprintf(stderr, "file size left: %d, content size: %d \n", (file_message.file_size - byte_counter), file_message.contents_size);

            while (byte_counter != file_size)
            {
              file_cursor += byte_counter; 

              readx(client_fd, &file_message, sizeof(file_message)); 

              memcpy(file_cursor, file_message.contents, file_message.contents_size);

              byte_counter += file_message.contents_size;

              fprintf(stderr, "file size left: %d, content size: %d \n", (file_message.file_size - byte_counter), file_message.contents_size);
            }

            write_entire_file("recieved-tcp-file", file_mem, file_message.file_size);

            free(file_mem);
          }
          else
          {
            fprintf(stderr, "Error: unable to accept connection\n");
          }
        }
      }
      else
      {
        fprintf(stderr, "Error: unable to listen on socket\n");
      }
    }
    else
    {
      fprintf(stderr, "Error: unable to bind on socket\n");
    }
  }
  else
  {
    fprintf(stderr, "Error: unable to open socket\n");
  }

  return 0;
}

client.c

#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <errno.h>
#include <time.h>
#include <stdbool.h>
#include <ctype.h>
#include <assert.h>
#include <signal.h>
#include <stdarg.h>

#include <arpa/inet.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/prctl.h>
#include <sys/mman.h>
#include <sys/wait.h>
#include <unistd.h>
#include <fcntl.h>


typedef int64_t s64;
typedef uint8_t u8;
typedef uint32_t u32;
typedef uint64_t u64;

typedef int64_t s64;
typedef uint8_t u8;
typedef uint32_t u32;

void
sleep_ms(int ms)
{
  struct timespec sleep_time = {0};
  sleep_time.tv_nsec = ms * 1000000;
  struct timespec leftover_sleep_time = {0};
  nanosleep(&sleep_time, &leftover_sleep_time);
}

void
writex(int fd, void *buf, size_t count)
{
  int bytes_written = write(fd, buf, count);
  if (bytes_written == -1)
  {
    fprintf(stderr, "Error: write failed (%s)\n", strerror(errno));
    exit(1);
  }
  if (bytes_written != count)
  {
    fprintf(stderr, "Warning: write failed to write all bytes in one go(%s)\n", strerror(errno));
    exit(1);
  }
}

typedef struct 
{
  void *contents;
  u32 size;
} ReadFileResult;

ReadFileResult
read_entire_file(char *file_name)
{
  ReadFileResult result = {0};

  int file_fd = open(file_name, O_RDONLY); 
  if (file_fd != -1) 
  {
    struct stat file_stat = {0};
    int fstat_res = fstat(file_fd, &file_stat);
    if (fstat_res != -1) 
    {
      result.contents = malloc(file_stat.st_size);
      if (result.contents != NULL)
      {
        result.size = file_stat.st_size;
        size_t bytes_to_read = file_stat.st_size;
        u8 *byte_location = (u8 *)result.contents;
        while (bytes_to_read > 0) 
        {
          int read_res = read(file_fd, byte_location, bytes_to_read); 
          if (read_res != -1) 
          {
            bytes_to_read -= read_res;
            byte_location += read_res;
          }
          else
          {
            fprintf(stderr, "Error: unable to read file %s (%s)\n", file_name, strerror(errno));
            free(result.contents);
            break;
          }
        }
      }
      else
      {
        fprintf(stderr, "Error: unable to malloc memory for file %s (%s)\n", file_name, strerror(errno));
      }
    }
    else
    {
      fprintf(stderr, "Error: unable to fstat file %s (%s)\n", file_name, strerror(errno));
    }
  }
  else
  {
    fprintf(stderr, "Error: unable to open file %s (%s)\n", file_name, strerror(errno));
  }
    
  return result;
}

#define MTU 16384

typedef struct 
{
  u32 file_size;
  u32 contents_size;
  char contents[MTU];
} FileMessage;

int
main(int argc, char *argv[])
{
  int server_sock = socket(AF_INET, SOCK_STREAM, 0);
  if (server_sock != -1)
  {
    int opt_val = 1;
    if (setsockopt(server_sock, SOL_SOCKET, SO_REUSEADDR, (void *)&opt_val, 
          sizeof(opt_val)) == -1)
    {
      fprintf(stderr, "Warning: unable to set resuable socket (%s)\n", strerror(errno));
    }

    struct sockaddr_in server_addr = {0};
    if (inet_pton(AF_INET, "127.0.0.1", &server_addr.sin_addr) == 1)
    {
      server_addr.sin_family = AF_INET;
      server_addr.sin_port = htons(8000);

      if (connect(server_sock, (struct sockaddr *)&server_addr, sizeof(server_addr)) != -1)
      {
        char *file_name = "test-large.txt";
        ReadFileResult file_res = read_entire_file(file_name);
        if (file_res.contents != NULL)
        {
          FileMessage file_message = {0};

          file_message.file_size = file_res.size;

          s64 file_size_left = file_res.size;
          u8 *file_cursor = (u8 *)file_res.contents;
          while (file_size_left != 0)
          {
            if (file_size_left - MTU >= 0)
            {
              memcpy(file_message.contents, file_cursor, MTU);

              file_message.contents_size = MTU;
              file_cursor += MTU;
              file_size_left -= MTU;
              writex(server_sock, &file_message, sizeof(file_message));

              // artificially throttle sending of bytes
              sleep_ms(100);

              fprintf(stderr, "file size left: %ld\n", file_size_left);
            }
            else
            {
              memcpy(file_message.contents, file_cursor, file_size_left);

              file_message.contents_size = file_size_left;
              writex(server_sock, &file_message, sizeof(file_message));
              file_size_left = 0;

              fprintf(stderr, "file size left: %ld\n", file_size_left);
            }
          }

          free(file_res.contents);
        }
        else
        {
          fprintf(stderr, "Error: unable to read file %s\n", file_name);
        }
      }
      else
      {
        fprintf(stderr, "Error: failed to connect (%s)\n", strerror(errno));
      }
    }
    else
    {
      fprintf(stderr, "Error: invalid IP address provided (%s)\n", strerror(errno));
    }
  }
  else
  {
    fprintf(stderr, "Error: unable to create server socket (%s)\n", strerror(errno));
  }

  return 0;
}
Mārtiņš Možeiko
#26984

In server.c this line: file_cursor += byte_counter; should actually be this:

file_cursor = file_mem + byte_counter;
Edited by Mārtiņš Možeiko on
Lachlan
#26985

Thank you so much Mārtiņš. Strange, why this was not segfaulting.

BTW Mārtiņš do you have a patreon or something similar. You've helped me out a lot on this site. I feel in some degree indebted to you

Replying to mmozeiko (#26984)
Reply to Thread
← Back to index