gistfile1.txt

package main

import (
	"fmt"
	"github.com/prometheus/procfs"
	"golang.org/x/sys/unix"
)

func main() {
	// $ uname --all
	// Linux fedora 6.4.13-100.fc37.x86_64 #1 SMP PREEMPT_DYNAMIC Wed Aug 30 16:43:51 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux

	// What are the minimum, default, and maximum bytes allocated to a socket socket receive buffer? The default value
	// (128 KiB) supersedes net.core.rmem_default. The maximum value (6 MiB) does not override net.core.rmem_max.
	// $ sysctl net.ipv4.tcp_rmem
	// net.ipv4.tcp_rmem = 4096        131072  6291456

	// What are the minimum, default, and maximum bytes allocated to a socket socket receive buffer? The default value
	// (16 KiB) supersedes net.core.wmem_default. The maximum value (4 MiB) does not override net.core.wmem_max.
	// $ sysctl net.ipv4.tcp_wmem
	// net.ipv4.tcp_wmem = 4096        16384   4194304

	// What is the default size (bytes) of the window for receiving TCP data?
	// $ sysctl net.core.rmem_default
	// net.core.rmem_default = 212992

	// What is the maximum size (bytes) of the window for receiving TCP data?
	// $ sysctl net.core.rmem_max
	// net.core.rmem_max = 212992

	// What is the default size (bytes) of the window for transmitting TCP data?
	// $ sysctl net.core.wmem_default
	// net.core.wmem_default = 212992

	// What is the max size (bytes) of the window for transmitting TCP data?
	// $ sysctl net.core.wmem_max
	// net.core.wmem_max = 212992

	TCPIPv4Connection()

	//	testTCPIPv4Socket()
	//	testTCPIPv6Socket()
	//	testUDPIPv4Socket()
	//	testUDPIPv6Socket()
}

func TCPIPv4Connection() {
	fmt.Println("Testing TCP/IPv4 Connections")
	printSockStat()
	// Create the Socket for the service
	serviceFd, err := unix.Socket(unix.AF_INET, unix.SOCK_STREAM, unix.IPPROTO_IP)
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Println("Created Service Socket. FD:", serviceFd)
	// Number of allocated sockets increase by 1. Send and receive buffer match tcp_rmem & tcp_wmem
	fmt.Println("Service Socket memory")
	printBufferSizes(serviceFd)
	printSockStat()

	serviceAddr := &unix.SockaddrInet4{
		Port: 8080,
		Addr: [4]byte{127, 0, 0, 1},
	}

	// Bind to 127.0.0.1:8080
	if err = unix.Bind(serviceFd, serviceAddr); err != nil {
		fmt.Println(err)
		return
	}
	fmt.Println("Service socket bound to 127.0.0.1:8080")
	// No change in metrics.
	printSockStat()

	if err = unix.Listen(serviceFd, 1024); err != nil {
		fmt.Println(err)
		return
	}
	fmt.Println("Service listening.")
	// Number of inuse sockets increase by 1.
	printSockStat()

	clientFd, err := unix.Socket(unix.AF_INET, unix.SOCK_STREAM, unix.IPPROTO_TCP)
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Println("Created client socket. FD:", clientFd)
	// Number of allocated sockets increase by 1. Send and receive buffer match tcp_rmem & tcp_wmem
	fmt.Println("Client Socket memory")
	printBufferSizes(clientFd)
	printSockStat()

	if err = unix.Connect(clientFd, serviceAddr); err != nil {
		fmt.Println(err)
		return
	}
	fmt.Println("Established connection")
	// Number of allocated sockets increase by 1. Number of inuse sockets increase by 2. Client send buffer increases
	// from 16384 to 2626560.
	fmt.Println("Service Socket memory")
	printBufferSizes(serviceFd)
	fmt.Println("Client Socket memory")
	printBufferSizes(clientFd)
	printSockStat()

	acceptedFd, acceptedAddr, err := unix.Accept(serviceFd)
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Println("Connection accepted from fd:", acceptedFd, "and addr:", acceptedAddr)
	// AcceptedFD socket send buffer is set to 2626560.
	fmt.Println("Accepted Socket memory")
	printBufferSizes(acceptedFd)
	printSockStat()

	_, err = unix.Write(clientFd, make([]byte, 1024*1024))
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Println("Writing data to buffer")
	// Sometimes the metrics don't change at all (Race condition?). Sometimes it changes by ~260 pages. This is likely 
	// 256 pages for the data that is written above plus overhead (1024 * 1024 bytes) / 4096 bytes per page = 256 pages.
	// A few runs:
	// Mem increased from 572 -> 829 (257)
	// Mem increased from 830 -> 1090 (260)
	// Mem increased from 572 -> 833 (261)
	// Mem increased from 833 -> 1091 (258)
	printSockStat()

	fmt.Println("Service Socket memory")
	printBufferSizes(serviceFd)
	fmt.Println("Client Socket memory")
	printBufferSizes(clientFd)
	fmt.Println("Accepted Socket memory")
	printBufferSizes(acceptedFd)
	printSockStat()
}

func printSockStat() {
	fs, _ := procfs.NewDefaultFS()
	stats, _ := fs.NetSockstat()
	for _, protocol := range stats.Protocols {
		if protocol.Protocol == "TCP" {
			fmt.Printf("TCP: inuse %d orphan %d tw %d alloc %d mem %d total %d\n\n", protocol.InUse,
				*protocol.Orphan,
				*protocol.TW,
				*protocol.Alloc,
				*protocol.Mem,
				*stats.Used)
		}
	}
}

func testTCPIPv4Socket() {
	fmt.Println("Testing TCP/IPv4 Socket")
	// See https://man7.org/linux/man-pages/man2/socket.2.html for more information about the
	// socket syscall. The following parameters are used
	// AF_INET - Indicates IPv4.
	// SOCK_STREAM - Indicates a connection-based socket type.
	// IPPROTO_TCP - Indicates TCP protocol
	socketFd, err := unix.Socket(unix.AF_INET, unix.SOCK_STREAM, unix.IPPROTO_TCP)
	if err != nil {
		fmt.Println(err)
		return
	}
	printBufferSizes(socketFd)
	// receive buffer:  131072         send buffer:  16384
	// This lines up with the default net.ipv4.tcp_rmem and default net.ipv4.tcp_wmem values.

	// Set the buffers to 32 KiB
	_ = unix.SetsockoptInt(socketFd, unix.SOL_SOCKET, unix.SO_RCVBUF, 32*1024)
	_ = unix.SetsockoptInt(socketFd, unix.SOL_SOCKET, unix.SO_SNDBUF, 32*1024)
	printBufferSizes(socketFd)
	// receive buffer:  65536  send buffer:  65536
	// I set the buffer size to 32 KiB, so why is it actually set to 64 KiB? According to
	// https://man7.org/linux/man-pages/man7/socket.7.html the kernel will double the given value when
	// in order to account for bookkeeping overhead.

	err = unix.SetsockoptInt(socketFd, unix.SOL_SOCKET, unix.SO_RCVBUF, 1024*1024*1024)
	err = unix.SetsockoptInt(socketFd, unix.SOL_SOCKET, unix.SO_SNDBUF, 1024*1024*1024)
	printBufferSizes(socketFd)
	// receive buffer:  425984         send buffer:  425984
	// The maximum size of the send and receive buffers are determined by net.core.wmem_max and net.core.rmem_max,
	// respectively. Each is set to 425984 bytes, or 416 KiB. Note that 425984 / 2 = 212992 (408 KiB).
}

func testUDPIPv4Socket() {
	fmt.Println("Testing UDP/IPv4 Socket")
	socketFd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, unix.IPPROTO_UDP)
	if err != nil {
		fmt.Println(err)
		return
	}
	printBufferSizes(socketFd)
	// receive buffer:  212992         send buffer:  212992    total:  416  KiB
	// This lines up with the default net.core.rmem_default and default net.core.wmem_default values.

	// Set the buffers to 32 KiB
	_ = unix.SetsockoptInt(socketFd, unix.SOL_SOCKET, unix.SO_RCVBUF, 32*1024)
	_ = unix.SetsockoptInt(socketFd, unix.SOL_SOCKET, unix.SO_SNDBUF, 32*1024)
	printBufferSizes(socketFd)
	// receive buffer:  65536  send buffer:  65536     total:  128  KiB

	err = unix.SetsockoptInt(socketFd, unix.SOL_SOCKET, unix.SO_RCVBUF, 1024*1024*1024)
	err = unix.SetsockoptInt(socketFd, unix.SOL_SOCKET, unix.SO_SNDBUF, 1024*1024*1024)
	printBufferSizes(socketFd)
	// receive buffer:  425984         send buffer:  425984    total:  832  KiB
}

func testTCPIPv6Socket() {
	fmt.Println("Testing TCP/IPv6 Socket")
	socketFd, err := unix.Socket(unix.AF_INET6, unix.SOCK_STREAM, unix.IPPROTO_TCP)
	if err != nil {
		fmt.Println(err)
		return
	}
	printBufferSizes(socketFd)
	// receive buffer:  131072         send buffer:  16384     total:  144  KiB

	// Set the buffers to 32 KiB
	_ = unix.SetsockoptInt(socketFd, unix.SOL_SOCKET, unix.SO_RCVBUF, 32*1024)
	_ = unix.SetsockoptInt(socketFd, unix.SOL_SOCKET, unix.SO_SNDBUF, 32*1024)
	printBufferSizes(socketFd)
	// receive buffer:  65536  send buffer:  65536     total:  128  KiB

	err = unix.SetsockoptInt(socketFd, unix.SOL_SOCKET, unix.SO_RCVBUF, 1024*1024*1024)
	err = unix.SetsockoptInt(socketFd, unix.SOL_SOCKET, unix.SO_SNDBUF, 1024*1024*1024)
	printBufferSizes(socketFd)
	// receive buffer:  425984         send buffer:  425984    total:  832  KiB
}

func testUDPIPv6Socket() {
	fmt.Println("Testing UDP/IPv6 Socket")
	socketFd, err := unix.Socket(unix.AF_INET6, unix.SOCK_DGRAM, unix.IPPROTO_UDP)
	if err != nil {
		fmt.Println(err)
		return
	}
	printBufferSizes(socketFd)
	// receive buffer:  131072         send buffer:  16384     total:  144  KiB

	// Set the buffers to 32 KiB
	_ = unix.SetsockoptInt(socketFd, unix.SOL_SOCKET, unix.SO_RCVBUF, 32*1024)
	_ = unix.SetsockoptInt(socketFd, unix.SOL_SOCKET, unix.SO_SNDBUF, 32*1024)
	printBufferSizes(socketFd)
	// receive buffer:  65536  send buffer:  65536     total:  128  KiB

	err = unix.SetsockoptInt(socketFd, unix.SOL_SOCKET, unix.SO_RCVBUF, 1024*1024*1024)
	err = unix.SetsockoptInt(socketFd, unix.SOL_SOCKET, unix.SO_SNDBUF, 1024*1024*1024)
	printBufferSizes(socketFd)
	// receive buffer:  425984         send buffer:  425984    total:  832  KiB
}

func printBufferSizes(socketFd int) {
	receiveBufferSize, _ := unix.GetsockoptInt(socketFd, unix.SOL_SOCKET, unix.SO_RCVBUF)
	sendBufferSize, _ := unix.GetsockoptInt(socketFd, unix.SOL_SOCKET, unix.SO_SNDBUF)

	fmt.Println("receive buffer: ", receiveBufferSize, "\tsend buffer: ", sendBufferSize, "\ttotal: ", (receiveBufferSize+sendBufferSize)/1024, " KiB")
}