Skip to content

Instantly share code, notes, and snippets.

@jtc-dolby
Forked from jeez/ Scheduled Tx Tools
Last active June 29, 2022 06:31
Show Gist options
  • Save jtc-dolby/9e08860175dd941e0b4fbd56cc870f55 to your computer and use it in GitHub Desktop.
Save jtc-dolby/9e08860175dd941e0b4fbd56cc870f55 to your computer and use it in GitHub Desktop.
[TSN] Scheduled Tx Tools - Examples and Helpers for testing SO_TXTIME, and the etf and taprio qdiscs
This is based on the good work of Jesus Sanchez-Palencia to provide a working example for the SO_TXTIME socket option. I've had some trouble using this example to get reliable output so I modifed it slightly so that is doesn't quit on the first error and I can collect a decent trace that shows the packet drops occurring
#!/bin/bash
tc qdisc replace dev $IFACE parent root handle 100 mqprio num_tc 3 map 2 2 1 0 2 2 2 2 2 2 2 2 2 2 2 2 queues 1@0 1@1 2@2 hw 0
tc qdisc replace dev $IFACE parent 100:1 etf offload delta 300000 clockid CLOCK_TAI
tc qdisc replace dev $IFACE parent 100:2 etf clockid CLOCK_TAI delta 300000 offload deadline_mode
PCAP_CFLAGS=$(shell pcap-config --cflags --libs)
all: udp_tai
dump-classifier: dump-classifier.c
${CC} ${CFLAGS} -lpthread $(PCAP_CFLAGS) -o $@ $<
udp_tai: udp_tai.c
${CC} ${CFLAGS} -o $@ $< -lpthread
clean:
@rm dump-classifier
@rm udp_tai
.PHONY: clean debug
[global]
#
# Default Data Set
#
twoStepFlag 1
slaveOnly 1
priority1 128
priority2 128
domainNumber 0
#utc_offset 37
clockClass 248
clockAccuracy 0xFE
offsetScaledLogVariance 0xFFFF
free_running 0
freq_est_interval 1
dscp_event 0
dscp_general 0
#
# Port Data Set
#
logAnnounceInterval 1
logSyncInterval -1
logMinDelayReqInterval 0
logMinPdelayReqInterval 0
announceReceiptTimeout 3
syncReceiptTimeout 0
delayAsymmetry 0
fault_reset_interval 4
neighborPropDelayThresh 20000000
#
# Run time options
#
assume_two_step 0
logging_level 6
path_trace_enabled 0
follow_up_info 0
hybrid_e2e 0
net_sync_monitor 0
tx_timestamp_timeout 10
use_syslog 1
verbose 0
summary_interval 0
kernel_leap 1
check_fup_sync 0
#
# Servo Options
#
pi_proportional_const 0.0
pi_integral_const 0.0
pi_proportional_scale 0.0
pi_proportional_exponent -0.3
pi_proportional_norm_max 0.7
pi_integral_scale 0.0
pi_integral_exponent 0.4
pi_integral_norm_max 0.3
step_threshold 0.0
first_step_threshold 0.00002
max_frequency 900000000
clock_servo pi
sanity_freq_limit 200000000
ntpshm_segment 0
#
# Transport options
#
transportSpecific 0x0
ptp_dst_mac 01:1B:19:00:00:00
p2p_dst_mac 01:80:C2:00:00:0E
udp_ttl 1
udp6_scope 0x0E
uds_address /var/run/ptp4l
#
# Default interface options
#
network_transport UDPv4
delay_mechanism E2E
time_stamping hardware
tsproc_mode filter
delay_filter moving_median
delay_filter_length 10
egressLatency 0
ingressLatency 0
boundary_clock_jbod 0
#
# Clock description
#
productDescription ;;
revisionData ;;
manufacturerIdentity 00:00:00
userDescription ;
timeSource 0xA0
#!/bin/bash
./udp_tai -c 1 -i enp7s0 -P 1000000 -p 99 -d 60000000 -E
#!/bin/bash
ptp4l -f /etc/linuxptp/ptp4l.conf -i enp7s0
phc2sys -w -s enp7s0
/*
* This program demonstrates transmission of UDP packets using the
* system TAI timer.
*
* Copyright (C) 2017 linutronix GmbH
*
* Large portions taken from the linuxptp stack.
* Copyright (C) 2011, 2012 Richard Cochran <richardcochran@gmail.com>
*
* Some portions taken from the sgd test program.
* Copyright (C) 2015 linutronix GmbH
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#define _GNU_SOURCE /*for CPU_SET*/
#include <arpa/inet.h>
#include <errno.h>
#include <fcntl.h>
#include <ifaddrs.h>
#include <linux/errqueue.h>
#include <linux/ethtool.h>
#include <linux/net_tstamp.h>
#include <linux/sockios.h>
#include <net/if.h>
#include <netinet/in.h>
#include <poll.h>
#include <pthread.h>
#include <sched.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#define ONE_SEC 1000000000ULL
#define DEFAULT_PERIOD 1000000
#define DEFAULT_DELAY 500000
#define DEFAULT_PRIORITY 3
#define MCAST_IPADDR "239.1.1.1"
#define UDP_PORT 7788
#define MARKER 'a'
#ifndef SO_TXTIME
#define SO_TXTIME 61
#define SCM_TXTIME SO_TXTIME
#endif
#ifndef SO_EE_ORIGIN_TXTIME
#define SO_EE_ORIGIN_TXTIME 6
#define SO_EE_CODE_TXTIME_INVALID_PARAM 1
#define SO_EE_CODE_TXTIME_MISSED 2
#endif
#define pr_err(s) fprintf(stderr, s "\n")
#define pr_info(s) fprintf(stdout, s "\n")
/* The API for SO_TXTIME is the below struct and enum, which will be
* provided by uapi/linux/net_tstamp.h in the near future.
*/
/*
struct sock_txtime {
clockid_t clockid;
uint16_t flags;
};
enum txtime_flags {
SOF_TXTIME_DEADLINE_MODE = (1 << 0),
SOF_TXTIME_REPORT_ERRORS = (1 << 1),
SOF_TXTIME_FLAGS_LAST = SOF_TXTIME_REPORT_ERRORS,
SOF_TXTIME_FLAGS_MASK = (SOF_TXTIME_FLAGS_LAST - 1) |
SOF_TXTIME_FLAGS_LAST
};
*/
static int running = 1, use_so_txtime = 1;
static int period_nsec = DEFAULT_PERIOD;
static int waketx_delay = DEFAULT_DELAY;
static int so_priority = DEFAULT_PRIORITY;
static int udp_port = UDP_PORT;
static int use_deadline_mode = 0;
static int receive_errors = 0;
static uint64_t base_time = 0;
static struct in_addr mcast_addr;
static struct sock_txtime sk_txtime;
static int mcast_bind(int fd, int index)
{
int err;
struct ip_mreqn req;
memset(&req, 0, sizeof(req));
req.imr_ifindex = index;
err = setsockopt(fd, IPPROTO_IP, IP_MULTICAST_IF, &req, sizeof(req));
if (err) {
pr_err("setsockopt IP_MULTICAST_IF failed: %m");
return -1;
}
return 0;
}
static int mcast_join(int fd, int index, const struct sockaddr *grp,
socklen_t grplen)
{
int err, off = 0;
struct ip_mreqn req;
struct sockaddr_in *sa = (struct sockaddr_in *) grp;
memset(&req, 0, sizeof(req));
memcpy(&req.imr_multiaddr, &sa->sin_addr, sizeof(struct in_addr));
req.imr_ifindex = index;
err = setsockopt(fd, IPPROTO_IP, IP_ADD_MEMBERSHIP, &req, sizeof(req));
if (err) {
pr_err("setsockopt IP_ADD_MEMBERSHIP failed: %m");
return -1;
}
err = setsockopt(fd, IPPROTO_IP, IP_MULTICAST_LOOP, &off, sizeof(off));
if (err) {
pr_err("setsockopt IP_MULTICAST_LOOP failed: %m");
return -1;
}
return 0;
}
static void normalize(struct timespec *ts)
{
while (ts->tv_nsec > 999999999) {
ts->tv_sec += 1;
ts->tv_nsec -= ONE_SEC;
}
while (ts->tv_nsec < 0) {
ts->tv_sec -= 1;
ts->tv_nsec += ONE_SEC;
}
}
static int sk_interface_index(int fd, const char *name)
{
struct ifreq ifreq;
int err;
memset(&ifreq, 0, sizeof(ifreq));
strncpy(ifreq.ifr_name, name, sizeof(ifreq.ifr_name) - 1);
err = ioctl(fd, SIOCGIFINDEX, &ifreq);
if (err < 0) {
pr_err("ioctl SIOCGIFINDEX failed: %m");
return err;
}
return ifreq.ifr_ifindex;
}
static int open_socket(const char *name, struct in_addr mc_addr, short port, clockid_t clkid)
{
struct sockaddr_in addr;
int fd, index, on = 1;
memset(&addr, 0, sizeof(addr));
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = htonl(INADDR_ANY);
addr.sin_port = htons(port);
fd = socket(PF_INET, SOCK_DGRAM, IPPROTO_UDP);
if (fd < 0) {
pr_err("socket failed: %m");
goto no_socket;
}
index = sk_interface_index(fd, name);
if (index < 0)
goto no_option;
if (setsockopt(fd, SOL_SOCKET, SO_PRIORITY, &so_priority, sizeof(so_priority))) {
pr_err("Couldn't set priority");
goto no_option;
}
if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on))) {
pr_err("setsockopt SO_REUSEADDR failed: %m");
goto no_option;
}
if (bind(fd, (struct sockaddr *) &addr, sizeof(addr))) {
pr_err("bind failed: %m");
goto no_option;
}
if (setsockopt(fd, SOL_SOCKET, SO_BINDTODEVICE, name, strlen(name))) {
pr_err("setsockopt SO_BINDTODEVICE failed: %m");
goto no_option;
}
addr.sin_addr = mc_addr;
if (mcast_join(fd, index, (struct sockaddr *) &addr, sizeof(addr))) {
pr_err("mcast_join failed");
goto no_option;
}
if (mcast_bind(fd, index)) {
goto no_option;
}
sk_txtime.clockid = clkid;
sk_txtime.flags = (use_deadline_mode | receive_errors);
if (use_so_txtime && setsockopt(fd, SOL_SOCKET, SO_TXTIME, &sk_txtime, sizeof(sk_txtime))) {
pr_err("setsockopt SO_TXTIME failed: %m");
goto no_option;
}
return fd;
no_option:
close(fd);
no_socket:
return -1;
}
static int udp_open(const char *name, clockid_t clkid)
{
int fd;
if (!inet_aton(MCAST_IPADDR, &mcast_addr))
return -1;
fd = open_socket(name, mcast_addr, udp_port, clkid);
return fd;
}
static int udp_send(int fd, void *buf, int len, __u64 txtime)
{
char control[CMSG_SPACE(sizeof(txtime))] = {};
struct sockaddr_in sin;
struct cmsghdr *cmsg;
struct msghdr msg;
struct iovec iov;
ssize_t cnt;
memset(&sin, 0, sizeof(sin));
sin.sin_family = AF_INET;
sin.sin_addr = mcast_addr;
sin.sin_port = htons(udp_port);
iov.iov_base = buf;
iov.iov_len = len;
memset(&msg, 0, sizeof(msg));
msg.msg_name = &sin;
msg.msg_namelen = sizeof(sin);
msg.msg_iov = &iov;
msg.msg_iovlen = 1;
/*
* We specify the transmission time in the CMSG.
*/
if (use_so_txtime) {
msg.msg_control = control;
msg.msg_controllen = sizeof(control);
cmsg = CMSG_FIRSTHDR(&msg);
cmsg->cmsg_level = SOL_SOCKET;
cmsg->cmsg_type = SCM_TXTIME;
cmsg->cmsg_len = CMSG_LEN(sizeof(__u64));
*((__u64 *) CMSG_DATA(cmsg)) = txtime;
}
cnt = sendmsg(fd, &msg, 0);
if (cnt < 1) {
pr_err("sendmsg failed: %m");
return cnt;
}
return cnt;
}
static unsigned char tx_buffer[256];
static unsigned int num_missed_deadline = 0;
static unsigned int num_invalid_params = 0;
static __u64 missed_deadline_tx_timestamp[1000];
static __u64 invalid_params_tx_timestamp[1000];
#define NUM_TIMESTAMPS 10000
static __u64 tx_timestamps[NUM_TIMESTAMPS] = {0};
static __u64 sched_timestamps[NUM_TIMESTAMPS] = {0};
static __u64 actual_timestamps[NUM_TIMESTAMPS] = {0};
static unsigned int num_timestamps = 0;
static int process_socket_error_queue(int fd)
{
uint8_t msg_control[CMSG_SPACE(sizeof(struct sock_extended_err))];
unsigned char err_buffer[sizeof(tx_buffer)];
struct sock_extended_err *serr;
struct cmsghdr *cmsg;
__u64 tstamp = 0;
struct iovec iov = {
.iov_base = err_buffer,
.iov_len = sizeof(err_buffer)
};
struct msghdr msg = {
.msg_iov = &iov,
.msg_iovlen = 1,
.msg_control = msg_control,
.msg_controllen = sizeof(msg_control)
};
if (recvmsg(fd, &msg, MSG_ERRQUEUE) == -1) {
pr_err("recvmsg failed");
return -1;
}
cmsg = CMSG_FIRSTHDR(&msg);
while (cmsg != NULL) {
serr = (void *) CMSG_DATA(cmsg);
if (serr->ee_origin == SO_EE_ORIGIN_TXTIME) {
tstamp = ((__u64) serr->ee_data << 32) + serr->ee_info;
switch(serr->ee_code) {
case SO_EE_CODE_TXTIME_INVALID_PARAM:
fprintf(stderr, "*");
//fprintf(stderr, "packet with tstamp %llu dropped due to invalid params\n", tstamp);
invalid_params_tx_timestamp[num_invalid_params++] = tstamp;
return 0;
case SO_EE_CODE_TXTIME_MISSED:
fprintf(stderr, "+");
missed_deadline_tx_timestamp[num_missed_deadline++] = tstamp;
//fprintf(stderr, "packet with tstamp %llu dropped due to missed deadline\n", tstamp);
return 0;
default:
return -1;
}
}
cmsg = CMSG_NXTHDR(&msg, cmsg);
}
return 0;
}
void sigint_handler(int s)
{
struct timespec ts, actual_ts;
int cnt, err, i, j, k,index;
__u64 txtime, base_ts, invalid_param_ts, missed_deadline_ts;
FILE *fd;
fd = fopen("packet_log.csv","w");
fprintf(stderr, "\nCaught signal %d - Writing packet log...",s);
fprintf(fd, "Missed Deadline\n");
for (i = 0 ; i < num_missed_deadline ; i++)
{
index = 0;
for (j = 0 ; j < NUM_TIMESTAMPS ; j++)
{
if (tx_timestamps[j] == missed_deadline_tx_timestamp[i])
{
index = j;
}
}
fprintf(fd,"%u,%llu,\n", index, missed_deadline_tx_timestamp[i] - base_ts);
}
fprintf(fd, "Invalid Params\n");
for (i = 0 ; i < num_invalid_params ; i++)
{
index = 0;
for (j = 0 ; j < NUM_TIMESTAMPS ; j++)
{
if (tx_timestamps[j] == invalid_params_tx_timestamp[i])
{
index = j;
}
}
fprintf(fd,"%u,%llu,\n", index, invalid_params_tx_timestamp[i] - base_ts);
}
base_ts = (sched_timestamps[0] / 100000000000ULL) * 100000000000ULL;
fprintf(fd, "Sched,Actual,Transmit, %u\n", num_timestamps);
for (i = num_timestamps + 1 ; i < num_timestamps + 1 + NUM_TIMESTAMPS ; i++)
{
j = i % NUM_TIMESTAMPS;
if (sched_timestamps[j] != 0)
{
missed_deadline_ts = base_ts; // means that 0 gets printed out if the packet didn't drop
for (k = 0 ; k < num_missed_deadline ; k++)
{
if (tx_timestamps[j] == missed_deadline_tx_timestamp[k])
{
missed_deadline_ts = tx_timestamps[j];
}
}
invalid_param_ts = base_ts; // means that 0 gets printed out if the packet didn't drop
for (k = 0 ; k < num_invalid_params ; k++)
{
if (tx_timestamps[j] == invalid_params_tx_timestamp[k])
{
invalid_param_ts = tx_timestamps[j];
}
}
fprintf(fd, "%d,%llu,%llu,%llu,%llu,%llu\n", j, sched_timestamps[j] - base_ts, actual_timestamps[j] - base_ts, tx_timestamps[j] - base_ts, missed_deadline_ts - base_ts, invalid_param_ts - base_ts);
}
}
fclose(fd);
fprintf(stderr, "Done\n\n");
exit(1);
}
static int run_nanosleep(clockid_t clkid, int fd)
{
struct timespec ts, actual_ts;
int cnt, err, i, j, k,index;
__u64 txtime, base_ts, invalid_param_ts, missed_deadline_ts;
struct pollfd p_fd = {
.fd = fd,
};
memset(tx_buffer, MARKER, sizeof(tx_buffer));
/* If no base-time was specified, start one to two seconds in the
* future.
*/
if (base_time == 0) {
clock_gettime(clkid, &ts);
ts.tv_sec += 1;
ts.tv_nsec = ONE_SEC - waketx_delay;
} else {
ts.tv_sec = base_time / ONE_SEC;
ts.tv_nsec = (base_time % ONE_SEC) - waketx_delay;
}
normalize(&ts);
txtime = ts.tv_sec * ONE_SEC + ts.tv_nsec;
txtime += waketx_delay;
fprintf(stderr, "\ntxtime of 1st packet is: %llu\n", txtime);
while (running) {
memcpy(tx_buffer, &txtime, sizeof(__u64));
err = clock_nanosleep(clkid, TIMER_ABSTIME, &ts, NULL);
switch (err) {
case 0:
clock_gettime(clkid, &actual_ts);
cnt = udp_send(fd, tx_buffer, sizeof(tx_buffer), txtime);
if (cnt != sizeof(tx_buffer)) {
pr_err("udp_send failed");
}
tx_timestamps[num_timestamps] = txtime;
sched_timestamps[num_timestamps] = ts.tv_sec * ONE_SEC + ts.tv_nsec;
actual_timestamps[num_timestamps++] = actual_ts.tv_sec * ONE_SEC + actual_ts.tv_nsec;
if (num_timestamps == NUM_TIMESTAMPS)
{
num_timestamps = 0;
}
ts.tv_nsec += period_nsec;
normalize(&ts);
txtime += period_nsec;
/* Check if errors are pending on the error queue. */
err = poll(&p_fd, 1, 0);
if (err == 1 && p_fd.revents & POLLERR) {
if (process_socket_error_queue(fd))
return -ECANCELED;
}
break;
case EINTR:
continue;
default:
fprintf(stderr, "clock_nanosleep returned %d: %s",
err, strerror(err));
return err;
}
}
return 0;
}
static int set_realtime(pthread_t thread, int priority, int cpu)
{
cpu_set_t cpuset;
struct sched_param sp;
int err, policy;
int min = sched_get_priority_min(SCHED_FIFO);
int max = sched_get_priority_max(SCHED_FIFO);
fprintf(stderr, "min %d max %d\n", min, max);
if (priority < 0) {
return 0;
}
err = pthread_getschedparam(thread, &policy, &sp);
if (err) {
fprintf(stderr, "pthread_getschedparam: %s\n", strerror(err));
return -1;
}
sp.sched_priority = priority;
err = pthread_setschedparam(thread, SCHED_FIFO, &sp);
if (err) {
fprintf(stderr, "pthread_setschedparam: %s\n", strerror(err));
return -1;
}
if (cpu < 0) {
return 0;
}
CPU_ZERO(&cpuset);
CPU_SET(cpu, &cpuset);
err = pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset);
if (err) {
fprintf(stderr, "pthread_setaffinity_np: %s\n", strerror(err));
return -1;
}
return 0;
}
static void usage(char *progname)
{
fprintf(stderr,
"\n"
"usage: %s [options]\n"
"\n"
" -c [num] run on CPU 'num'\n"
" -d [num] delta from wake up to txtime in nanoseconds (default %d)\n"
" -h prints this message and exits\n"
" -i [name] use network interface 'name'\n"
" -p [num] run with RT priorty 'num'\n"
" -P [num] period in nanoseconds (default %d)\n"
" -s do not use SO_TXTIME\n"
" -t [num] set SO_PRIORITY to 'num' (default %d)\n"
" -D set deadline mode for SO_TXTIME\n"
" -E enable error reporting on the socket error queue for SO_TXTIME\n"
" -b [tstamp] txtime of 1st packet as a 64bit [tstamp]. Default: now + ~2seconds\n"
" -u [port] use udp port 'port'\n"
"\n",
progname, DEFAULT_DELAY, DEFAULT_PERIOD, DEFAULT_PRIORITY);
}
int main(int argc, char *argv[])
{
int c, cpu = -1, err, fd, priority = -1;
clockid_t clkid = CLOCK_TAI;
char *iface = NULL, *progname;
signal (SIGINT,sigint_handler);
/* Process the command line arguments. */
progname = strrchr(argv[0], '/');
progname = progname ? 1 + progname : argv[0];
while (EOF != (c = getopt(argc, argv, "c:d:hi:p:P:st:DEb:u:"))) {
switch (c) {
case 'c':
cpu = atoi(optarg);
break;
case 'd':
waketx_delay = atoi(optarg);
break;
case 'h':
usage(progname);
return 0;
case 'i':
iface = optarg;
break;
case 'p':
priority = atoi(optarg);
break;
case 'P':
period_nsec = atoi(optarg);
break;
case 's':
use_so_txtime = 0;
break;
case 't':
so_priority = atoi(optarg);
break;
case 'D':
use_deadline_mode = SOF_TXTIME_DEADLINE_MODE;
break;
case 'E':
receive_errors = SOF_TXTIME_REPORT_ERRORS;
break;
case 'b':
base_time = atoll(optarg);
break;
case 'u':
udp_port = atoi(optarg);
break;
case '?':
usage(progname);
return -1;
}
}
if (waketx_delay > 999999999 || waketx_delay < 0) {
pr_err("Bad wake up to transmission delay.");
usage(progname);
return -1;
}
if (period_nsec < 1000) {
pr_err("Bad period.");
usage(progname);
return -1;
}
if (!iface) {
pr_err("Need a network interface.");
usage(progname);
return -1;
}
if (set_realtime(pthread_self(), priority, cpu)) {
return -1;
}
fd = udp_open(iface, clkid);
if (fd < 0) {
return -1;
}
err = run_nanosleep(clkid, fd);
close(fd);
return err;
}
@kcn21
Copy link

kcn21 commented Oct 29, 2021

Hello @jtc-dolby ,

When I tried to configure TAPRIO and ETF qdisc it give me same type of error for the both qdisc.

For ETF I used commands from your configure-etf.sh file : Unknown qdisc "etf", hence option "clockid" is unparsable

For TAPRIO I used command
sudo tc qdisc replace dev enp2s0 parent root handle 100 taprio num_tc 3 map 2 2 1 0 2 2 2 2 2 2 2 2 2 2 2 2 queues 1@0 1@0 1@0 base-time 1000000000 sched-entry S 01 300000 sched-entry S 03 300000 sched-entry S 04 400000 flags 0x1 txtime-delay 500000 clockid CLOCK_TAI
: Unknown qdisc "taprio", hence option "num_tc" is unparsable

Kindly help me regarding this.

Thanks,
Kunj

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment