Let FFmpeg deal with HLS X-TIMESTAMP-MAP tags. This should be an option but that did not work for me so it is hardcoded in this patch to be always on.

avformat-webvttdec_support-HLS-timestamp-maps.patch

From 892d0851cfd93737c679328583c2fe920a65c2ff Mon Sep 17 00:00:00 2001
From: Sebastian Zeller <SebiderSushi@t-online.de>
Date: Tue, 5 Jan 2021 03:38:01 +0100
Subject: [PATCH] avformat/webvttdec: Add support for HLS WebVTT MPEG
 timestamp map

---
 libavformat/webvttdec.c | 55 ++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 54 insertions(+), 1 deletion(-)

diff --git a/libavformat/webvttdec.c b/libavformat/webvttdec.c
index 8d2fdfed37..8e82594ab7 100644
--- a/libavformat/webvttdec.c
+++ b/libavformat/webvttdec.c
@@ -35,6 +35,7 @@ typedef struct {
     const AVClass *class;
     FFDemuxSubtitlesQueue q;
     int kind;
+    int prefer_hls_mpegts_pts;
 } WebVTTContext;
 
 static int webvtt_probe(const AVProbeData *p)
@@ -49,6 +50,11 @@ static int webvtt_probe(const AVProbeData *p)
     return 0;
 }
 
+static int64_t convert_to_hls_mpegts_ts(int64_t timestamp, int64_t offset)
+{
+    return (timestamp * 90 + offset) & ((1LL << 33) - 1);
+}
+
 static int64_t read_ts(const char *s)
 {
     int hh, mm, ss, ms;
@@ -63,6 +69,9 @@ static int webvtt_read_header(AVFormatContext *s)
     AVBPrint cue;
     int res = 0;
     AVStream *st = avformat_new_stream(s, NULL);
+    int has_hls_timestamp_map = 0;
+    int64_t hls_ts_offset;
+    int64_t hls_ts_offset_global = -1;
 
     if (!st)
         return AVERROR(ENOMEM);
@@ -92,8 +101,45 @@ static int webvtt_read_header(AVFormatContext *s)
         /* ignore header chunk */
         if (!strncmp(p, "\xEF\xBB\xBFWEBVTT", 9) ||
             !strncmp(p, "WEBVTT", 6) ||
-            !strncmp(p, "NOTE", 4))
+            !strncmp(p, "NOTE", 4)) {
+
+            /* somehow the option is always the default value? */
+            if (webvtt->prefer_hls_mpegts_pts == 0) {
+                /*
+                * WebVTT files in HLS streams contain a timestamp offset for
+                * syncing with the main stream:
+                *
+                * X-TIMESTAMP-MAP=LOCAL:00:00:00.000,MPEGTS:900000
+                * (LOCAL and MPEGTS can be reversed even though HLS spec
+                *  does not say so)
+                */
+
+                char *hls_timestamp_map = strstr(p, "\nX-TIMESTAMP-MAP=");
+                if (hls_timestamp_map) {
+                    char *native_str = strstr(hls_timestamp_map, "LOCAL:");
+                    char *mpegts_str = strstr(hls_timestamp_map, "MPEGTS:");
+                    if (native_str && mpegts_str) {
+                        int64_t native_ts = read_ts(native_str + 6);
+                        int64_t mpegts_ts = strtoll(mpegts_str + 7, NULL, 10);
+
+                        if (native_ts != AV_NOPTS_VALUE) {
+                            /*
+                            * In livestreams the timestamp map can start at any mpegts time.
+                            * Consequently, everything should be treated relative to the first timestamp.
+                            */
+                            if (hls_ts_offset_global == -1) {
+                                hls_ts_offset_global = mpegts_ts;
+                            }
+                            hls_ts_offset = mpegts_ts - native_ts * 90 - hls_ts_offset_global;
+                            has_hls_timestamp_map = 1;
+                            avpriv_set_pts_info(st, 33, 1, 90000);
+                        }
+                    }
+                }
+            }
+
             continue;
+        }
 
         /* optional cue identifier (can be a number like in SRT or some kind of
          * chaptering id) */
@@ -124,6 +170,12 @@ static int webvtt_read_header(AVFormatContext *s)
         if ((ts_end = read_ts(p)) == AV_NOPTS_VALUE)
             break;
 
+        if (has_hls_timestamp_map) {
+            /* convert and truncate to MPEG TS timestamps */
+            ts_start = convert_to_hls_mpegts_ts(ts_start, hls_ts_offset);
+            ts_end = convert_to_hls_mpegts_ts(ts_end, hls_ts_offset);
+        }
+
         /* optional cue settings */
         p += strcspn(p, "\n\r\t ");
         while (*p == '\t' || *p == ' ')
@@ -200,6 +252,7 @@ static const AVOption options[] = {
         { "captions",     "WebVTT captions kind",     0, AV_OPT_TYPE_CONST, { .i64 = AV_DISPOSITION_CAPTIONS },     INT_MIN, INT_MAX, KIND_FLAGS, "webvtt_kind" },
         { "descriptions", "WebVTT descriptions kind", 0, AV_OPT_TYPE_CONST, { .i64 = AV_DISPOSITION_DESCRIPTIONS }, INT_MIN, INT_MAX, KIND_FLAGS, "webvtt_kind" },
         { "metadata",     "WebVTT metadata kind",     0, AV_OPT_TYPE_CONST, { .i64 = AV_DISPOSITION_METADATA },     INT_MIN, INT_MAX, KIND_FLAGS, "webvtt_kind" },
+    { "prefer_hls_mpegts_pts", "Use WebVTT embedded HLS MPEGTS timestamps if available.", OFFSET(prefer_hls_mpegts_pts), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, AV_OPT_FLAG_SUBTITLE_PARAM | AV_OPT_FLAG_DECODING_PARAM, NULL },
     { NULL }
 };
 
-- 
2.29.2

Adapted from this patch by Franklin Phillips: https://patchwork.ffmpeg.org/project/ffmpeg/patch/20170205194344.333-1-franklinphillips@gmx.com/