twitch_chat_logstash.yml

gistfile1.txt

input {
  file {
    path => "/tmp/irssi_raw.log"
    start_position => "end"
    type => "irc"
    tags => ["irc"]
  }
  http {
    port => 31311 # default: 8080
    tags => ["http"]
  }
}

filter {
  if "irc" in [tags] {
    grok {
        add_tag => [ "chat_message" ]
        match => [
          "message", "^>>.*PRIVMSG"
        ]
    }
    if "chat_message" not in [tags] {
      drop {}
    }

    mutate {
      remove_tag => [ "chat_message" ]
    }

    grok {
      match => [
        "message", ">> @(?<badges_raw>[^ ]+) :(?<username>[^!]+)!(?<identity>[^ ]+) PRIVMSG #(?<channel>[^ ]+) :(?<msg>.*)"
      ]
    }

    grok {
        add_tag => [ "clipit" ]
        match => [
          "message", "TTours"
        ]
    }


    kv { value_split => "="  field_split => ";" source => "badges_raw" target => "badges" }

    mutate {
      split => { "[badges][emotes]" => "," }
    }

    ruby {
      code => "event.set('number_of_smiles', !event.get('[badges][emotes]').nil? ? event.get('[badges][emotes]').length : 0)"
    }

    ruby {
      code => "event.set('msg_time', Time.now().strftime('%Y-%m-%dT%H:%M:%S+04:00'))"
    }
    ruby {
      code => "
        require 'json'
        require 'date'
        file = File.open '/tmp/twitch_data.json'
        data = JSON.load file
        user_name = event.get('channel')
        event.set('viewers', 0)
        event.set('stream_time', 0)
        for user in data['data']
          if user_name == user['user_login']
            event.set('viewers', user['viewer_count'])
            started_at_time = Time.strptime(user['started_at'], '%Y-%m-%dT%H:%M:%S%z')
            stream_time = Time.now.getlocal().strftime('%s').to_i - (started_at_time.strftime('%s').to_i)
            event.set('stream_time', stream_time)

            break
          end
        end
        file.close
      "
    }
    prune {
      whitelist_names => [
        "^channel$",
        "^viewers$",
        "^stream_time$",
        "^username$",
        "^number_of_smiles$",
        "^msg$",
        "^msg_time$",
        "^tags$"
      ]
    }
  }

}

output {
#    file {
#      path => "/tmp/output_logstash.txt"
#      codec => line { format => "%{stream_time} #%{channel}(%{viewers}) %{username} -> %{msg} ..... %{number_of_smiles}"}
#    }

    if "irc" in [tags] {
      elasticsearch {
        hosts => ['localhost:9200']
        user => 'elastic'
        password => 'changeme'
        # index => "twitch_clipz_%{+yyyy.MM.dd}"
        index => "logstash-%{+yyyy.MM.dd}"
      }
      kafka {
        codec => json
        bootstrap_servers => "localhost:9092"
        topic_id => "twitch-clipz"
      }
    }
    if "http" in [tags] or "clipit" in [tags] {
      kafka {
        codec => json
        bootstrap_servers => "localhost:9092"
        topic_id => "auto-clipz"
      }
#      exec {
#        command => "/home/bebebeko/spaces/twitch-dl/bin/auto_clip.sh %{channel} %{time}"
        #command => "su - bebebeko /bin/sh -c \"/home/bebebeko/spaces/twitch-dl/bin/auto_clip.sh %{channel} %{time}\""
#      }
    }


}

@metacoma is this how you display twitch tags as per https://dev.twitch.tv/docs/irc/tags/ in irssi? Do you have any other steps on how to make this visible in irssi?

@loopedice

servers = (
  {
    address = "irc.chat.twitch.tv";
    chatnet = "Twitch";
    port = "6697";
    password = "oauth:3qllxxxi46ykvitfds7se54nurc138j";
    use_tls = "yes";
    tls_verify = "no";
    autoconnect = "yes";
    autosendcmd = "/quote CAP REQ :twitch.tv/tags"; 
  }
);

To receive twitch tags in the irssi client, you need to send /quote CAP REQ :twitch.tv/tags after connecting to the server.

@metacoma
Yea I have that, but it just displays and logs regular text without sub badges, etc. 🤔 Maybe it's this irssi docker image I am using... Thanks for responding 🙏

@loopedice It's actually strange; I've switched from irssi to eggdrop for irc logging purposes, and I haven't encountered any issues with this.

Would you like to receive messages like following?

@badge-info=;badges=;client-nonce=cf100bebeb8980dd3460ff54b5376143;color=#8A2BE2;display-name=Kotenz;emotes=;first-msg=0;flags=;id=de91a6f6-1bdb-4bf8-b6cc-cc01833f0c51;mod=0;returning-chatter=0;room-id=87186401;subscriber=0;tmi-sent-ts=1711880270908;turbo=0;user-id=164651825;user-type= :kotenz!kotenz@kotenz.tmi.twitch.tv PRIVMSG #lenagol0vach :@LenaGol0vach

If you're using Irssi, you can easily check how it works. Just run irssi on your host (not inside a Docker container), then connect to the Twitch IRC server and manually enter the command: /quote CAP REQ :twitch.tv/tags. After that, you should see the badge tags message from the twitch server.

just in case, this is my full irssi config, please pay attention to the first and last sections. In the /tmp/irssi_raw.log file, you should see tags and badges.

servers = (
  {
    address = "irc.chat.twitch.tv";
    chatnet = "Twitch";
    port = "6697";
    password = "oauth:XXXX";
    use_tls = "yes";
    tls_verify = "no";
    autoconnect = "yes";
    autosendcmd = "/quote CAP REQ :twitch.tv/tags";
  }
);

aliases = {
  ATAG = "WINDOW SERVER";
  ADDALLCHANS = "SCRIPT EXEC foreach my \\$channel (Irssi::channels()) { Irssi::command(\"CHANNEL ADD -auto \\$channel->{name} \\$channel->{server}->{tag} \\$channel->{key}\")\\;}";
  B = "BAN";
  BACK = "AWAY";
  BANS = "BAN";
  BYE = "QUIT";
  C = "CLEAR";
  CALC = "EXEC - if command -v bc >/dev/null 2>&1\\; then printf '%s=' '$*'\\; echo '$*' | bc -l\\; else echo bc was not found\\; fi";
  CHAT = "DCC CHAT";
  DATE = "TIME";
  DEHIGHLIGHT = "DEHILIGHT";
  DESCRIBE = "ACTION";
  DHL = "DEHILIGHT";
  EXEMPTLIST = "MODE $C +e";
  EXIT = "QUIT";
  GOTO = "SCROLLBACK GOTO";
  HIGHLIGHT = "HILIGHT";
  HL = "HILIGHT";
  HOST = "USERHOST";
  INVITELIST = "MODE $C +I";
  J = "JOIN";
  K = "KICK";
  KB = "KICKBAN";
  KN = "KNOCKOUT";
  LAST = "LASTLOG";
  LEAVE = "PART";
  M = "MSG";
  MUB = "UNBAN *";
  N = "NAMES";
  NMSG = "^MSG";
  P = "PART";
  Q = "QUERY";
  RESET = "SET -default";
  RUN = "SCRIPT LOAD";
  SAY = "MSG *";
  SB = "SCROLLBACK";
  SBAR = "STATUSBAR";
  SIGNOFF = "QUIT";
  SV = "MSG * Irssi $J ($V) - https://irssi.org";
  T = "TOPIC";
  UB = "UNBAN";
  UMODE = "MODE $N";
  UNSET = "SET -clear";
  W = "WHO";
  WC = "WINDOW CLOSE";
  WG = "WINDOW GOTO";
  WJOIN = "JOIN -window";
  WI = "WHOIS";
  WII = "WHOIS $0 $0";
  WL = "WINDOW LIST";
  WN = "WINDOW NEW HIDDEN";
  WQUERY = "QUERY -window";
  WW = "WHOWAS";
};

statusbar = {

  items = {

    barstart = "{sbstart}";
    barend = "{sbend}";

    topicbarstart = "{topicsbstart}";
    topicbarend = "{topicsbend}";

    time = "{sb $Z}";
    user = "{sb {sbnickmode $cumode}$N{sbmode $usermode}{sbaway $A}}";

    window = "{sb $winref:$tag/$itemname{sbmode $M}}";
    window_empty = "{sb $winref{sbservertag $tag}}";

    prompt = "{prompt $[.15]itemname}";
    prompt_empty = "{prompt $winname}";

    topic = " $topic";
    topic_empty = " Irssi v$J - https://irssi.org";

    lag = "{sb Lag: $0-}";
    act = "{sb Act: $0-}";
    more = "-- more --";
  };

  default = {

    window = {

      disabled = "no";
      type = "window";
      placement = "bottom";
      position = "1";
      visible = "active";

      items = {
        barstart = { priority = "100"; };
        time = { };
        user = { };
        window = { };
        window_empty = { };
        lag = { priority = "-1"; };
        act = { priority = "10"; };
        more = { priority = "-1"; alignment = "right"; };
        barend = { priority = "100"; alignment = "right"; };
      };
    };

    window_inact = {

      type = "window";
      placement = "bottom";
      position = "1";
      visible = "inactive";

      items = {
        barstart = { priority = "100"; };
        window = { };
        window_empty = { };
        more = { priority = "-1"; alignment = "right"; };
        barend = { priority = "100"; alignment = "right"; };
      };
    };

    prompt = {

      type = "root";
      placement = "bottom";
      position = "100";
      visible = "always";

      items = {
        prompt = { priority = "-1"; };
        prompt_empty = { priority = "-1"; };
        input = { priority = "10"; };
      };
    };

    topic = {

      type = "root";
      placement = "top";
      position = "1";
      visible = "always";

      items = {
        topicbarstart = { priority = "100"; };
        topic = { };
        topic_empty = { };
        topicbarend = { priority = "100"; alignment = "right"; };
      };
    };
  };
};
settings = {
  core = {
    real_name = "Unknown";
    user_name = "useruser33";
    nick = "useruser33";
    server_reconnect_time = "30";
  };
  "fe-text" = { actlist_sort = "refnum"; };
};
chatnets = {
  Twitch = {
    type = "IRC";
    autosendcmd = "/rawlog open /log/irssi_raw.log; /quote CAP REQ :twitch.tv/tags";
  };
};

channels = (
  { name = "#PGL"; chatnet = "Twitch"; autojoin = "Yes"; },
);

Nice that was it! It was the /rawlog I reckon that was the issue. Huge thanks! 🙏