Video Player with Annotation Bounding Boxes (Vue)

Captions.vue

<template>
  <v-card color="black" compact class="pa-0" min-height="52" max-height="52">
    <template v-slot:text>
      <p v-for="(caption, index) in captions" :key="index" align="center">
        <span style="color: white; font-size: 16px; font-family: 'Trebuchet MS', sans-serif;">{{
            currentDuration >= caption.startTime && currentDuration <= caption.endTime ? caption.text : ""
          }}</span>
      </p>
    </template>
  </v-card>
</template>

<script>
import {videoDataStore} from "@/store/videoData";

export default {
  data: () => ({
    captions: ""
  }),
  props: ['currentDuration'],
  computed: {
    getCaptions() {
      // Get request for vtt file and save as text. (Only vtt is supported)
      return videoDataStore().transcript;
    }
  },
  watch: {
    getCaptions() {
      console.log("Watch Caption", this.getCaptions);
      this.parseCaptions(this.getCaptions);
    }
  },
  mounted() {
    this.parseCaptions(this.getCaptions);
  },
  methods: {
    parseCaptions(vttData) {
      if (!vttData) return null;
      const lines = vttData.trim().split('\n\n');
      lines.shift();
      this.captions = lines.map((line) => {
        const [count, time, text] = line.split('\n');
        const [startTime, endTime] = time.split(' --> ');
        return {
          startTime: this.convertTimeToSeconds(startTime),
          endTime: this.convertTimeToSeconds(endTime),
          text,
          active: false,
        };
      });
    },
    // Function to convert VTT time format to seconds
    convertTimeToSeconds(time) {
      const [hours, minutes, seconds] = time.split(':').map(parseFloat);
      return hours * 3600 + minutes * 60 + seconds;
    },
    // Function to update captions based on video current time
    updateCaptions() {
      const currentTime = this.currentDuration;
      // console.log(currentTime);
      if (currentTime > 0)
        for (const caption of this.captions) {
          console.log(caption)
          caption.active = currentTime >= caption.startTime && currentTime <= caption.endTime;
        }
    },
  }
}
</script>

<style scoped>

</style>

VideoControls.vue

<template>
  <v-row class="pa-2" justify="center">
    <v-slider
        :model-value="currentDuration"
        v-model="thumb"
        @update:modelValue="emit('playFrame', thumb)"
        :max="duration"
        :step="1"
        color="white"
        thumb-color="red"
        hide-details
        rounded
        class="ma-1"
    ></v-slider>
  </v-row>
  <v-row class="pa-2 mb-2" justify="center">
    <v-col cols="4"
    <v-chip color="surface-variant" theme="dark" size="large" class="mt-2">
      {{ formatDuration(currentDuration > 0 ? currentDuration : 0) }} / {{ formatDuration(duration ?? 0) }}
    </v-chip>
    </v-col>
    <v-col cols="1">
      <v-btn icon="mdi-skip-previous-outline" @click="emit('previousFrame')"></v-btn>
    </v-col>
    <v-col cols="1" v-if="!isPlaying">
      <v-btn icon="mdi-play-outline" @click="emit('playVideo')"></v-btn>
    </v-col>
    <v-col cols="1" v-else>
      <v-btn icon="mdi-pause" @click="emit('pauseVideo')"></v-btn>
    </v-col>
    <v-col cols="1">
      <v-btn icon="mdi-skip-next-outline" @click="emit('nextFrame')"></v-btn>
    </v-col>
    <v-col cols="4" style="text-align: right;">
      <v-chip color="surface-variant" theme="dark" size="large" class="mt-2">Frame: {{ currentFrame }}</v-chip>
    </v-col>
  </v-row>
</template>

<script setup>
import { ref, watch, computed } from 'vue';

// Define props
const props = defineProps({
  isPlaying: Boolean,
  currentFrame: Number,
  currentDuration: Number,
  duration: Number
});

// Define emits
const emit = defineEmits(["previousFrame", "playVideo", "pauseVideo", "nextFrame", "playFrame"]);

const thumb = ref(0);

// Watchers
watch(() => props.currentDuration, (newVal) => {
  thumb.value = newVal;
});

// Methods
const formatDuration = (duration) => {
  const hours = Math.floor(duration / 3600);
  const minutes = Math.floor((duration % 3600) / 60);
  const seconds = Math.floor(duration % 60);

  const formattedHours = hours.toString().padStart(2, '0');
  const formattedMinutes = minutes.toString().padStart(2, '0');
  const formattedSeconds = seconds.toString().padStart(2, '0');
  if (hours > 0)
    return `${formattedHours}:${formattedMinutes}:${formattedSeconds}`;
  else
    return `${formattedMinutes}:${formattedSeconds}`;
};
</script>

VideoPlayer.vue

<template>
  <v-container class="mx-auto pa-0" id="canvas-container" style="max-width: 1280px; max-height: 720px">
    <div>
      <video ref="video" style="display: none;">
        <source :src="videoData.source" :type="videoData.type">
        Your browser does not support the video tag.
      </video>
      <canvas ref="canvas" id="canvas" width="1280" height="720"></canvas>
      <button class="mx-auto text-center ma-3" @click="showBoundingBoxes = !showBoundingBoxes">
        BBOX
      </button>
      <VideoControls @previousFrame="previousFrame" @playVideo="playVideo"
                     @pauseVideo="pauseVideo" @nextFrame="nextFrame" @playFrame="playFrame"
                     :isPlaying="isPlaying"
                     :currentFrame="videoData.currentFrame" :currentDuration="videoData.currentDuration" :duration="videoData.duration"/>
    </div>
  </v-container>
</template>

<script setup>
import {ref, onMounted} from 'vue';


const annotations = ref({
  // ... the annotations constant or load from from http request
});
const showBoundingBoxes = ref(false);
const video = ref(null);
const canvas = ref(null);
const isPlaying = ref(false);
const requestedFrame = ref(0);
const frameRequestId = ref(null);
const ctx = ref(null);
const videoData = ref({
  source: "samplevideo.mp4",
  type: "video/mp4",
  fps: 60,
  duration: 0,
  currentDuration: 0,
  currentFrame: 0
});

const estimateCurrentFrame = (currentTime) => {
  return Math.floor(currentTime * videoData.value.fps) ?? 1;
}

const drawFrame = () => {

  videoData.value.currentFrame = estimateCurrentFrame(video.value.currentTime);
  videoData.value.currentDuration = video.value.currentTime;

  // Draw the current frame from the video onto the canvas
  ctx.value.drawImage(video.value, 0, 0, canvas.value.width, canvas.value.height);

  // Continue drawing the next frame
  frameRequestId.value = requestAnimationFrame(drawFrame);
  if (showBoundingBoxes) drawBoundingBoxes(estimateCurrentFrame(video.value.currentTime));
}

const drawBoundingBox = (frameNo, label, x, y, width, height, color) => {
  // Set the color and border width for the bounding box
  ctx.value.strokeStyle = color;
  ctx.value.lineWidth = 2;

  // Draw the bounding box
  ctx.value.beginPath();
  ctx.value.rect(x, y, width, height);
  ctx.value.stroke();

  // Add the label text above the bounding box
  ctx.value.fillStyle = color;
  ctx.value.font = '12px Arial';
  ctx.value.fillText(`${label}`, x, y - 5);
}

const drawBoundingBoxes = (frameNo) => {
  const boundingBoxes = annotations.value[frameNo];
  if (boundingBoxes.length > 0) {
    for (const item of boundingBoxes) {
      let label = item.class;
      const color = `rgb(${item.fillColor.join(',')})`;
      const [x, y, width, height] = item.bbox;
      drawBoundingBox(frameNo, label, x, y, width, height, color);
    }
  }
}

const playFromFrame = () => {
  const frameNumber = requestedFrame.value;
  const frameRate = videoData.value.fps;
  video.value.currentTime = frameNumber / frameRate;
}

const playFrame = (requestedFrame) => {
  requestedFrame.value = estimateCurrentFrame(requestedFrame);
  playFromFrame();
}

// Function to play the video
const playVideo = () => {
  video.value.play();
  // Draw the current frame
  drawFrame();
}

// Function to pause the playback
const pauseVideo = () => {
  video.value.pause();

  // Stop drawing the frames
  isPlaying.value = false;

  // Cancel any pending frame requests
  cancelAnimationFrame(frameRequestId);
}

// Function to move to the next frame
const nextFrame = () => {
  // Move one frame forward
  video.value.currentTime += 1 / videoData.value.fps;
  drawFrame();
}

// Function to move to the previous frame
const previousFrame = () => {
  // Move one frame backward
  video.value.currentTime -= 1 / videoData.value.fps;
  drawFrame();
}

onMounted(() => {
  ctx.value = canvas.value.getContext('2d');

  video.value.addEventListener('loadedmetadata', () => {
    // The 'loadedmetadata' event is fired when the video's metadata, including the duration, is loaded.
    videoData.value.duration = video.value.duration;
  });

  video.value.addEventListener('play', () => {
    console.log("playing....");
    videoData.value.currentDuration = video.value.currentTime;
    videoData.value.currentFrame = estimateCurrentFrame(video.value.currentTime);
    isPlaying.value = true;

    drawFrame();
  });

  video.value.addEventListener('pause', () => {
    console.log("paused....")
    isPlaying.value = false;
  });

  video.value.addEventListener('ended', () => {
    console.log("ended....")
    isPlaying.value = false;
  });

  drawFrame();
});
</script>




<style scoped>
#canvas-container {
  position: relative;
}
</style>