Skip to content

Instantly share code, notes, and snippets.

@papr
Last active October 9, 2022 12:28
Show Gist options
  • Save papr/ad8298ccd1c955dfafb21c3cbce130c8 to your computer and use it in GitHub Desktop.
Save papr/ad8298ccd1c955dfafb21c3cbce130c8 to your computer and use it in GitHub Desktop.
Video overlay plugin for Pupil Player with manual temporal-alignment option. See this link for install instructions https://docs.pupil-labs.com/developer/core/plugin-api/#adding-a-plugin
import logging
from pyglui import ui
from video_overlay.controllers.overlay_manager import OverlayManager
from video_overlay.models.config import Configuration
from video_overlay.plugins.generic_overlay import Video_Overlay
from video_overlay.ui.management import UIManagementGeneric
from video_overlay.ui.menu import GenericOverlayMenuRenderer
from video_overlay.utils.constraints import ConstraintedValue, InclusiveConstraint
from video_overlay.workers.overlay_renderer import OverlayRenderer
logger = logging.getLogger(__name__)
class Temporal_Alignment_Overlay(Video_Overlay):
icon_chr = "aO"
def __init__(self, g_pool):
super().__init__(g_pool)
self.manager = TemporallyAlignedOverlayManager(g_pool.rec_dir)
def init_ui(self):
self.add_menu()
self.menu.label = "Temporally Aligned Video Overlays"
self.ui = UIManagementTemporallyAligned(self, self.menu, self.manager.overlays)
self.ui.add_observer("remove_overlay", self.manager.remove_overlay)
def _add_overlay_to_storage(self, video_path, initial_pos=(0, 0)):
config = TemporallyAlignedConfiguration(
video_path=video_path,
origin_x=initial_pos[0],
origin_y=initial_pos[1],
temporal_offset=0.0,
)
self.manager.add(config)
ts_scene_start = self.g_pool.timestamps[0]
ts_overlay_start = self.manager.most_recent.video.source.timestamps[0]
config.temporal_offset.value = ts_scene_start - ts_overlay_start
self.manager.save_to_disk()
self._overlay_added_to_storage(self.manager.most_recent)
class UIManagementTemporallyAligned(UIManagementGeneric):
def _add_overlay_menu(self, overlay):
renderer = TemporallyAlignedMenuRenderer(overlay)
renderer.add_observer("remove_button_clicked", self.remove_overlay)
self._menu_renderers[overlay] = renderer
self._parent_menu.append(renderer.menu)
class TemporallyAlignedMenuRenderer(GenericOverlayMenuRenderer):
def _generic_overlay_elements(self):
config = self.overlay().config
elements = super()._generic_overlay_elements()
return elements + (_make_offset_field(config),)
def _make_offset_field(config):
return ui.Text_Input("value", config.temporal_offset, label="Temporal offset")
class TemporallyAlignedConfiguration(Configuration):
version = 0
def __init__(
self,
temporal_offset=0.0,
scale=0.6,
**kwargs,
):
super().__init__(scale=scale, **kwargs)
self.scale = ConstraintedValue(scale, InclusiveConstraint(low=0.1, high=2.0))
self.temporal_offset = ConstraintedValue(temporal_offset, InclusiveConstraint())
@property
def as_tuple(self):
return super().as_tuple + (self.temporal_offset.value,)
@staticmethod
def from_tuple(tuple_):
as_dict = Configuration.from_tuple(tuple_[:-1]).as_dict()
as_dict["temporal_offset"] = tuple_[-1]
return TemporallyAlignedConfiguration(**as_dict)
def as_dict(self):
as_dict = super().as_dict()
as_dict["temporal_offset"] = self.temporal_offset.value
return as_dict
class TemporallyAlignedOverlayManager(OverlayManager):
def __init__(self, rec_dir):
super().__init__(rec_dir)
self._overlays = []
self._load_from_disk()
@property
def _storage_file_name(self):
return "temporally_aligned_video_overlays.msgpack"
@property
def _item_class(self):
return TemporallyAlignedConfiguration
def add(self, item):
overlay = TemporallyAlignedOverlayRenderer(item)
self._overlays.append(overlay)
class TemporallyAlignedOverlayRenderer(OverlayRenderer):
def draw_on_frame(self, target_frame):
if not self.valid_video_loaded:
return
# only change to parent class: subtract offset
overlay_frame = self.video.closest_frame_to_ts(
target_frame.timestamp - self.config.temporal_offset.value
)
overlay_image = overlay_frame.img
try:
is_fake_frame = overlay_frame.is_fake
# TODO: once we have the unified Frame class, we should just pass the frames
# to the image manipulation pipeline (instead of the images). Then we can
# get rid of the additional parameter stuff in ImageManipulation!
except AttributeError:
# TODO: this is only fore extra safety until we have a unified Frame class
# and can be sure that everything ending up here has the 'is_fake'
# attribute!
logger.warning(
f"Frame passed to overlay renderer does not have 'is_fake' attribute!"
f" Frame: {overlay_frame}"
)
is_fake_frame = False
for param, manipulation in self.pipeline:
overlay_image = manipulation.apply_to(
overlay_image, param.value, is_fake_frame=overlay_frame.is_fake
)
self._adjust_origin_constraint(target_frame.img, overlay_image)
self._render_overlay(target_frame.img, overlay_image)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment