debugging an issue where, using Matrix transforms similar to look_at or perspective, the prior screen_to_world method that works with view_clip/translate no longer works. An additional step of some sort is required to translate screen to world, and I am not sure what it is. I would love any help! NOTE: +/- on keyboard to control camera "zoom".

screen_to_world.py

from array import array

from kivy.app import App
from kivy.clock import Clock
from kivy.core.window import Window
from kivy.graphics.context_instructions import Color
from kivy.graphics.instructions import RenderContext
from kivy.graphics.texture import Texture
from kivy.graphics.transformation import Matrix
from kivy.graphics.vertex_instructions import Rectangle
from kivy.lang import Builder
from kivy.properties import ListProperty, ObjectProperty
from kivy.uix.boxlayout import BoxLayout
from kivy.uix.floatlayout import FloatLayout
from kivy.uix.widget import Widget


"""
The goal is to determine how to successfully have screen mouse coordinates translated into the world space coordinates
that would align underneath the screen coordinates in the transformed view.  With view_clip/translate projection matrix,
the current screen_to_world method functions properly, regardless of zoom or camera_target.  

The problem is that we want to use the other camera update method, look_at_perspective, but when we do this, the 
screen_to_world method will no longer properly convert to world coordinates from the screen coordinate.  

I have tried to use different transform_points, dividing by w, and attempted a near/far plane ray picking approach,
but I have yet to achieve a decent approximation to convert from screen position to world position.
"""


controls = {k: False for k in ['plus', 'minus']}
keyboard_mapping = {
    269: 'minus',
    45: 'minus',
    270: 'plus',
    61: 'plus',
}


def on_key_down(window, keycode, *rest):
    # print('down', keycode)
    controls[keyboard_mapping.get(keycode, keycode)] = True


def on_key_up(window, keycode, *rest):
    # print('keycode', keycode)
    controls[keyboard_mapping.get(keycode, keycode)] = False


KV = '''
#:import F kivy.factory.Factory
#:import random random.random

BoxLayout:
    id: container
    View:
        container: container
        layer: layer
        FloatLayout:
            GridLayout:
                cols: 20
                on_parent:
                    for i in range(20 * 20): self.add_widget(F.ColorLabel(text=str(i)))
            BoxLayout:
                # just here for a bit of a dimmer between colorful tile background and debug drawing on WorldLayer
                canvas:
                    Color:
                        rgba: [0, 0, 0, .8]
                    Rectangle:
                        pos: self.pos
                        size: self.size   
            WorldLayer:
                id: layer
                texture_size: 100, 100
                pencil_size: 2, 2


<ColorLabel@Label>:
    canvas.before:
        Color:
            rgba: [random() for i in range(4)]
        Rectangle:
            pos: self.pos
            size: self.size


'''


class View(FloatLayout):
    layer = ObjectProperty()

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.canvas = RenderContext()
        self.camera_scale = .5
        self.camera_target = 600, 600
        print('init!')
        Clock.schedule_interval(self.update, 1 / 60.)

    def on_touch_up(self, touch):
        print(f'view touch: {touch.pos}')
        world_coords = self.screen_to_world(*touch.pos)
        self.layer.touch_in_world(*world_coords)
        self.paint_on_container(*touch.pos)

    def paint_on_container(self, x, y):
        with self.container.canvas:
            Color(rgba=(1, .5, .5, .7))
            Rectangle(pos=(x-4, y-4), size=(8, 8))

    def on_touch_move(self, touch):
        world_coords = self.screen_to_world(*touch.pos)
        self.layer.touch_in_world(*world_coords)
        self.paint_on_container(*touch.pos)

    def update(self, dt):
        # comment out one of the two camera update methods below

        # self.camera_view_clip()
        self.camera_look_at_perspective()

        if controls.get('plus'):
            controls.pop('plus')
            print('zooming in')
            self.zoom_in()
        elif controls.get('minus'):
            controls.pop('minus')
            self.zoom_out()
            print('zooming out')

    def zoom_in(self):
        if self.camera_scale * .8 > 0:
            self.camera_scale *= .8

    def zoom_out(self):
        if self.camera_scale * 1.2 < 1000:
            self.camera_scale *= 1.2

    def camera_look_at_perspective(self):
        p = Matrix()
        p.perspective(90., 16 / 9, 1, 1000)

        w_x, w_y = self.camera_target

        # if we want to translate and rotate instead of perspective/look_at transforms
        # p.translate(-fx, -fy + 20, -self.camera.scale).rotate(radians(-30.), 1.0, 0.0, 0.0)
        self.canvas['projection_mat'] = p
        self.canvas['modelview_mat'] = Matrix().look_at(w_x, w_y - 30, self.camera_scale * 350, w_x, w_y, 0, 0, 1, 0)

    def camera_view_clip(self):
        w = self.width
        h = self.height
        w2, h2 = w // 2, h // 2
        world_x, world_y = self.camera_target

        # Camera size
        sx, sy = w * self.camera_scale / 2, h * self.camera_scale / 2

        tm = Matrix().translate(-world_x, -world_y, 0)  # Bring frame to origin
        proj = Matrix().view_clip(-sx, sx, -sy, sy, 0, 100, 0).multiply(tm)

        self.canvas['projection_mat'] = proj

    def screen_to_world(self, x, y):
        proj = self.canvas['projection_mat']
        model = self.canvas['modelview_mat']

        # get the inverse of the current matrices, MVP
        m = Matrix().multiply(proj).multiply(model)
        inverse = m.inverse()
        w, h = self.size

        # normalize pos in window
        norm_x = x / w * 2.0 - 1.0
        norm_y = y / h * 2.0 - 1.0

        p = inverse.transform_point(norm_x, norm_y, 0)
        # print('convert_from_screen_to_world', x, y, p)
        return p[:2]


class WorldLayer(Widget):
    def touch_in_world(self, x, y):
        print(f'world touched at: {x, y}')
        width, height = self.texture_size

        with self.canvas:
            Color(rgba=(.5, 1, .5, .5))
            Rectangle(pos=(x-8, y-8), size=(16, 16))


class CoordinateApp(App):
    def build(self):
        return Builder.load_string(KV)


if __name__ == '__main__':
    Window.bind(on_key_down=on_key_down, on_key_up=on_key_up)
    CoordinateApp().run()