Kitty backend

[port19x]

The kitty terminal emulator has a native graphics protocol supporting image preview.
It does so through one of its default kittens (= extensions) : icat
Here is the relevant documentation for icat: https://sw.kovidgoyal.net/kitty/kittens/icat/

In an issue from 2020, the author of kitty expressed that he doesn't want to work on a video player since videos include sound as well. See here: kovidgoyal/kitty#2947

What I am requesting is that someone takes a quick look at kittys graphics protocol and checks how easy this feature would be to implement an maintain, and if it is as trivial as it appears to me from a users perspective, implement it.
Alternatively, if you find a way to make video work indirectly by piping the vo=image driver through some stuff, that would be a cool placeholder.

Either way, thank you for your great work and have a good christmas season.

PS: I will crossreference this issue on the above mentioned issue regarding video playback in kitty.

Expected behavior of the wanted feature

--vo=kitty

Alternative behavior of the wanted feature

--vo=image | ... | icat

Log file

mpv 0.32.0 Copyright © 2000-2020 mpv/MPlayer/mplayer2 projects
built on UNKNOWN
ffmpeg library versions:
libavutil 56.51.100
libavcodec 58.91.100
libavformat 58.45.100
libswscale 5.7.100
libavfilter 7.85.100
libswresample 3.7.100
ffmpeg version: 4.3.3-0+deb11u1

[avih]

No. Terminal output is a toy at best. Few are supported, but only using standard methods (tct, sixel).

If you're running kitty then you're already in a graphical environment, so run a graphical mpv properly in this environment.

[sfan5]

If you want to attempt implementing this yourself, take a look at the source of tct and sixel outputs:
https://github.com/mpv-player/mpv/blob/master/video/out/vo_tct.c
https://github.com/mpv-player/mpv/blob/master/video/out/vo_sixel.c

[port19x]

I understand that you don't want to add support for obscure video backends and expected a wontfix outcome.
Thank you for refering me to tct and sixel source code.
I would try to implement it myself but haven't even written a hello world in C before.
I'll comment here and on the related kitty issue if I find a workaround via the image backend and clever piping

[tantei3]

@ura43 You can try building mpv using waf with the below diff as a starting point for kitty terminal which was derived from vo_sixel mainly. It seems to work way better than Sixel/TCT using the Shared Memory approach, but still there are frame tearing happening at near 1080p, so wouldn't recommend this for serious usage. Since kitty protocol is not a standard and doesn't work in any other terminal emulator, I don't think it will be accepted officially in mpv, but since it is simple enough and doesn't have any external dependencies, it shouldn't be hard to make it work and build with this patch if you really want.

diff --git a/video/out/vo.c b/video/out/vo.c
index 11ef596227..c163819030 100644
--- a/video/out/vo.c
+++ b/video/out/vo.c
@@ -66,6 +66,7 @@ extern const struct vo_driver video_out_wlshm;
 extern const struct vo_driver video_out_rpi;
 extern const struct vo_driver video_out_tct;
 extern const struct vo_driver video_out_sixel;
+extern const struct vo_driver video_out_kitty;
 
 const struct vo_driver *const video_out_drivers[] =
 {
@@ -113,6 +114,9 @@ const struct vo_driver *const video_out_drivers[] =
 #endif
 #if HAVE_SIXEL
     &video_out_sixel,
+#endif
+#if HAVE_KITTY
+    &video_out_kitty,
 #endif
     &video_out_lavc,
     NULL
diff --git a/video/out/vo_kitty.c b/video/out/vo_kitty.c
new file mode 100644
index 0000000000..6cd5bfc026
--- /dev/null
+++ b/video/out/vo_kitty.c
@@ -0,0 +1,361 @@
+/*
+ * This file is part of mpv.
+ *
+ * mpv is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * mpv is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with mpv.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <fcntl.h>
+#include <stdio.h>
+#include <time.h>
+#include <libswscale/swscale.h>
+
+#include <unistd.h>
+#include <sys/mman.h>
+
+#include "config.h"
+#include "options/m_config.h"
+#include "osdep/terminal.h"
+#include "sub/osd.h"
+#include "vo.h"
+#include "video/mp_image.h"
+#include "video/sws_utils.h"
+
+#define TERMINAL_FALLBACK_PX_WIDTH  320
+#define TERMINAL_FALLBACK_PX_HEIGHT 240
+#define SHM_NAME "/kitty_img"
+#define SHM_NAME_BASE64 "a2l0dHlfaW1n"
+
+#define ESC_GOTOXY                  "\033[%d;%df"
+#define ESC_HIDE_CURSOR             "\033[?25l"
+#define ESC_RESTORE_CURSOR          "\033[?25h"
+#define ESC_CLEAR_SCREEN            "\033[2J"
+
+struct priv {
+    // User specified options
+    int opt_width;
+    int opt_height;
+    int opt_top;
+    int opt_left;
+    int opt_clear;
+
+    // Internal data
+    int             fmt;
+    int             mmap_fd;
+    uint8_t        *buffer;
+    bool            skip_frame_draw;
+
+    int left, top;  // image origin cell (1 based)
+    int width, height;  // actual image px size - always reflects dst_rect.
+    int num_cols, num_rows;  // terminal size in cells
+    int canvas_ok;  // whether canvas vo->dwidth and vo->dheight are positive
+
+    struct mp_rect src_rect;
+    struct mp_rect dst_rect;
+    struct mp_osd_res osd;
+    struct mp_image *frame;
+    struct mp_sws_context *sws;
+};
+
+static const unsigned int depth = 4;
+
+static void dealloc_buffers(struct vo *vo)
+{
+    struct priv *priv = vo->priv;
+
+    if (priv->buffer) {
+        munmap(priv->buffer, priv->width * priv->height * depth);
+        priv->buffer = NULL;
+    }
+
+    if (priv->mmap_fd != -1) {
+        close(priv->mmap_fd);
+        priv->mmap_fd = -1;
+    }
+
+    if (priv->frame) {
+        talloc_free(priv->frame);
+        priv->frame = NULL;
+    }
+}
+
+
+static void update_canvas_dimensions(struct vo *vo)
+{
+    struct priv *priv   = vo->priv;
+    int num_rows        = 0;
+    int num_cols        = 0;
+    int total_px_width  = 0;
+    int total_px_height = 0;
+
+    terminal_get_size2(&num_rows, &num_cols, &total_px_width, &total_px_height);
+
+    if (priv->opt_width > 0) {
+        total_px_width = priv->opt_width;
+    } else if (total_px_width <= 0) {
+       // ioctl failed to read terminal width
+       total_px_width = TERMINAL_FALLBACK_PX_WIDTH;
+    }
+
+    if (priv->opt_height > 0) {
+        total_px_height = priv->opt_height;
+    } else if (total_px_height <= 0) {
+        total_px_height = TERMINAL_FALLBACK_PX_HEIGHT;
+    }
+
+    vo->dheight = total_px_height;
+    vo->dwidth  = total_px_width;
+
+    priv->num_rows = num_rows;
+    priv->num_cols = num_cols;
+
+    priv->canvas_ok = vo->dwidth > 0 && vo->dheight > 0;
+}
+
+static void set_output_parameters(struct vo *vo)
+{
+    struct priv *priv = vo->priv;
+
+    vo_get_src_dst_rects(vo, &priv->src_rect, &priv->dst_rect, &priv->osd);
+
+    priv->width  = priv->dst_rect.x1 - priv->dst_rect.x0;
+    priv->height = priv->dst_rect.y1 - priv->dst_rect.y0;
+
+    priv->top  = (priv->opt_top  > 0) ?  priv->opt_top :
+                  priv->num_rows * priv->dst_rect.y0 / vo->dheight + 1;
+    priv->left = (priv->opt_left > 0) ?  priv->opt_left :
+                  priv->num_cols * priv->dst_rect.x0 / vo->dwidth  + 1;
+}
+
+static int update_params(struct vo *vo, struct mp_image_params *params)
+{
+    struct priv *priv = vo->priv;
+    priv->sws->src = *params;
+    priv->sws->src.w = mp_rect_w(priv->src_rect);
+    priv->sws->src.h = mp_rect_h(priv->src_rect);
+    priv->sws->dst = (struct mp_image_params) {
+        .imgfmt = priv->fmt,
+        .w = priv->width,
+        .h = priv->height,
+        .p_w = 1,
+        .p_h = 1,
+    };
+
+    dealloc_buffers(vo);
+
+    priv->frame = mp_image_alloc(IMGFMT_RGBA, priv->width, priv->height);
+    if (!priv->frame)
+        return -1;
+
+    if (mp_sws_reinit(priv->sws) < 0)
+        return -1;
+
+    return 0;
+}
+
+static int reconfig(struct vo *vo, struct mp_image_params *params)
+{
+    struct priv *priv = vo->priv;
+    int ret = 0;
+    update_canvas_dimensions(vo);
+
+    if (priv->canvas_ok) {  // if too small - succeed but skip the rendering
+        set_output_parameters(vo);
+        ret = update_params(vo, params);
+    }
+
+    printf(ESC_CLEAR_SCREEN);
+    vo->want_redraw = true;
+    return ret;
+}
+
+
+static void draw_frame(struct vo *vo, struct vo_frame *frame)
+{
+    struct priv *priv = vo->priv;
+    struct mp_image *mpi = NULL;
+    int  prev_height = vo->dheight;
+    int  prev_width  = vo->dwidth;
+    bool resized     = false;
+
+    update_canvas_dimensions(vo);
+    if (!priv->canvas_ok)
+        return;
+
+    if (prev_width != vo->dwidth || prev_height != vo->dheight) {
+        set_output_parameters(vo);
+        // Not checking for vo->config_ok because draw_frame is never called
+        // with a failed reconfig.
+        update_params(vo, vo->params);
+
+        printf(ESC_CLEAR_SCREEN);
+        resized = true;
+    }
+
+    if (frame->repeat && !frame->redraw && !resized) {
+        // Frame is repeated, and no need to update OSD either
+        priv->skip_frame_draw = true;
+        return;
+    } else {
+        // Either frame is new, or OSD has to be redrawn
+        priv->skip_frame_draw = false;
+    }
+
+    // Normal case where we have to draw the frame and the image is not NULL
+    if (frame->current) {
+        mpi = mp_image_new_ref(frame->current);
+        struct mp_rect src_rc = priv->src_rect;
+        src_rc.x0 = MP_ALIGN_DOWN(src_rc.x0, mpi->fmt.align_x);
+        src_rc.y0 = MP_ALIGN_DOWN(src_rc.y0, mpi->fmt.align_y);
+        mp_image_crop_rc(mpi, src_rc);
+
+        // scale/pan to our dest rect
+        mp_sws_scale(priv->sws, priv->frame, mpi);
+    } else {
+        // Image is NULL, so need to clear image and draw OSD
+        mp_image_clear(priv->frame, 0, 0, priv->width, priv->height);
+    }
+
+    struct mp_osd_res dim = {
+        .w = priv->width,
+        .h = priv->height
+    };
+    osd_draw_on_image(vo->osd, dim, mpi ? mpi->pts : 0, 0, priv->frame);
+
+
+    priv->mmap_fd = shm_open(SHM_NAME, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR);
+    if (priv->mmap_fd == -1) {
+        MP_WARN(vo, "Failed to create SHM object");
+        return;
+    }
+
+    int size = priv->width * priv->height * depth;
+
+    if (ftruncate(priv->mmap_fd, size) == -1) {
+        MP_WARN(vo, "Failed to truncate SHM object");
+        shm_unlink(SHM_NAME);
+        close(priv->mmap_fd);
+        return;
+    }
+
+    priv->buffer = mmap(NULL, size,
+      PROT_READ | PROT_WRITE, MAP_SHARED, priv->mmap_fd, 0);
+    if (priv->buffer == MAP_FAILED) {
+        MP_WARN(vo, "Failed to mmap SHM object");
+        shm_unlink(SHM_NAME);
+        close(priv->mmap_fd);
+        return;
+    }
+
+    memcpy_pic(priv->buffer, priv->frame->planes[0], priv->width * depth,
+               priv->height, priv->width * depth, priv->frame->stride[0]);
+
+    if (mpi)
+        talloc_free(mpi);
+}
+
+static void flip_page(struct vo *vo)
+{
+    struct priv* priv = vo->priv;
+    if (!priv->canvas_ok)
+        return;
+
+    if (priv->skip_frame_draw)
+        return;
+
+    printf(ESC_GOTOXY, priv->top, priv->left);
+    printf("\033_Ga=T,f=32,t=s,s=%u,v=%u;%s\033\\", priv->width, priv->height, SHM_NAME_BASE64);
+    fflush(stdout);
+
+    // SHM_NAME will be unlinked by kitty, so not unlinking in mpv
+    munmap(priv->buffer, priv->width * priv->height * depth);
+    priv->buffer = NULL;
+    close(priv->mmap_fd);
+    priv->mmap_fd = -1;
+}
+
+static int preinit(struct vo *vo)
+{
+    struct priv *priv = vo->priv;
+    printf(ESC_HIDE_CURSOR);
+    priv->mmap_fd = -1;
+
+    priv->sws = mp_sws_alloc(vo);
+    priv->sws->log = vo->log;
+    mp_sws_enable_cmdline_opts(priv->sws, vo->global);
+
+    return 0;
+}
+
+static int query_format(struct vo *vo, int format)
+{
+    struct priv *p = vo->priv;
+    if (mp_sws_supports_formats(p->sws, IMGFMT_RGB0, format)){
+        p->fmt = format;
+        return 1;
+    }
+    return 0;
+}
+
+static int control(struct vo *vo, uint32_t request, void *data)
+{
+    if (request == VOCTRL_SET_PANSCAN)
+        return (vo->config_ok && !reconfig(vo, vo->params)) ? VO_TRUE : VO_FALSE;
+    return VO_NOTIMPL;
+}
+
+static void uninit(struct vo *vo)
+{
+    struct priv *priv = vo->priv;
+
+    printf(ESC_RESTORE_CURSOR);
+
+    if (priv->opt_clear) {
+        printf(ESC_CLEAR_SCREEN);
+        printf(ESC_GOTOXY, 1, 1);
+    }
+    fflush(stdout);
+
+    dealloc_buffers(vo);
+}
+
+#define OPT_BASE_STRUCT struct priv
+
+const struct vo_driver video_out_kitty = {
+    .name = "kitty",
+    .description = "terminal graphics using kitty protocol",
+    .preinit = preinit,
+    .query_format = query_format,
+    .reconfig = reconfig,
+    .control = control,
+    .draw_frame = draw_frame,
+    .flip_page = flip_page,
+    .uninit = uninit,
+    .priv_size = sizeof(struct priv),
+    .priv_defaults = &(const struct priv) {
+        .opt_width = 0,
+        .opt_height = 0,
+        .opt_top = 0,
+        .opt_left = 0,
+        .opt_clear = 1,
+    },
+    .options = (const m_option_t[]) {
+        {"width", OPT_INT(opt_width)},
+        {"height", OPT_INT(opt_height)},
+        {"top", OPT_INT(opt_top)},
+        {"left", OPT_INT(opt_left)},
+        {"exit-clear", OPT_FLAG(opt_clear), },
+        {0}
+    },
+    .options_prefix = "vo-kitty",
+};
diff --git a/wscript b/wscript
index 2331624ff3..4964c7d02c 100644
--- a/wscript
+++ b/wscript
@@ -764,6 +764,10 @@ video_output_features = [
         'name': '--sixel',
         'desc': 'Sixel',
         'func': check_pkg_config('libsixel', '>= 1.5'),
+    }, {
+        'name': '--kitty',
+        'desc': 'Kitty Terminal Graphics',
+        'func': check_true
     }
 ]
 
diff --git a/wscript_build.py b/wscript_build.py
index 46191b8196..502f528a32 100644
--- a/wscript_build.py
+++ b/wscript_build.py
@@ -504,6 +504,7 @@ def build(ctx):
         ( "video/out/vo_rpi.c",                  "rpi-mmal" ),
         ( "video/out/vo_sdl.c",                  "sdl2-video" ),
         ( "video/out/vo_sixel.c",                "sixel" ),
+        ( "video/out/vo_kitty.c" ),
         ( "video/out/vo_tct.c" ),
         ( "video/out/vo_vaapi.c",                "vaapi-x11 && gpl" ),
         ( "video/out/vo_vdpau.c",                "vdpau" ),