How to convert vp9 data to the format which MediaSource supports?

[go-jar]

I want to fill the buffer of the MediaSource. But there is an error TypeError: Type error at the last line:

    // Check that browser has support for media codec
    var mimeCodec = 'video/webm; codecs="vp9, vorbis"';
    console.log(MediaSource.isTypeSupported(mimeCodec));

    // Create Media Source
    var mediaSource = new MediaSource(); // mediaSource.readyState === 'closed'

    // Get video element
    var video = document.querySelector('video');

    // Attach media source to video element
    video.src = URL.createObjectURL(mediaSource);

    var buffer;

    // Wait for media source to be open
    // mediaSource.addEventListener('sourceopen', handleSourceOpen.bind(mediaSource));
    mediaSource.addEventListener('sourceopen', function(e) {
        video.play();
        buffer = mediaSource.addSourceBuffer(mimeCodec);

        buffer.addEventListener('updatestart', function(e) { console.log('updatestart: ' + mediaSource.readyState); });
        buffer.addEventListener('update', function(e) { console.log('update: ' + mediaSource.readyState); });
        buffer.addEventListener('updateend', function(e) { console.log('updateend: ' + mediaSource.readyState); });
        buffer.addEventListener('error', function(e) { console.log('error: ' + mediaSource.readyState); });
        buffer.addEventListener('abort', function(e) { console.log('abort: ' + mediaSource.readyState); });

        buffer.addEventListener('update', function() { // Note: Have tried 'updateend'
            if (frames.length > 0 && !buffer.updating) {
                buffer.appendBuffer(frames.shift());
            }
        });
    }, false);

    mediaSource.addEventListener('sourceopen', function(e) { console.log('sourceopen: ' + mediaSource.readyState); });
    mediaSource.addEventListener('sourceended', function(e) { console.log('sourceended: ' + mediaSource.readyState); });
    mediaSource.addEventListener('sourceclose', function(e) { console.log('sourceclose: ' + mediaSource.readyState); });
    mediaSource.addEventListener('error', function(e) { console.log('error: ' + mediaSource.readyState); });

    var frames = [];
    let tauri_window = tauri.window.getCurrent();
    let listen_video = tauri_window.listen("video-event", event => {
        if (buffer.updating) {
            frames.push(event.payload.frames);
        } else {
            buffer.appendBuffer(event.payload.frames);  // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! TypeError: Type error
        }
    });

The event.payload.frames are just VP9 frames. And the VP9 frames are built as follows:

    #[derive(serde::Serialize)]
    pub struct VP9Frame {
        pub data: ::std::vec::Vec<u8>,
        pub key: bool,
        pub pts: i64,
    }

    #[derive(serde::Serialize)]
    pub struct VideoFrame {
        pub frames: Vec<VP9Frame>,
    }

    fn handle_one_frame(&mut self, image: &[u8]) -> ResultType<(String, Vec<VP9Frame>)> {
        let mut yuv: Vec<u8> = Default::default();
        scrap::rgb_to_i420(
            self.video_encoder.frame_width as usize,
            self.video_encoder.frame_height as usize,
            image,
            &mut yuv,
        );
        let now = Instant::now();
        let time = now - self.video_encoder.start;
        let ms = (time.as_secs() * 1000 + time.subsec_millis() as u64) as i64;
        let mut frames = Vec::new();
        let mut hasher = crc32fast::Hasher::new();
        hasher.update(&yuv);
        let checksum = hasher.finalize();
        if checksum != self.video_encoder.crc.0 {
            self.video_encoder.crc.0 = checksum;
            self.video_encoder.crc.1 = 0;
        } else {
            self.video_encoder.crc.1 += 1;
        }
        if self.video_encoder.crc.1 <= 180 && self.video_encoder.crc.1 % 5 == 0 {
            for ref frame in self
                .video_encoder
                .vpx
                .encode(ms, &yuv, STRIDE_ALIGN)
                .map_err(|e| anyhow!("Failed to encode: {}", e))?
            {
                frames.push(create_frame(frame));
            }
            for ref frame in self
                .video_encoder
                .vpx
                .flush()
                .map_err(|_e| anyhow!("Failed to flush"))?
            {
                frames.push(create_frame(frame));
            }
        }
        Ok(("".to_owned(), frames))
    }

How to convert VP9 frames to the format which MediaSource supports?

[justthatrio]

The mimeCodec should not be video/webm; codecs="vp9, vorbis" but instead it should be video/webm; vp9, vorbis

[SakhnevichKirill]

Judging by the documentation, "MediaSource" for Webm also needs codecs keyword: video/webm;codecs='vp9,vorbis'

[justthatrio]

Interesting, I only commented that because making the change I described is what fixed my issue. with video/webm; codecs="vp9, vorbis" it didn't work and with video/webm; vp9, vorbis it did

[SakhnevichKirill]

And I got an unsupported error with video/webm; vp9, vorbis

[justthatrio]

Hmm that's strange, I just reconfirmed that in my manifest file my clients are being served it does say video/webm; vp9, vorbis without the codecs part, and that my video player code is using that value as is without modifying it. Ive only tested on Safari, Firefox and Edge, so Ill have to test it with google some time soon.

[SakhnevichKirill]

To solve this problem, you need to get from VideoFrame.frames only the last frame of the VP9Frame structure and pass VP9Frame.data as an argument to the appendBuffer method.

Also to make sure, you can try to decode the last frame using rust and then transmit only the last frame