Merge pull request #42 from hughfenghen/feat/mp4-previewer

Feat/mp4 previewer
bilibili · Jan 15, 2024 · 85aafa7 · 85aafa7
2 parents 5d9f7df + 5906270
commit 85aafa7
Show file tree

Hide file tree

Showing 9 changed files with 576 additions and 188 deletions.
diff --git a/doc-site/docs/demo/1_4-mp4-previewer.md b/doc-site/docs/demo/1_4-mp4-previewer.md
@@ -0,0 +1,51 @@
+---
+nav: DEMO
+group: 解码
+order: 4
+---
+
+# 视频预览
+
+从 MP4 文件中提取指定时间的图像，点击 Slider 预览任意时间点的图像。
+
+```tsx
+import React, { useState } from 'react';
+import { Slider } from 'antd';
+import { MP4Previewer } from '@webav/av-cliper';
+import { assetsPrefix } from './utils';
+
+const videoSrc = assetsPrefix(['video/webav1.mp4']);
+
+const previewer = new MP4Previewer((await fetch(videoSrc)).body!);
+const mp4Info = await previewer.getInfo();
+const duration = Number((mp4Info.duration / mp4Info.timescale).toFixed(0));
+
+export default function UI() {
+  const [imgSrc, setImgSrc] = useState<string>('');
+
+  return (
+    <div>
+      <div style={{ display: 'flex', alignItems: 'center' }}>
+        <span> 时间：</span>
+        <div style={{ flex: 1 }}>
+          <Slider
+            min={0}
+            max={duration}
+            step={0.1}
+            onChange={async (val) => {
+              setImgSrc(await previewer.getImage(val));
+            }}
+          />
+        </div>
+      </div>
+      {imgSrc && <img src={imgSrc} style={{ width: '100%' }} />}
+    </div>
+  );
+}
+```
+
+:::info
+如果只是为了绘制图像，使用视频帧更合适，`await previewer.getVideoFrame(time)`。
+
+**注意**，视频帧使用完需要立即调用 `videoFrame.close()`
+:::
diff --git a/packages/av-cliper/demo/mp4-previewer.demo.ts b/packages/av-cliper/demo/mp4-previewer.demo.ts
@@ -0,0 +1,20 @@
+import { MP4Previewer } from "../src/mp4-utils/mp4-previewer";
+import { OPFSFileWrap } from "../src/mp4-utils/opfs-file-wrap";
+
+const previewer = new MP4Previewer((await fetch('./video/webav1.mp4')).body!)
+const imgEl = document.querySelector('#img') as HTMLImageElement
+
+for (let i = 0; i < 10; i += 1) {
+  const t = performance.now()
+  const img = await previewer.getImage(i)
+  console.log('cost:', performance.now() - t, img)
+  if (img == null) break
+  imgEl.src = img
+}
+
+
+// 测试 OPFSFileWrap
+const opfsFile = new OPFSFileWrap('1111')
+
+await opfsFile.write(new Uint8Array([1, 2, 3, 4, 5]))
+await opfsFile.write(new Uint8Array([6, 7, 8, 9, 0]))
diff --git a/packages/av-cliper/demo/mp4-previewer.html b/packages/av-cliper/demo/mp4-previewer.html
@@ -0,0 +1,21 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>MP4Previewer</title>
+  <style>
+    #img {
+      width: 900px;
+      height: 500px;
+    }
+  </style>
+</head>
+
+<body>
+  <img id="img">
+  <script src="./mp4-previewer.demo.ts" type="module"></script>
+</body>
+
+</html>
diff --git a/...av-cliper/src/__tests__/mp4-utils.test.ts → ...src/mp4-utils/__tests__/mp4-utils.test.ts b/...av-cliper/src/__tests__/mp4-utils.test.ts → ...src/mp4-utils/__tests__/mp4-utils.test.ts
@@ -1,7 +1,7 @@
-import './mock'
+import '../../__tests__/mock'
 import { beforeAll, describe, expect, test, vi } from 'vitest'
 import mp4box from '@webav/mp4box.js'
-import { file2stream } from '../mp4-utils'
+import { file2stream } from '..'
 
 beforeAll(() => {
   vi.useFakeTimers()

diff --git a/packages/av-cliper/src/mp4-utils.ts → packages/av-cliper/src/mp4-utils/index.ts b/packages/av-cliper/src/mp4-utils.ts → packages/av-cliper/src/mp4-utils/index.ts
@@ -5,22 +5,22 @@ import mp4box, {
   MP4Sample,
   SampleOpts,
   TrakBoxParser,
-  VideoTrackOpts,
-  AudioTrackOpts,
-  MP4ABoxParser
 } from '@webav/mp4box.js'
-import { Log } from './log'
+import { Log } from '../log'
 import {
   autoReadStream,
   extractPCM4AudioData,
   extractPCM4AudioBuffer,
   mixinPCM,
   ringSliceFloat32Array,
-  sleep,
   concatPCMFragments
-} from './av-utils'
-import { DEFAULT_AUDIO_CONF } from './clips'
-import { EventTool } from './event-tool'
+} from '../av-utils'
+import { DEFAULT_AUDIO_CONF } from '../clips'
+import { EventTool } from '../event-tool'
+import { SampleTransform } from './sample-transform'
+import { extractFileConfig, sample2ChunkOpts } from './mp4box-utils'
+
+export { MP4Previewer } from './mp4-previewer'
 
 type TCleanFn = () => void
 
@@ -379,94 +379,7 @@ export function _deprecated_stream2file(stream: ReadableStream<Uint8Array>): {
   }
 }
 
-/**
- * 将原始字节流转换成 MP4Sample 流
- */
-class SampleTransform {
-  readable: ReadableStream<
-    | {
-      chunkType: 'ready'
-      data: { info: MP4Info; file: MP4File }
-    }
-    | {
-      chunkType: 'samples'
-      data: { id: number; type: 'video' | 'audio'; samples: MP4Sample[] }
-    }
-  >
-
-  writable: WritableStream<Uint8Array>
-
-  #inputBufOffset = 0
-
-  constructor() {
-    const file = mp4box.createFile()
-    let outCtrlDesiredSize = 0
-    let streamCancelled = false
-    this.readable = new ReadableStream(
-      {
-        start: ctrl => {
-          file.onReady = info => {
-            const vTrackId = info.videoTracks[0]?.id
-            if (vTrackId != null)
-              file.setExtractionOptions(vTrackId, 'video', { nbSamples: 100 })
-
-            const aTrackId = info.audioTracks[0]?.id
-            if (aTrackId != null)
-              file.setExtractionOptions(aTrackId, 'audio', { nbSamples: 100 })
-
-            ctrl.enqueue({ chunkType: 'ready', data: { info, file } })
-            file.start()
-          }
 
-          file.onSamples = (id, type, samples) => {
-            ctrl.enqueue({
-              chunkType: 'samples',
-              data: { id, type, samples }
-            })
-            outCtrlDesiredSize = ctrl.desiredSize ?? 0
-          }
-
-          file.onFlush = () => {
-            ctrl.close()
-          }
-        },
-        pull: ctrl => {
-          outCtrlDesiredSize = ctrl.desiredSize ?? 0
-        },
-        cancel: () => {
-          file.stop()
-          streamCancelled = true
-        }
-      },
-      {
-        // 每条消息 100 个 samples
-        highWaterMark: 50
-      }
-    )
-
-    this.writable = new WritableStream({
-      write: async ui8Arr => {
-        if (streamCancelled) {
-          this.writable.abort()
-          return
-        }
-
-        const inputBuf = ui8Arr.buffer as MP4ArrayBuffer
-        inputBuf.fileStart = this.#inputBufOffset
-        this.#inputBufOffset += inputBuf.byteLength
-        file.appendBuffer(inputBuf)
-
-        // 等待输出的数据被消费
-        while (outCtrlDesiredSize < 0) await sleep(50)
-      },
-      close: () => {
-        file.flush()
-        file.stop()
-        file.onFlush?.()
-      }
-    })
-  }
-}
 
 export function file2stream(
   file: MP4File,
@@ -661,24 +574,6 @@ function mp4File2OPFSFile(inMP4File: MP4File): () => (Promise<File | null>) {
   }
 }
 
-// track is H.264, H.265 or VPX.
-function parseVideoCodecDesc(track: TrakBoxParser): Uint8Array {
-  for (const entry of track.mdia.minf.stbl.stsd.entries) {
-    // @ts-expect-error
-    const box = entry.avcC ?? entry.hvcC ?? entry.vpcC
-    if (box != null) {
-      const stream = new mp4box.DataStream(
-        undefined,
-        0,
-        mp4box.DataStream.BIG_ENDIAN
-      )
-      box.write(stream)
-      return new Uint8Array(stream.buffer.slice(8)) // Remove the box header.
-    }
-  }
-  throw Error('avcC, hvcC or VPX not found')
-}
-
 /**
  * EncodedAudioChunk | EncodedVideoChunk 转换为 MP4 addSample 需要的参数
  */
@@ -699,61 +594,7 @@ function chunk2MP4SampleOpts(
   }
 }
 
-function extractFileConfig(file: MP4File, info: MP4Info) {
-  const vTrack = info.videoTracks[0]
-  const rs: {
-    videoTrackConf?: VideoTrackOpts
-    videoDecoderConf?: Parameters<VideoDecoder['configure']>[0]
-    audioTrackConf?: AudioTrackOpts
-    audioDecoderConf?: Parameters<AudioDecoder['configure']>[0]
-  } = {}
-  if (vTrack != null) {
-    const videoDesc = parseVideoCodecDesc(file.getTrackById(vTrack.id)).buffer
-    const { descKey, type } = vTrack.codec.startsWith('avc1')
-      ? { descKey: 'avcDecoderConfigRecord', type: 'avc1' }
-      : vTrack.codec.startsWith('hvc1')
-        ? { descKey: 'hevcDecoderConfigRecord', type: 'hvc1' }
-        : { descKey: '', type: '' }
-    if (descKey !== '') {
-      rs.videoTrackConf = {
-        timescale: vTrack.timescale,
-        duration: vTrack.duration,
-        width: vTrack.video.width,
-        height: vTrack.video.height,
-        brands: info.brands,
-        type,
-        [descKey]: videoDesc
-      }
-    }
-
-    rs.videoDecoderConf = {
-      codec: vTrack.codec,
-      codedHeight: vTrack.video.height,
-      codedWidth: vTrack.video.width,
-      description: videoDesc
-    }
-  }
 
-  const aTrack = info.audioTracks[0]
-  if (aTrack != null) {
-    rs.audioTrackConf = {
-      timescale: aTrack.timescale,
-      samplerate: aTrack.audio.sample_rate,
-      channel_count: aTrack.audio.channel_count,
-      hdlr: 'soun',
-      type: aTrack.codec.startsWith('mp4a') ? 'mp4a' : aTrack.codec,
-      description: getESDSBoxFromMP4File(file)
-    }
-    rs.audioDecoderConf = {
-      codec: aTrack.codec.startsWith('mp4a')
-        ? DEFAULT_AUDIO_CONF.codec
-        : aTrack.codec,
-      numberOfChannels: aTrack.audio.channel_count,
-      sampleRate: aTrack.audio.sample_rate
-    }
-  }
-  return rs
-}
 
 /**
  * 快速顺序合并多个mp4流，要求所有mp4的属性是一致的
@@ -1110,17 +951,6 @@ export function mixinMP4AndAudio(
   return outStream
 }
 
-function sample2ChunkOpts(
-  s: MP4Sample
-): EncodedAudioChunkInit | EncodedVideoChunkInit {
-  return {
-    type: (s.is_sync ? 'key' : 'delta') as EncodedVideoChunkType,
-    timestamp: (1e6 * s.cts) / s.timescale,
-    duration: (1e6 * s.duration) / s.timescale,
-    data: s.data
-  }
-}
-
 function createESDSBox(config: ArrayBuffer | ArrayBufferView) {
   const configlen = config.byteLength
   const buf = new Uint8Array([
@@ -1168,11 +998,3 @@ function createESDSBox(config: ArrayBuffer | ArrayBufferView) {
   return esdsBox
 }
 
-function getESDSBoxFromMP4File(file: MP4File, codec = 'mp4a') {
-  const mp4aBox = file.moov?.traks.map(
-    t => t.mdia.minf.stbl.stsd.entries
-  ).flat()
-    .find(({ type }) => type === codec) as MP4ABoxParser
-
-  return mp4aBox?.esds
-}