feat: revamped screencast (#571)

* feat: revamped screencast

* refactor: use sharp instead of ffmpeg

Author: Kikobeats

packages/screencast/examples/server.js

@@ -0,0 +1,84 @@
+'use strict'
+
+const timeSpan = require('@kikobeats/time-span')({ format: n => `${n.toFixed(2)}ms` })
+const { createCanvas, Image } = require('canvas')
+const { GifEncoder } = require('@skyra/gifenc')
+const createBrowser = require('browserless')
+const sharp = require('sharp')
+const http = require('http')
+
+const createScreencast = require('..')
+
+const browser = createBrowser({
+  timeout: 25000,
+  lossyDeviceName: true,
+  ignoreHTTPSErrors: true
+})
+
+const CACHE = Object.create(null)
+
+const server = http.createServer(async (req, res) => {
+  if (req.url === '/favicon.ico') return res.end()
+
+  const duration = timeSpan()
+  let firstFrame = true
+
+  const url = req.url.slice(1)
+
+  if (CACHE[url]) {
+    const pngBuffer = CACHE[url].toBuffer('image/png')
+    res.setHeader('Content-Type', 'image/png')
+    res.write(pngBuffer)
+    return res.end()
+  }
+
+  const browserless = await browser.createContext()
+  const page = await browserless.page()
+  let lastCanvas = null
+
+  res.setHeader('Content-Type', 'image/gif')
+
+  const width = 1280
+  const height = 800
+  const deviceScaleFactor = 0.5
+
+  const outputSize = { width: width * deviceScaleFactor, height: height * deviceScaleFactor }
+
+  const canvas = createCanvas(outputSize.width, outputSize.height)
+  const ctx = canvas.getContext('2d')
+
+  const encoder = new GifEncoder(outputSize.width, outputSize.height)
+  encoder.createReadStream().pipe(res)
+
+  const screencast = createScreencast(page, { maxWidth: width, maxHeight: height })
+
+  screencast.onFrame(async data => {
+    const frame = Buffer.from(data, 'base64')
+    const buffer = await sharp(frame).resize(outputSize).toBuffer()
+
+    const img = new Image()
+    img.src = buffer
+    ctx.drawImage(img, 0, 0, img.width, img.height)
+    encoder.addFrame(ctx)
+
+    if (firstFrame === true) firstFrame = duration()
+
+    lastCanvas = canvas
+  })
+
+  screencast.start()
+  encoder.start()
+  await browserless.goto(page, { url })
+  encoder.finish()
+  await screencast.stop()
+
+  console.log(`\n  Resolved ${url}; first frame ${firstFrame}, total ${duration()}`)
+
+  CACHE[url] = lastCanvas
+})
+
+server.listen(3000, () =>
+  console.log(`
+  Listen: http://localhost:3000/{URL}
+ Example: http://localhost:3000/https://browserless.js.org\n`)
+)

packages/screencast/package.json

@@ -32,13 +32,14 @@
     "screencast",
     "video"
   ],
-  "dependencies": {
-    "tinyspawn": "~1.2.6"
-  },
   "devDependencies": {
     "@browserless/test": "^10.3.0",
+    "@kikobeats/time-span": "latest",
+    "@skyra/gifenc": "latest",
     "ava": "5",
-    "file-type": "16"
+    "canvas": "latest",
+    "sharp": "latest",
+    "tinyspawn": "latest"
   },
   "engines": {
     "node": ">= 12"

packages/screencast/src/index.js

@@ -1,86 +1,20 @@
 'use strict'
 
-const { unlink, readFile } = require('fs/promises')
-const { randomUUID } = require('crypto')
-const { Readable } = require('stream')
-const { tmpdir } = require('os')
-const $ = require('tinyspawn')
-const path = require('path')
-
-const { startScreencast } = require('./utils')
-
-// Inspired by https://github.com/microsoft/playwright/blob/37b3531a1181c99990899c15000925a98f035eb7/packages/playwright-core/src/server/chromium/videoRecorder.ts#L101
-const ffmpegArgs = format => {
-  // `-an` disables audio
-  // `-b:v 0` disables video bitrate control
-  // `-c:v` alias for -vcodec
-  // `-avioflags direct` reduces buffering
-  // `-probesize 32` size of the data to analyze to get stream information
-  // `-analyzeduration 0` specify how many microseconds are analyzed to probe the input
-  // `-fpsprobesize 0` set number of frames used to probe fps
-  // `-fflags nobuffer` disables buffering when reading or writing multimedia data
-  const args =
-    '-loglevel error -an -b:v 0 -avioflags direct -probesize 32 -analyzeduration 0 -fpsprobesize 0 -fflags nobuffer'
-
-  if (format === 'mp4') {
-    // ffmpeg -h encoder=h264
-    return `${args} -c:v libx264 -pix_fmt yuv420p -preset ultrafast -realtime true`
-  }
-
-  if (format === 'gif') {
-    return args
+const getCDPClient = page => page._client()
+
+module.exports = (page, opts) => {
+  const client = getCDPClient(page)
+  let onFrame
+
+  client.on('Page.screencastFrame', ({ data, metadata, sessionId }) => {
+    client.send('Page.screencastFrameAck', { sessionId }).catch(() => {})
+    if (metadata.timestamp) onFrame(data, metadata)
+  })
+
+  return {
+    // https://chromedevtools.github.io/devtools-protocol/tot/Page/#method-startScreencast
+    start: () => client.send('Page.startScreencast', opts),
+    onFrame: fn => (onFrame = fn),
+    stop: async () => client.send('Page.stopScreencast')
   }
-
-  if (format === 'webm') {
-    // ffmpeg -h encoder=vp9
-    return `${args} -c:v libvpx-vp9 -quality realtime`
-  }
-
-  throw new TypeError(`Format '${format}' not supported`)
-}
-
-module.exports = async ({
-  ffmpegPath,
-  format = 'webm',
-  frameRate = 25,
-  frames: framesOpts = {},
-  getBrowserless,
-  gotoOpts,
-  timeout,
-  tmpPath = tmpdir(),
-  withPage
-} = {}) => {
-  const browserless = await getBrowserless()
-
-  const fn = (page, goto) => async gotoOpts => {
-    await goto(page, gotoOpts)
-    const screencastStop = await startScreencast(page, framesOpts)
-    await withPage(page)
-    const frames = await screencastStop()
-
-    const interpolatedFrames = frames.reduce((acc, { data, metadata }, index) => {
-      const previousIndex = index - 1
-      const previousFrame = index > 0 ? frames[previousIndex] : undefined
-      const durationSeconds = previousFrame
-        ? metadata.timestamp - previousFrame.metadata.timestamp
-        : 0
-      const numFrames = Math.max(1, Math.round(durationSeconds * frameRate))
-      for (let i = 0; i < numFrames; i++) acc.push(data)
-      return acc
-    }, [])
-
-    const filepath = path.join(tmpPath, `${randomUUID()}.${format}`)
-    const subprocess = $(
-      `${ffmpegPath} -f image2pipe -i pipe:0 -r ${frameRate} ${ffmpegArgs(format)} ${filepath}`
-    )
-    Readable.from(interpolatedFrames).pipe(subprocess.stdin)
-
-    await subprocess
-    const buffer = await readFile(filepath)
-    await unlink(filepath)
-
-    return buffer
-  }
-
-  return browserless.withPage(fn, { timeout })(gotoOpts)
 }

packages/screencast/src/utils.js

@@ -1,83 +1,33 @@
 'use strict'
 
 const { getBrowserContext } = require('@browserless/test/util')
-const { writeFile } = require('fs/promises')
-const { randomUUID } = require('crypto')
-const FileType = require('file-type')
-const { unlinkSync } = require('fs')
-const { tmpdir } = require('os')
-const $ = require('tinyspawn')
-const path = require('path')
 const test = require('ava')
 
-const screencast = require('..')
+const createScreencast = require('..')
 
-const isCI = !!process.env.CI
+test('capture frames', async t => {
+  const frames = []
 
-test('get a webm video', async t => {
-  const [browserless, { stdout: ffmpegPath }] = await Promise.all([
-    getBrowserContext(t),
-    $('which ffmpeg')
-  ])
+  const browserless = await getBrowserContext(t)
+  const page = await browserless.page()
 
-  const buffer = await screencast({
-    getBrowserless: () => browserless,
-    ffmpegPath,
-    frames: {
-      everyNthFrame: 2
-    },
-    gotoOpts: {
-      url: 'https://vercel.com',
-      animations: true,
-      abortTypes: [],
-      waitUntil: 'load'
-    },
-    withPage: async page => {
-      const TOTAL_TIME = 7_000 * isCI ? 0.5 : 1
-
-      const timing = {
-        topToQuarter: (TOTAL_TIME * 1.5) / 7,
-        quarterToQuarter: (TOTAL_TIME * 0.3) / 7,
-        quarterToBottom: (TOTAL_TIME * 1) / 7,
-        bottomToTop: (TOTAL_TIME * 2) / 7
-      }
-
-      const scrollTo = (partial, ms) =>
-        page.evaluate(
-          (partial, ms) =>
-            new Promise(resolve => {
-              window.requestAnimationFrame(() => {
-                window.scrollTo({
-                  top: document.scrollingElement.scrollHeight * partial,
-                  behavior: 'smooth'
-                })
-                setTimeout(resolve, ms)
-              })
-            }),
-          partial,
-          ms
-        )
-
-      await scrollTo(1 / 3, timing.topToQuarter)
-      await scrollTo(2 / 3, timing.quarterToQuarter)
-      await scrollTo(3 / 3, timing.quarterToBottom)
-      await scrollTo(0, timing.bottomToTop)
-    }
+  const screencast = createScreencast(page, {
+    quality: 0,
+    format: 'png',
+    everyNthFrame: 1
   })
 
-  const { ext, mime } = await FileType.fromBuffer(buffer)
-  t.is(ext, 'webm')
-  t.is(mime, 'video/webm')
-
-  const filepath = path.join(tmpdir(), randomUUID())
-  t.teardown(() => unlinkSync(filepath))
-  await writeFile(filepath, buffer)
+  screencast.onFrame((data, metadata) => {
+    frames.push({ data, metadata })
+  })
 
-  const { stdout: ffprobe } = await $.json(
-    `ffprobe ${filepath} -print_format json -v quiet -show_format -show_streams -show_error`
-  )
+  screencast.start()
+  await page.goto('https://example.com', { waitUntil: 'domcontentloaded' })
+  await screencast.stop()
 
-  t.is(ffprobe.streams[0].codec_name, 'vp9')
-  t.is(ffprobe.streams[0].pix_fmt, 'yuv420p')
-  t.is(ffprobe.streams[0].avg_frame_rate, '25/1')
+  frames.forEach(({ data, metadata }) => {
+    t.truthy(data)
+    t.is(typeof metadata, 'object')
+    t.truthy(metadata.timestamp)
+  })
 })