Upload HLS playlist manifest file and media segment files to GCP directly

Question

I am working on VOD feature and for the past few days I am unable to output generated HLS playlist and files to GCP. I am using fluent-ffmpeg for uploading hls playlist to GCP. Here is the test code environment for uploading HLS to GCP using Node.js

const path = require('path');
const { Storage } = require('@google-cloud/storage');
const { createHLSPlaylist } = require('./utils/ffmpeg');

//* Creates a client, keyFileName is link to service account authentication credentials
const storage = new Storage({
  keyFilename: path.join(
    __dirname,
    './config/FILE.json'
  ),
  projectId: process.env.GCP_PROJECT_ID,
});

//* To be used in upload GCP videos controller (copy this file contents within controller)
const url = 'URL';
const bucket = storage.bucket('BUCKETNAME');
const playlistName = 'file.m3u8';

const newFile = bucket.file(playlistName);

createHLSPlaylist(url, newFile.createWriteStream());

and here is the ffmpeg code that transcodes video in url to hls playlist

const ffmpeg = require('fluent-ffmpeg');
const ffmpegInstaller = require('@ffmpeg-installer/ffmpeg');

ffmpeg.setFfmpegPath(ffmpegInstaller.path);

const createHLSPlaylist = (url, stream) => {
  const command = ffmpeg(url, { timeout: 432000 })
    .addOption([
      '-profile:v baseline',
      '-level 3.0',
      '-start_number 0',
      '-hls_time 10',
      '-hls_list_size 0',
      '-f hls',
    ])
    .on('end', (stdout, stderr) => {
      console.log('Transcoding succeeded !');
      process.exit(1);
    })
    .on('start', (commandLine) => {
      console.log('start', commandLine);
    })
    .on('codecData', (data) => {
      console.log(`Input is ${data.audio} audio with ${data.video} video`);
    })
    .on('progress', (progress) => {
      console.log(`Processing. Timemark: -> ${progress.timemark}`);
    })
    .on('stderr', (stderrLine) => {
      // do nothing
    })
    .on('error', (err, stdout, stderr) => {
      console.log(`error1: ${err}`);
      console.log(`error2: ${stdout}`);
      console.log(`error3: ${stderr}`);
      console.log(`Cannot process video: ${err.message}`);
    })
    .on('data', (chunk) => {
      console.log(`ffmpeg just wrote ${chunk.length} bytes`);
    })
    .pipe(stream, { end: true }); //end = true, close output stream after writing
};

module.exports = {
  createHLSPlaylist,
};

I get the following messages via console.log

Processing. Timemark: -> 00:00:28.62

Processing. Timemark: -> 00:00:28.62

Processing. Timemark: -> 00:00:28.62

Processing. Timemark: -> 00:00:28.89

Transcoding succeeded !

But nothing in GCP bucket, any pointers will be appreciated

EDIT: I also converted URL to readStream and followed the following answer on github but this time no progress is showed.

@AkshanshaSinghal yes I have, I am able to upload videos but unfortunately I am unable to directly upload ffmpeg encoded hls playlist to GCP — dev_2k20, Dec 28 '21 at 10:37

score 2 · Accepted Answer · answered Dec 29 '21 at 07:12

For the time being, I have managed to upload HLS playlist to GCP by using a temporary file storage as mentioned here. I managed to solve this via the following way

newVideo = somebucket.file(filename);

      file
        .pipe(newVideo.createWriteStream({ gzip: 'auto' }))
        .on('error', (err) => {
          console.log(err);
        })
        .on('finish', () => {
          console.log(`File ${filename} upload finished`);
          newVideo
            .getSignedUrl({ action: 'read', expires: '03-17-2022' })
            .then((url) => {
              const command = ffmpeg(url[0], { timeout: 432000 })
                .addOptions([
                  '-profile:v baseline',
                  '-f segment',
                  '-level 3.0',
                  '-start_number 0',
                  '-hls_base_url segments/',
                  `-hls_segment_filename ./assets/${playlistName}/file%03d.ts`,
                  // Apple recommended time = 6,
                  // In seconds
                  '-hls_time 6',
                  '-hls_list_size 0',
                  // format
                  '-f hls',
                ])
                .output(`./assets/${playlistName}/${playlistName}.m3u8`)
                .on('end', async () => {
                  console.log('Transcoding succeeded !');
                  try {
                    const files = await readdir(`./assets/${playlistName}`);
                    console.log('files', files);
                    await Promise.all(
                      files.map((folderFile) =>
                        securebucket.upload(
                          `./assets/${playlistName}/${folderFile}`,
                          {
                            destination: folderFile,
                          }
                        )
                      )
                    ).then(() => {
                      fs.rm(
                        `./assets/${playlistName}`,
                        { recursive: true },
                        (err) => {
                          if (err) {
                            throw err;
                          }
                          console.log(`./assets/${playlistName} is deleted!`);
                          res.status(201).json(url[0]);
                          res.end();
                        }
                      );
                    });
                  } catch (err) {
                    console.error(err);
                  }
                })
                .on('start', (commandLine) => {
                  console.log('start', commandLine);
                })
                .on('codecData', (data) => {
                  console.log(
                    `Input is ${data.audio} audio with ${data.video} video`
                  );
                })
                .on('progress', (progress) => {
                  console.log(`Processing. Timemark: -> ${progress.timemark}`);
                })
                .on('error', (err, stdout, stderr) => {
                  console.log(`Cannot process video: ${err.message}`);
                });

              fs.mkdir(`./assets/${playlistName}`, (err) => {
                if (err && err.code !== 'EEXIST') {
                  throw err;
                } else {
                  command.run();
                }
              });
            });

I am still not convinced with this approach as I believe this could have been achieved in an easier way via streaming but seems like I am unable to find a definitive answer to my solution since fluent-ffmpeg didn't provide an example of s3 or GCP

Upload HLS playlist manifest file and media segment files to GCP directly

1 Answers1