Is there a way to generate an SRT file (or similar) using Google Cloud Speech?

Question

In order to generate subtitles for my videos, I converted them to audio files and used the Cloud Speech-to-Text. It works, but it only generates transcriptions, whereas what I need is a *.srt/*.vtt/similar file.

What I need is what YouTube does: to generate transcriptions and sync them with the video, like a subtitle format, ie.: transcriptions with the times when captions should appear.

Although I could upload them to YouTube and then download their auto-generated captions, it doesn't seem very correct.

Is there a way to generate an SRT file (or similar) using Google Cloud Speech?

Answer 1

There's no way really to do this directly from the Speech-to-Text API. What you could try to do is some post-processing on the speech recognition result.

For example, here's a request to the REST API using a model meant to transcribe video, with a public google-provided sample file:

curl -s -H "Content-Type: application/json" \
    -H "Authorization: Bearer "$(gcloud auth application-default print-access-token) \
    https://speech.googleapis.com/v1p1beta1/speech:longrunningrecognize \
    --data "{
  'config': {
    'encoding': 'LINEAR16',
    'sampleRateHertz': 16000,
    'languageCode': 'en-US',
    'enableWordTimeOffsets': true,
    'enableAutomaticPunctuation': true,
    'model': 'video'
  },
  'audio': {
    'uri':'gs://cloud-samples-tests/speech/Google_Gnome.wav'
  }
}"

The above uses asynchronous recognition (speech:longrunningrecognize), which is more fitting for larger files. Enabling punctuation ('enableAutomaticPunctuation': true) in combination with the start and end times of words ('enableWordTimeOffsets': true) near the start and end of each sentence (which you'd also have to convert from nanos to timestamps) could allow you to provide a text file in the srt format. You would probably also have to include some rules about the maximum length of a sentence appearing on the screen at any given time.

The above should not be too difficult to implement, however, there's a strong possibility that you would still encounter timing/synchronization issues.

Answer 2

There is no way to do it using Google Cloud itself buy as suggested you may post-process the result.

In this file I have made a quick code that kind of does the job. You may want to adapt it to your needs:

function convertGSTTToSRT(string) {
    var obj = JSON.parse(string);
    var i = 1;
    var result = ''
    for (const line of obj.response.results) {
        result += i++;
        result += '\n'
        var word = line.alternatives[0].words[0]
        var time = convertSecondStringToRealtime(word.startTime);
        result += formatTime(time) + ' --> '

        var word = line.alternatives[0].words[line.alternatives[0].words.length - 1]
        time = convertSecondStringToRealtime(word.endTime);
        result += formatTime(time) + '\n'
        result += line.alternatives[0].transcript + '\n\n'
    }
    return result;
}

function formatTime(time) {
    return String(time.hours).padStart(2, '0')+ ':' + String(time.minutes).padStart(2, '0') + ':' + 
   String(time.seconds).padStart(2, '0') + ',000';
}

function convertSecondStringToRealtime(string) {
    var seconds = string.substring(0, string.length - 1);
    var hours = Math.floor(seconds / 3600);
    var minutes = Math.floor(seconds % 3600 / 60);
    seconds = Math.floor(seconds % 3600 % 60);
    return {
        hours, minutes, seconds
    }
}

Answer 3

here is the code I used

    import math
import json
import datetime

def to_hms(s):
    m, s = divmod(s, 60)
    h, m = divmod(m, 60)
    return '{}:{:0>2}:{:0>2}'.format(h, m, s)
def srt_generation(filepath, filename):
    filename = 'DL_BIRTHDAY'
    with open('{}{}.json'.format(filepath, filename), 'r') as file:
        data = file.read()
        results = json.loads(data)['response']['annotationResults'][0]['speechTranscriptions']
        processed_results = []
        counter = 1
        
        lines = []
        wordlist = []
        for transcription in results:
            alternative = transcription['alternatives'][0]
            if alternative.has_key('transcript'):
            # print(counter)
    #            lines.append(counter)
                tsc = alternative['transcript']
                stime = alternative['words'][0]['startTime'].replace('s','').split('.')
                etime = alternative['words'][-1]['endTime'].replace('s','').split('.')
                if(len(stime) == 1):
                    stime.append('000')
                if(len(etime) == 1):
                    etime.append('000')
                lines.append('{}\n{},{} --> {},{}\n{}\n\n\n'.format(counter, to_hms(int(stime[0])), stime[1], to_hms(int(etime[0])), etime[1],tsc.encode('ascii', 'ignore')))
                counter = counter+1
                wordlist.extend(alternative['words'])

        srtfile = open('{}{}.srt'.format(filepath, filename), 'wr') 
        srtfile.writelines(lines)
        srtfile.close()        

        ## Now generate 3 seconds duration chunks of those words.
        lines = []
        counter = 1
        strtime =0
        entime = 0
        words = []
        standardDuration = 3
        srtcounter = 1
        for word in wordlist:
            stime = word['startTime'].replace('s','').split('.')
            etime = word['endTime'].replace('s','').split('.')
            if(len(stime) == 1):
                stime.append('000 ')
            if(len(etime) == 1):
                etime.append('000')
            if(counter == 1):
                strtime = '{},{}'.format(stime[0], stime[1])
                entime = '{},{}'.format(etime[0], etime[1])
                words.append(word['word'])

            else:
                tempstmime = int(stime[0])
                tempentime = int(etime[0])
                stimearr = strtime.split(',')
                etimearr = entime.split(',')
                if(tempentime - int(strtime.split(',')[0]) > standardDuration ):
                    transcript = ' '.join(words)
                    
                    lines.append('{}\n{},{} --> {},{}\n{}\n\n\n'.format(srtcounter, to_hms(int(stimearr[0])), stimearr[1], to_hms(int(etimearr[0])), etimearr[1],transcript.encode('ascii', 'ignore')))        
                    srtcounter = srtcounter+1
                    words = []
                    strtime = '{},{}'.format(stime[0], stime[1])
                    entime = '{},{}'.format(etime[0], etime[1])    
                    words.append(' ')
                    words.append(word['word'])
                    
                else:
                    words.append(' ')
                    words.append(word['word'])
                    entime = '{},{}'.format(etime[0], etime[1])
            counter = counter +1         
        if(len(words) > 0):
            tscp = ' '.join(words)
            stimearr = strtime.split(',')
            etimearr = entime.split(',')
            lines.append('{}\n{},{} --> {},{}\n{}\n\n\n'.format(srtcounter, to_hms(int(stimearr[0])), stimearr[1], to_hms(int(etimearr[0])), etimearr[1],tscp.encode('ascii', 'ignore')))        

        srtfile = open('{}{}_3_Sec_Custom.srt'.format(filepath, filename), 'wr') 
        srtfile.writelines(lines)
        srtfile.close()        

Answer 4

If you require a *.vtt file, here is a snippet to convert the API response received from GCP speech-to-text client into a valid *.vtt. Some answers above are for *.srt so sharing this here.

const client = new speech.SpeechClient();
const [response] = await client.recognize(request);

createVTT(response);

function createVTT(response) {
  const wordsArray = response.results[0].alternatives[0].words;

  let VTT = '';
  let buffer = [];
  const phraseLength = 10;
  let startPointer = '00:00:00';
  let endPointer = '00:00:00';

  VTT += 'WEBVTT\n\n';

  wordsArray.forEach((wordItem) => {
    const { startTime, endTime, word } = wordItem;
    const start = startTime.seconds;
    const end = endTime.seconds;

    if (buffer.length === 0) {
      // first word of the phrase
      startPointer = secondsToFormat(start);
    }

    if (buffer.length < phraseLength) {
      buffer.push(word);
    }
    if (buffer.length === phraseLength) {
      endPointer = secondsToFormat(end);

      const phrase = buffer.join(' ');

      VTT += `${startPointer + ' --> ' + endPointer}\n`;
      VTT += `${phrase}\n\n`;

      buffer = [];
    }
  });

  if (buffer.length) {
    // handle the left over buffer items
    const lastItem = wordsArray[wordsArray.length - 1];
    const end = lastItem.endTime.seconds;

    endPointer = secondsToFormat(end);
    const phrase = buffer.join(' ');

    VTT += `${startPointer + ' --> ' + endPointer}\n`;
    VTT += `${phrase}\n\n`;
  }

  return VTT;
}

function secondsToFormat(seconds) {
  const timeHours = Math.floor(seconds / 3600)
    .toString()
    .padStart(2, '0');
  const timeMinutes = Math.floor(seconds / 60)
    .toString()
    .padStart(2, '0');
  const timeSeconds = (seconds % 60).toString().padStart(2, '0');

  const formattedTime = timeHours + ':' + timeMinutes + ':' + timeSeconds + '.000';
  return formattedTime;
}

Note: enableWordTimeOffsets: true must be set but that's already answered above. This answer is for people who want .vtt copy.

Hope this was helpful to someone :)

Answer 5

Use this request parameter "enable_word_time_offsets: True" to get the time stamps for the word groups. Then create an srt programmatically.