6

I am trying to create a rtsp Server that streams the OpenGL output of my program. I had a look at How to write a Live555 FramedSource to allow me to stream H.264 live, but I need the stream to be unicast. So I had a look at testOnDemandRTSPServer. Using the same Code fails. To my understanding I need to provide memory in which I store my h264 frames so the OnDemandServer can read them on Demand.

H264VideoStreamServerMediaSubsession.cpp

H264VideoStreamServerMediaSubsession*
H264VideoStreamServerMediaSubsession::createNew(UsageEnvironment& env,
                          Boolean reuseFirstSource) {
  return new H264VideoStreamServerMediaSubsession(env, reuseFirstSource);
}

H264VideoStreamServerMediaSubsession::H264VideoStreamServerMediaSubsession(UsageEnvironment& env, Boolean reuseFirstSource)
  : OnDemandServerMediaSubsession(env, reuseFirstSource), fAuxSDPLine(NULL), fDoneFlag(0), fDummyRTPSink(NULL) {
}

H264VideoStreamServerMediaSubsession::~H264VideoStreamServerMediaSubsession() {
  delete[] fAuxSDPLine;
}

static void afterPlayingDummy(void* clientData) {
  H264VideoStreamServerMediaSubsession* subsess = (H264VideoStreamServerMediaSubsession*)clientData;
  subsess->afterPlayingDummy1();
}

void H264VideoStreamServerMediaSubsession::afterPlayingDummy1() {
  // Unschedule any pending 'checking' task:
  envir().taskScheduler().unscheduleDelayedTask(nextTask());
  // Signal the event loop that we're done:
  setDoneFlag();
}

static void checkForAuxSDPLine(void* clientData) {
  H264VideoStreamServerMediaSubsession* subsess = (H264VideoStreamServerMediaSubsession*)clientData;
  subsess->checkForAuxSDPLine1();
}

void H264VideoStreamServerMediaSubsession::checkForAuxSDPLine1() {
  char const* dasl;

  if (fAuxSDPLine != NULL) {
    // Signal the event loop that we're done:
    setDoneFlag();
  } else if (fDummyRTPSink != NULL && (dasl = fDummyRTPSink->auxSDPLine()) != NULL) {
    fAuxSDPLine = strDup(dasl);
    fDummyRTPSink = NULL;

    // Signal the event loop that we're done:
    setDoneFlag();
  } else {
    // try again after a brief delay:
    int uSecsToDelay = 100000; // 100 ms
    nextTask() = envir().taskScheduler().scheduleDelayedTask(uSecsToDelay,
                  (TaskFunc*)checkForAuxSDPLine, this);
  }
}

char const* H264VideoStreamServerMediaSubsession::getAuxSDPLine(RTPSink* rtpSink, FramedSource* inputSource) {
  if (fAuxSDPLine != NULL) return fAuxSDPLine; // it's already been set up (for a previous client)

  if (fDummyRTPSink == NULL) { // we're not already setting it up for another, concurrent stream
    // Note: For H264 video files, the 'config' information ("profile-level-id" and "sprop-parameter-sets") isn't known
    // until we start reading the file.  This means that "rtpSink"s "auxSDPLine()" will be NULL initially,
    // and we need to start reading data from our file until this changes.
    fDummyRTPSink = rtpSink;

    // Start reading the file:
    fDummyRTPSink->startPlaying(*inputSource, afterPlayingDummy, this);

    // Check whether the sink's 'auxSDPLine()' is ready:
    checkForAuxSDPLine(this);
  }

  envir().taskScheduler().doEventLoop(&fDoneFlag);

  return fAuxSDPLine;
}

FramedSource* H264VideoStreamServerMediaSubsession::createNewStreamSource(unsigned /*clientSessionId*/, unsigned& estBitrate) {
  estBitrate = 500; // kb
  megamol::remotecontrol::View3D_MRC *parent = (megamol::remotecontrol::View3D_MRC*)this->parent;
  return H264VideoStreamFramer::createNew(envir(), parent->h264FramedSource);
}

RTPSink* H264VideoStreamServerMediaSubsession::createNewRTPSink(Groupsock* rtpGroupsock, unsigned char rtpPayloadTypeIfDynamic, FramedSource* /*inputSource*/) {
  return H264VideoRTPSink::createNew(envir(), rtpGroupsock, rtpPayloadTypeIfDynamic);
}

FramedSource.cpp

H264FramedSource* H264FramedSource::createNew(UsageEnvironment& env,
                                          unsigned preferredFrameSize,
                                          unsigned playTimePerFrame)
{
    return new H264FramedSource(env, preferredFrameSize, playTimePerFrame);
}

H264FramedSource::H264FramedSource(UsageEnvironment& env,
                               unsigned preferredFrameSize,
                               unsigned playTimePerFrame)
    : FramedSource(env),
    fPreferredFrameSize(fMaxSize),
    fPlayTimePerFrame(playTimePerFrame),
    fLastPlayTime(0),
    fCurIndex(0)
{

    x264_param_default_preset(&param, "veryfast", "zerolatency");
    param.i_threads = 1;
    param.i_width = 1024;
    param.i_height = 768;
    param.i_fps_num = 30;
    param.i_fps_den = 1;
    // Intra refres:
    param.i_keyint_max = 60;
    param.b_intra_refresh = 1;
    //Rate control:
    param.rc.i_rc_method = X264_RC_CRF;
    param.rc.f_rf_constant = 25;
    param.rc.f_rf_constant_max = 35;
    param.i_sps_id = 7;
    //For streaming:
    param.b_repeat_headers = 1;
    param.b_annexb = 1;
    x264_param_apply_profile(&param, "baseline");

    param.i_log_level = X264_LOG_ERROR;

    encoder = x264_encoder_open(&param);
    pic_in.i_type            = X264_TYPE_AUTO;
    pic_in.i_qpplus1         = 0;
    pic_in.img.i_csp         = X264_CSP_I420;
    pic_in.img.i_plane       = 3;


    x264_picture_alloc(&pic_in, X264_CSP_I420, 1024, 768);

    convertCtx = sws_getContext(1024, 768, PIX_FMT_RGBA, 1024, 768, PIX_FMT_YUV420P, SWS_FAST_BILINEAR, NULL, NULL, NULL);
    eventTriggerId = envir().taskScheduler().createEventTrigger(deliverFrame0);
}

H264FramedSource::~H264FramedSource()
{
    envir().taskScheduler().deleteEventTrigger(eventTriggerId);
    eventTriggerId = 0;
}

void H264FramedSource::AddToBuffer(uint8_t* buf, int surfaceSizeInBytes)
{
    uint8_t* surfaceData = (new uint8_t[surfaceSizeInBytes]);

    memcpy(surfaceData, buf, surfaceSizeInBytes);

    int srcstride = 1024*4;
    sws_scale(convertCtx, &surfaceData, &srcstride,0, 768, pic_in.img.plane, pic_in.img.i_stride);
    x264_nal_t* nals = NULL;
    int i_nals = 0;
    int frame_size = -1;


    frame_size = x264_encoder_encode(encoder, &nals, &i_nals, &pic_in, &pic_out);

    static bool finished = false;

    if (frame_size >= 0)
    {
    static bool alreadydone = false;
    if(!alreadydone)
    {

        x264_encoder_headers(encoder, &nals, &i_nals);
        alreadydone = true;
    }
    for(int i = 0; i < i_nals; ++i)
    {
        m_queue.push(nals[i]);
    }
    }
    delete [] surfaceData;
    surfaceData = nullptr;

    envir().taskScheduler().triggerEvent(eventTriggerId, this);
}

void H264FramedSource::doGetNextFrame()
{
    deliverFrame();
}

void H264FramedSource::deliverFrame0(void* clientData)
{
    ((H264FramedSource*)clientData)->deliverFrame();
}

void H264FramedSource::deliverFrame()
{
    x264_nal_t nalToDeliver;

    if (fPlayTimePerFrame > 0 && fPreferredFrameSize > 0) {
    if (fPresentationTime.tv_sec == 0 && fPresentationTime.tv_usec == 0) {
        // This is the first frame, so use the current time:
        gettimeofday(&fPresentationTime, NULL);
    } else {
        // Increment by the play time of the previous data:
        unsigned uSeconds   = fPresentationTime.tv_usec + fLastPlayTime;
        fPresentationTime.tv_sec += uSeconds/1000000;
        fPresentationTime.tv_usec = uSeconds%1000000;
    }

    // Remember the play time of this data:
    fLastPlayTime = (fPlayTimePerFrame*fFrameSize)/fPreferredFrameSize;
    fDurationInMicroseconds = fLastPlayTime;
    } else {
    // We don't know a specific play time duration for this data,
    // so just record the current time as being the 'presentation time':
    gettimeofday(&fPresentationTime, NULL);
    }

    if(!m_queue.empty())
    {
    m_queue.wait_and_pop(nalToDeliver);

    uint8_t* newFrameDataStart = (uint8_t*)0xD15EA5E;

    newFrameDataStart = (uint8_t*)(nalToDeliver.p_payload);
    unsigned newFrameSize = nalToDeliver.i_payload;

    // Deliver the data here:
    if (newFrameSize > fMaxSize) {
        fFrameSize = fMaxSize;
        fNumTruncatedBytes = newFrameSize - fMaxSize;
    }
    else {
        fFrameSize = newFrameSize;
    }

    memcpy(fTo, nalToDeliver.p_payload, nalToDeliver.i_payload);

    FramedSource::afterGetting(this);
    }
}

Relevant part of the RTSP-Server Therad

  RTSPServer* rtspServer = RTSPServer::createNew(*(parent->env), 8554, NULL);
  if (rtspServer == NULL) {
    *(parent->env) << "Failed to create RTSP server: " << (parent->env)->getResultMsg() << "\n";
    exit(1);
  }
  char const* streamName = "Stream";
  parent->h264FramedSource = H264FramedSource::createNew(*(parent->env), 0, 0);
  H264VideoStreamServerMediaSubsession *h264VideoStreamServerMediaSubsession = H264VideoStreamServerMediaSubsession::createNew(*(parent->env), true);
  h264VideoStreamServerMediaSubsession->parent = parent;
  sms->addSubsession(h264VideoStreamServerMediaSubsession);
  rtspServer->addServerMediaSession(sms);

  parent->env->taskScheduler().doEventLoop(); // does not return

Once a connection exists the render loop calls

h264FramedSource->AddToBuffer(videoData, 1024*768*4);
Community
  • 1
  • 1
user2660369
  • 245
  • 1
  • 4
  • 11

1 Answers1

15

First thing You have to do is write a wrapper around x264 Encoder which you can use to encode RGB data with nice given interface. the following class will give you the idea how to do that. i have used this class to encode RAW BGR frame which i was getting from my opencv capture.

x264Encoder.h

#ifdef __cplusplus
#define __STDINT_MACROS
#define __STDC_CONSTANT_MACROS
#endif
#include <iostream>
#include <concurrent_queue.h>
#include "opencv2\opencv.hpp"
#include <queue>
#include <stdint.h>
extern "C" {
#include "x264\x264.h"
}

class x264Encoder
{
public:
    x264Encoder(void);
    ~x264Encoder(void);

public:
    void initilize();
    void unInitilize();
    void encodeFrame(cv::Mat& image);
    bool isNalsAvailableInOutputQueue();
    x264_nal_t getNalUnit();
private:
    // Use this context to convert your BGR Image to YUV image since x264 do not support RGB input
    SwsContext* convertContext;
    std::queue<x264_nal_t> outputQueue;
    x264_param_t parameters;
    x264_picture_t picture_in,picture_out;
    x264_t* encoder;
};

x264Encoder.cpp

#include "x264Encoder.h"


x264Encoder::x264Encoder(void)
{

}


x264Encoder::~x264Encoder(void)
{

}

void x264Encoder::initilize()
{
    x264_param_default_preset(&parameters, "veryfast", "zerolatency");
    parameters.i_log_level = X264_LOG_INFO;
    parameters.i_threads = 1;
    parameters.i_width = 640;
    parameters.i_height = 480;
    parameters.i_fps_num = 25;
    parameters.i_fps_den = 1;
    parameters.i_keyint_max = 25;
    parameters.b_intra_refresh = 1;
    parameters.rc.i_rc_method = X264_RC_CRF;
    parameters.rc.i_vbv_buffer_size = 1000000;
    parameters.rc.i_vbv_max_bitrate = 90000;
    parameters.rc.f_rf_constant = 25;
    parameters.rc.f_rf_constant_max = 35;
    parameters.i_sps_id = 7;
    // the following two value you should keep 1
    parameters.b_repeat_headers = 1;    // to get header before every I-Frame
    parameters.b_annexb = 1; // put start code in front of nal. we will remove start code later
    x264_param_apply_profile(&parameters, "baseline");

    encoder = x264_encoder_open(&parameters);
    x264_picture_alloc(&picture_in, X264_CSP_I420, parameters.i_width, parameters.i_height);
    picture_in.i_type = X264_TYPE_AUTO;
    picture_in.img.i_csp = X264_CSP_I420;
    // i have initilized my color space converter for BGR24 to YUV420 because my opencv video capture gives BGR24 image. You can initilize according to your input pixelFormat
    convertContext = sws_getContext(parameters.i_width,parameters.i_height, PIX_FMT_BGR24, parameters.i_width,parameters.i_height,PIX_FMT_YUV420P, SWS_FAST_BILINEAR, NULL, NULL, NULL);
}

void x264Encoder::unInitilize()
{
    x264_encoder_close(encoder);
    sws_freeContext(convertContext);
}

void x264Encoder::encodeFrame(cv::Mat& image)
{
    int srcStride = parameters.i_width * 3;
    sws_scale(convertContext, &(image.data), &srcStride, 0, parameters.i_height, picture_in.img.plane, picture_in.img.i_stride);
    x264_nal_t* nals ;
    int i_nals = 0;
    int frameSize = -1;

    frameSize = x264_encoder_encode(encoder, &nals, &i_nals, &picture_in, &picture_out);
    if(frameSize > 0)
    {
        for(int i = 0; i< i_nals; i++)
        {
            outputQueue.push(nals[i]);
        }
    }
}

bool x264Encoder::isNalsAvailableInOutputQueue()
{
    if(outputQueue.empty() == true)
    {
        return false;
    }
    else
    {
        return true;
    }
}

x264_nal_t x264Encoder::getNalUnit()
{
    x264_nal_t nal;
    nal = outputQueue.front();
    outputQueue.pop();
    return nal;
}

Now we have the encoder which will take BGR picture and encode it. My encoder will encode the frame and put all the output nals into the output queue which will be streamed by Live555. To Implement live video source you have to create two class which will be subclass of (subclass of OnDemandServerMediaSubsession and another FramedSource). both are there is live555 media library.this class will serve data to more than one client also.
To create subclass of these two classes you can refer the following classes.

H264LiveServerMediaSession.h (Subclass of OnDemandServerMediaSubsession)

#include "liveMedia.hh"
#include "OnDemandServerMediaSubsession.hh"
#include "LiveSourceWithx264.h"

class H264LiveServerMediaSession:public OnDemandServerMediaSubsession
{
public:
    static H264LiveServerMediaSession* createNew(UsageEnvironment& env, bool reuseFirstSource);
    void checkForAuxSDPLine1();
    void afterPlayingDummy1();
protected:
    H264LiveServerMediaSession(UsageEnvironment& env, bool reuseFirstSource);
    virtual ~H264LiveServerMediaSession(void);
    void setDoneFlag() { fDoneFlag = ~0; }
protected:
    virtual char const* getAuxSDPLine(RTPSink* rtpSink, FramedSource* inputSource);
    virtual FramedSource* createNewStreamSource(unsigned clientSessionId, unsigned& estBitrate);
    virtual RTPSink* createNewRTPSink(Groupsock* rtpGroupsock, unsigned char rtpPayloadTypeIfDynamic, FramedSource* inputSource);
private:
    char* fAuxSDPLine;
    char fDoneFlag;
    RTPSink* fDummySink;
};  

H264LiveServerMediaSession.cpp

#include "H264LiveServerMediaSession.h"


H264LiveServerMediaSession* H264LiveServerMediaSession::createNew(UsageEnvironment& env, bool reuseFirstSource)
{
    return new H264LiveServerMediaSession(env, reuseFirstSource);
}

H264LiveServerMediaSession::H264LiveServerMediaSession(UsageEnvironment& env, bool reuseFirstSource):OnDemandServerMediaSubsession(env,reuseFirstSource),fAuxSDPLine(NULL), fDoneFlag(0), fDummySink(NULL)
{

}


H264LiveServerMediaSession::~H264LiveServerMediaSession(void)
{
    delete[] fAuxSDPLine;
}


static void afterPlayingDummy(void* clientData)
{
    H264LiveServerMediaSession *session = (H264LiveServerMediaSession*)clientData;
    session->afterPlayingDummy1();
}

void H264LiveServerMediaSession::afterPlayingDummy1()
{
    envir().taskScheduler().unscheduleDelayedTask(nextTask());
    setDoneFlag();
}

static void checkForAuxSDPLine(void* clientData)
{
    H264LiveServerMediaSession* session = (H264LiveServerMediaSession*)clientData;
    session->checkForAuxSDPLine1();
}

void H264LiveServerMediaSession::checkForAuxSDPLine1()
{
    char const* dasl;
    if(fAuxSDPLine != NULL)
    {
        setDoneFlag();
    }
    else if(fDummySink != NULL && (dasl = fDummySink->auxSDPLine()) != NULL)
    {
        fAuxSDPLine = strDup(dasl);
        fDummySink = NULL;
        setDoneFlag();
    }
    else
    {
        int uSecsDelay = 100000;
        nextTask() = envir().taskScheduler().scheduleDelayedTask(uSecsDelay, (TaskFunc*)checkForAuxSDPLine, this);
    }
}

char const* H264LiveServerMediaSession::getAuxSDPLine(RTPSink* rtpSink, FramedSource* inputSource)
{
    if(fAuxSDPLine != NULL) return fAuxSDPLine;
    if(fDummySink == NULL)
    {
        fDummySink = rtpSink;
        fDummySink->startPlaying(*inputSource, afterPlayingDummy, this);
        checkForAuxSDPLine(this);
    }

    envir().taskScheduler().doEventLoop(&fDoneFlag);
    return fAuxSDPLine;
}

FramedSource* H264LiveServerMediaSession::createNewStreamSource(unsigned clientSessionID, unsigned& estBitRate)
{
    // Based on encoder configuration i kept it 90000
    estBitRate = 90000;
    LiveSourceWithx264 *source = LiveSourceWithx264::createNew(envir());
    // are you trying to keep the reference of the source somewhere? you shouldn't.  
    // Live555 will create and delete this class object many times. if you store it somewhere  
    // you will get memory access violation. instead you should configure you source to always read from your data source
    return H264VideoStreamDiscreteFramer::createNew(envir(),source);
}

RTPSink* H264LiveServerMediaSession::createNewRTPSink(Groupsock* rtpGroupsock, unsigned char rtpPayloadTypeIfDynamic, FramedSource* inputSource)
{
    return H264VideoRTPSink::createNew(envir(), rtpGroupsock, rtpPayloadTypeIfDynamic);
}  

Now we have to subclass the FramedSource class which is there in LiveMedia. For Model you can refer to DeviceSource.cpp in live555 library. Following will show how i have done it.
LiveSourceWithx264.h

#include <queue>
#include "x264Encoder.h"
#include "opencv2\opencv.hpp"

class LiveSourceWithx264:public FramedSource
{
public:
    static LiveSourceWithx264* createNew(UsageEnvironment& env);
    static EventTriggerId eventTriggerId;
protected:
    LiveSourceWithx264(UsageEnvironment& env);
    virtual ~LiveSourceWithx264(void);
private:
    virtual void doGetNextFrame();
    static void deliverFrame0(void* clientData);
    void deliverFrame();
    void encodeNewFrame();
    static unsigned referenceCount;
    std::queue<x264_nal_t> nalQueue;
    timeval currentTime;
    // videoCaptureDevice is my BGR data source. You can have according to your need
    cv::VideoCapture videoCaptureDevice;
    cv::Mat rawImage;
    // Remember the x264 encoder wrapper we wrote in the start
    x264Encoder *encoder;
}; 

LiveSourceWithx264.cpp

#include "LiveSourceWithx264.h"


LiveSourceWithx264* LiveSourceWithx264::createNew(UsageEnvironment& env)
{
    return new LiveSourceWithx264(env);
}

EventTriggerId LiveSourceWithx264::eventTriggerId = 0;

unsigned LiveSourceWithx264::referenceCount = 0;

LiveSourceWithx264::LiveSourceWithx264(UsageEnvironment& env):FramedSource(env)
{
    if(referenceCount == 0)
    {

    }
    ++referenceCount;
    videoCaptureDevice.open(0);
    encoder = new x264Encoder();
    encoder->initilize();
    if(eventTriggerId == 0)
    {
        eventTriggerId = envir().taskScheduler().createEventTrigger(deliverFrame0);
    }
}


LiveSourceWithx264::~LiveSourceWithx264(void)
{
    --referenceCount;
    videoCaptureDevice.release();
    encoder->unInitilize();
    envir().taskScheduler().deleteEventTrigger(eventTriggerId);
    eventTriggerId = 0;
}

void LiveSourceWithx264::encodeNewFrame()
{
        rawImage.data = NULL;
        while(rawImage.data == NULL)
        {
            videoCaptureDevice >> rawImage;
            cv::waitKey(100);
        }
        // Got new image to stream
        assert(rawImage.data != NULL);
        encoder->encodeFrame(rawImage);
        // Take all nals from encoder output queue to our input queue
        while(encoder->isNalsAvailableInOutputQueue() == true)
        {
            x264_nal_t nal = encoder->getNalUnit();
            nalQueue.push(nal);
        }
}

void LiveSourceWithx264::deliverFrame0(void* clientData)
{
    ((LiveSourceWithx264*)clientData)->deliverFrame();
}

void LiveSourceWithx264::doGetNextFrame()
{
    if(nalQueue.empty() == true)
    {
        encodeNewFrame();
        gettimeofday(&currentTime,NULL);
        deliverFrame();
    }
    else
    {
        deliverFrame();
    }
}

void LiveSourceWithx264::deliverFrame()
{
    if(!isCurrentlyAwaitingData()) return;
    x264_nal_t nal = nalQueue.front();
    nalQueue.pop();
    assert(nal.p_payload != NULL);
    // You need to remove the start code which is there in front of every nal unit.  
    // the start code might be 0x00000001 or 0x000001. so detect it and remove it. pass remaining data to live555    
    int trancate = 0;
    if (nal.i_payload >= 4 && nal.p_payload[0] == 0 && nal.p_payload[1] == 0 && nal.p_payload[2] == 0 && nal.p_payload[3] == 1 )
    {
        trancate = 4;
    }
    else
    {
        if(nal.i_payload >= 3 && nal.p_payload[0] == 0 && nal.p_payload[1] == 0 && nal.p_payload[2] == 1 )
        {
            trancate = 3;
        }
    }

    if(nal.i_payload-trancate > fMaxSize)
    {
        fFrameSize = fMaxSize;
        fNumTruncatedBytes = nal.i_payload-trancate - fMaxSize;
    }
    else
    {
        fFrameSize = nal.i_payload-trancate;
    }
    fPresentationTime = currentTime;
    memmove(fTo,nal.p_payload+trancate,fFrameSize);
    FramedSource::afterGetting(this);
}  

Now we are done with the classes implementation. Now to do the streaming setup you can follow same as testOnDemandRTSPServer.cpp sample. Here is my main where i did the setup

#include <iostream>
#include <liveMedia.hh>
#include <BasicUsageEnvironment.hh>
#include <GroupsockHelper.hh>
#include "H264LiveServerMediaSession.h"
#include "opencv2\opencv.hpp"
#include "x264Encoder.h" 
int main(int argc, char* argv[])
{
    TaskScheduler* taskSchedular = BasicTaskScheduler::createNew();
    BasicUsageEnvironment* usageEnvironment = BasicUsageEnvironment::createNew(*taskSchedular);
    RTSPServer* rtspServer = RTSPServer::createNew(*usageEnvironment, 8554, NULL);
    if(rtspServer == NULL)
    {
        *usageEnvironment << "Failed to create rtsp server ::" << usageEnvironment->getResultMsg() <<"\n";
        exit(1);
    }

    std::string streamName = "usb1";
    ServerMediaSession* sms = ServerMediaSession::createNew(*usageEnvironment, streamName.c_str(), streamName.c_str(), "Live H264 Stream");
    H264LiveServerMediaSession *liveSubSession = H264LiveServerMediaSession::createNew(*usageEnvironment, true);
    sms->addSubsession(liveSubSession);
    rtspServer->addServerMediaSession(sms);
    char* url = rtspServer->rtspURL(sms);
    *usageEnvironment << "Play the stream using url "<<url << "\n";
    delete[] url;
    taskSchedular->doEventLoop();
    return 0;
}

and you have the URL for Your LiveSource. I had for my USB Cam :)

bhawesh
  • 1,310
  • 2
  • 19
  • 57
  • Awesome work on putting together a whole example. I've been struggling with this exact same problem for some days now and having something to look at definitely helps. However, the code needed some editing to actually compile (headers, inclusion guards etc) and it does not really work for me. As soon as I direct an instance of VLC to the printed URL I see the camera get initialized and some X264 debugging in the console. I also see one single garbled image being received by VLC, but after that nothing happens anymore until I stop VLC. OpenCV camera grabbing works fine for me in my own app. – Jan Ekholm Apr 02 '14 at 08:46
  • Also, why do you set "parameters.b_annexb = 1;" and then later remove the added header? Why not simply set it to 0? – Jan Ekholm Apr 02 '14 at 08:51
  • putting parameters.b_annexb = 0 will only replace the start code with length of the payload which will be 4 byte. so it doesn't matter what you have set. later you have to truncate 4 bytes before passing to Live555..... also you can optimize the code in your own way. this is just example to show the use of Live555. i cannot post my production code here ...:) though this sample is working for me without any problem – bhawesh Apr 02 '14 at 09:05
  • Thanks for the clarification. Your example streams a single garbled image for me, but perhaps it's something related to OSX. Live555 seems to be ridiculously complicated to get to work and uses a lot of questionable C++ practices. :) There are a fair deal of moving parts here to get working at the same time. – Jan Ekholm Apr 02 '14 at 12:49
  • Adding "parameters.b_annexb = 1" and stripping of the headers gives me a single image in VLC in my own application. It's not garbled, or only has some minor error in one edge. But there is only one image, I see nothing else changing. Perhaps the stream gets stuck streaming into the dummy sink or something like that... – Jan Ekholm Apr 02 '14 at 13:24
  • are you sure your openCV capture is working correct because i had the same problem and finally it turns out that my streaming and encoding was fine but the problem was in opencv capture where after first capture opencv was not capturing any but giving the same image again and again – bhawesh Apr 02 '14 at 13:30
  • I think it's fine, as when I did a simple OpenCV only test which just showed the grabbed image in an OpenCV window it was perfectly fine. I also have the X264 decoder on debug level, and the values it shows differ for every frame. The encoded frame size also differs, it's quite small when nothing happens and jumps if I do any sudden movements in front of the camera. VLC also complains about images being late. The openRTSP example app seems to save a file "video-H264-1: JVT NAL sequence, H.264 video, baseline @ L 52". – Jan Ekholm Apr 02 '14 at 13:52
  • Hm, I tried adding the OpenCV window thing into my application, and it also shows the same image all the time. Not sure if the window simply does not work in my context or if the frame indeed is the same all the time. Did you mean that opening the camera for the second time gave the same image all the time? What did you do in your case to solve it? – Jan Ekholm Apr 02 '14 at 14:15
  • its hard to say about your problem without looking at your code. You should post your code with your question. – bhawesh Apr 02 '14 at 16:21
  • Of course. I was mostly curious as to what caused your VideoCapture to not update anymore. However, if I do a cv::namedWindow() and show each frame there, the frame shown in that window is different from the single one that gets streamed, so I think the capture is working as it should. If I can't figure it out I'll have to strip it down to a minimal case and post it. Thank you for the help, I appreciate it very much. – Jan Ekholm Apr 03 '14 at 06:28
  • Ok, after trying openRTSP it seems to save a stream just fine, and after renaming the file to something.h264 VLC can open it and play it back. The speed is way too fast, so I'm now guessing my presentation time or frame rate is somehow broken. But it does mean that the ancoding and streaming does indeed work, there's just a bug somewhere. :) – Jan Ekholm Apr 03 '14 at 09:22
  • video playback is expected to be fast because .h264 is not a container and you cannot add pts time in that file. for right playback you have to put the stream into a container say .mp4 or .mkv with pts time. – bhawesh Apr 03 '14 at 09:25
  • I guessed as much. But the video is there at least. By increasing the network caching in VLC I manage to get a choppy video running, but it's 1s or so delayed. Ideally I need much lower than that. But it seems the streaming is actually working now, more or less with the code that you had. Your standalone code also works with a larger VLC buffer, but it's also 1s+ delayed due to the buffering. So, it's about trying to lower the latency now. – Jan Ekholm Apr 03 '14 at 10:31
  • Thanks for the code, I'm struggling with sending H264 frames for a while now. However, your code does not solve my issue: with both my and your code, I'm experiencing frames being truncated due to small buffer size. fMaxSize seems to be changing a lot - how does live555 do this, just guess? there are frames every other time, that have to be truncated, as a result, the rendering on client side is missing data and serious artifacts show up. How do you deal with this? – iko79 Jan 12 '17 at 11:33
  • @bhawesh I know this is a bit of an older question but your solution does not seem to be unicast. I can still access the stream from multiple devices. Could you please clarify how to convert this code to unicast? – ktb92677 Mar 18 '19 at 23:00