视频编解码,FFMPEG,RTP问题汇总

分手后的思念是犯贱 2022-06-13 11:53 564阅读 0赞

http://blog.csdn.net/smilestone\_322/article/details/8529564

1）ffmpeg如何判断一帧数据是正确的？ffmpeg有没有错误处理的模式，能使花屏的帧(h264格式的)不显示出来？

2) H264网络传输过程中丢包造成马赛克问题?

原因：

1. 接收网络数据包后没有调整包的顺序，譬如说接受包的顺序是1，3，4，2，如果没有调整顺序的话，发送给解码器的顺序也是1，3，4，2，这样肯定会出现马赛克 ；  
2. 接收网络数据包后没有没有合并数据包，众所周知，一个Video帧可能被分割成多个网络数据包传送，譬如说是1，2，3，如果在接受端没有将这三个包合并成一个Video帧发送给解码器，而是当成三个Video帧发送给解码器，也肯定会出现马赛克 ；  
3. 没有正确处理好网络丢包的情况，如果所丢的数据包正好传送的是另外一个帧所参考的的数据，这样也会出现马赛克 ；  
4. 解码器有问题，如果播放本地文件也出现马赛克的话。

解决方法：

1.服务器软件用多线程：   
（1）主线程：读出（看你的图象具体怎么上PC机了）一帧视频数据，送给拆分线程。   
（2）拆分线程：接到一帧视频，开始拆包、做帧标记、打序列号，送给发送线程。   
（3）发送线程：用RTP socket把封装好的数据包发给客户端。此socket是点对多点、单向 有根方式的组播套接字，实际上是基于UDP派生的，但他用到了RTP和RTCP（实时传输 协议和实时传输控制协议），如果你传输的不是实时数据，直接用UDP就行了。  
  
2.客户端软件结构一般用多线程，线程间用事件对象进行同步，而共享数据区用临界区对象进   
行同步。   
（1）主线程：接收网络数据包的线程，优先级最高，可以尽量保证不丢数据，也采用RTP协 议,用网络事件来触发。主线程接收到视频数据包后，将数据放入一个链表中，然后用事件对象触发组装线程。  
（2）组装线程：从链表中读出数据包，要进行帧确认、排序等工作，当把一帧图象的所有 包都取到时，再调用组装模块（可以是一个函数），将这些数据包组装成完整的一个帧，然后送到解压线程。  
（3）若干解压播放线程。主要考虑到如果你客户端软件想同时播放多画面，比如说4画面图 象，就要用4个解压播放线程。   
（4）至于图象存储，要看你的客户需要怎么存了，如果是手工存当然不需要单开线程，如果 是规定定时存或在某个事件发生时自动存盘，就需要单开一个线程，由定时器到时消息或此事件发生来触发。  
  
后来我们项目也将图象送上了广域网和Internet，主要就是传输的速率明显慢了。把服务器软件放在Web服务器上，用UDP点对点的方式给提出视频服务的Internet客户端发出相应视频帧。还可以实现对每个客户端的各种服务请求的响应和控制。

建议：

（1）Winsock传输部分最好不要用MFC提供的类，自己用API做，但是应用 程序可以用MFC做，   
（2）在局域网上用点对多点的组播方式传输时，基于RTP和RTCP协议来做。 服务器方得拆包传，每包最好不要大于2K，以我去年做的项目经验， 用1.5K比较好，每包间隔1-2毫秒。发送方给每包打上时间戳和序列 号，接收方重新排序和组装。  
（3）如果是点对点传，应该基于UDP协议，可以一帧一帧的传，我最大传过 30K。没问题，我已经在实际项目中试过，JPEG格式的视频流在局域网 中1秒15帧，一帧12K；在广域网中1秒中4帧，一帧15K。  
（4） 如果你传输的是监控的告警数据，要求准确性，你必须基于TCP协议，用点对点方式传输。

VLC 对应的解决策略：

static inline uint16\_t rtp\_seq (const block\_t \*block)  
\{  
    assert (block->i\_buffer >= 4);  
    return GetWBE (block->p\_buffer + 2);  
\}

\#define GetWBE( p )     U16\_AT( p )

/\* MSB (big endian)/LSB (little endian) conversions - network order is always  
 \* MSB, and should be used for both network communications and files. \*/  
LIBVLC\_USED  
static inline uint16\_t U16\_AT( const void \* \_p )  
\{  
    const uint8\_t \* p = (const uint8\_t \*)\_p;  
    return ( ((uint16\_t)p\[0\] << 8) | p\[1\] );  
\}

/\*\* State for a RTP session: \*/  
struct rtp\_session\_t  
\{  
    rtp\_source\_t \*\*srcv;  
    unsigned       srcc;  
    uint8\_t        ptc;  
    rtp\_pt\_t      \*ptv;  
\};

/\*\* State for an RTP source \*/  
struct rtp\_source\_t  
\{  
    uint32\_t ssrc;  
    uint32\_t jitter;  /\* interarrival delay jitter estimate \*/  
    mtime\_t  last\_rx; /\* last received packet local timestamp \*/  
    uint32\_t last\_ts; /\* last received packet RTP timestamp \*/

uint32\_t ref\_rtp; /\* sender RTP timestamp reference \*/  
    mtime\_t  ref\_ntp; /\* sender NTP timestamp reference \*/

uint16\_t bad\_seq; /\* tentatively next expected sequence for resync \*/  
    uint16\_t max\_seq; /\* next expected sequence \*/

uint16\_t last\_seq; /\* sequence of the next dequeued packet \*/  
    block\_t \*blocks; /\* re-ordered blocks queue \*/  
    void    \*opaque\[0\]; /\* Per-source private payload data \*/  
\};

/\*\*  
 \* Destroys an RTP source and its associated streams.  
 \*/  
static void  
rtp\_source\_destroy (demux\_t \*demux, const rtp\_session\_t \*session,  
                    rtp\_source\_t \*source)  
\{  
    msg\_Dbg (demux, "removing RTP source (%08x)", source->ssrc);

for (unsigned i = 0; i < session->ptc; i++)  
        session->ptv\[i\].destroy (demux, source->opaque\[i\]);  
    block\_ChainRelease (source->blocks);  
    free (source);  
\}

/\*\*  
 \* Receives an RTP packet and queues it. Not a cancellation point.  
 \*  
 \* @param demux VLC demux object  
 \* @param session RTP session receiving the packet  
 \* @param block RTP packet including the RTP header  
 \*/  
void  
rtp\_queue (demux\_t \*demux, rtp\_session\_t \*session, block\_t \*block)  
\{  
    demux\_sys\_t \*p\_sys = demux->p\_sys;

/\* RTP header sanity checks (see RFC 3550) \*/  
    if (block->i\_buffer < 12)   //如果RTP包的长度小于12，说明包传输有错误  
        goto drop;  
    if ((block->p\_buffer\[0\] >> 6 ) != 2) /\* RTP version number(rtp版本号必须为2) \*/  
        goto drop;

/\* Remove padding if present (判断RTP数据包是否有填充字节，如果有填充字节，解析数据包时，必须将填充字节去掉)\*/  
    if (block->p\_buffer\[0\] & 0x20)  
    \{  
        uint8\_t padding = block->p\_buffer\[block->i\_buffer - 1\];  
        if ((padding == 0) || (block->i\_buffer < (12u + padding)))  
            goto drop; /\* illegal value \*/

block->i\_buffer -= padding;  
    \}

mtime\_t        now = mdate ();   //获取精确的时钟信息  
    rtp\_source\_t  \*src  = NULL;  
    const uint16\_t seq  = rtp\_seq (block); //获取序列号  
    const uint32\_t ssrc = GetDWBE (block->p\_buffer + 8); //获取SSRC

/\* In most case, we know this source already 找到相同的SSRC\*/  
    for (unsigned i = 0, max = session->srcc; i < max; i++)  
    \{  
        rtp\_source\_t \*tmp = session->srcv\[i\];  
        if (tmp->ssrc == ssrc)  
        \{  
            src = tmp;  
            break;  
        \}

/\* RTP source garbage collection \*/  
        if ((tmp->last\_rx + p\_sys->timeout) < now)    //超时了  
        \{  
            rtp\_source\_destroy (demux, session, tmp);  
            if (--session->srcc > 0)  
                session->srcv\[i\] = session->srcv\[session->srcc - 1\]; //将最后一个赋值给删除的那个ssrc  
        \}  
    \}

if (src == NULL)  //在原来的会话中没有找到同样的ssrc，说明是新来的ssrc  
    \{  
        /\* New source \*/  
        if (session->srcc >= p\_sys->max\_src)  //判断是不是到达了最大的ssrc数的极限值(max)  
        \{  
            msg\_Warn (demux, "too many RTP sessions");  
            goto drop;  
        \}

rtp\_source\_t \*\*tab;  
        tab = realloc (session->srcv, (session->srcc + 1) \* sizeof (\*tab));  
        if (tab == NULL)  
            goto drop;  
        session->srcv = tab;

src = rtp\_source\_create (demux, session, ssrc, seq); //创建ssrc  
        if (src == NULL)  
            goto drop;

tab\[session->srcc++\] = src;  
        /\* Cannot compute jitter yet \*/  
    \}  
    else  
    \{  
        const rtp\_pt\_t \*pt = rtp\_find\_ptype (session, src, block, NULL);

if (pt != NULL)  
        \{  
            /\* Recompute jitter estimate.  
             \* That is computed from the RTP timestamps and the system clock.  
             \* It is independent of RTP sequence. \*/  
            uint32\_t freq = pt->frequency;  
            int64\_t ts = rtp\_timestamp (block);  
            int64\_t d = ((now - src->last\_rx) \* freq) / CLOCK\_FREQ;  
            d        -=    ts - src->last\_ts;  
            if (d < 0) d = -d;  
            src->jitter += ((d - src->jitter) + 8) >> 4;  
        \}  
    \}  
    src->last\_rx = now;  
    block->i\_pts = now; /\* store reception time until dequeued \*/  
    src->last\_ts = rtp\_timestamp (block);

/\* Check sequence number \*/  
    /\* NOTE: the sequence number is per-source,  
     \* but is independent from the payload type. \*/  
    int16\_t delta\_seq = seq - src->max\_seq;  
    if ((delta\_seq > 0) ? (delta\_seq > p\_sys->max\_dropout)  
                        : (-delta\_seq > p\_sys->max\_misorder))  
    \{  
        msg\_Dbg (demux, "sequence discontinuity"  
                 " (got: %"PRIu16", expected: %"PRIu16")", seq, src->max\_seq);  
        if (seq == src->bad\_seq)  
        \{  
            src->max\_seq = src->bad\_seq = seq + 1;  
            src->last\_seq = seq - 0x7fffe; /\* hack for rtp\_decode() \*/  
            msg\_Warn (demux, "sequence resynchronized");  
            block\_ChainRelease (src->blocks);  
            src->blocks = NULL;  
        \}  
        else  
        \{  
            src->bad\_seq = seq + 1;  
            goto drop;  
        \}  
    \}  
    else  
    if (delta\_seq >= 0)  
        src->max\_seq = seq + 1;

/\* Queues the block in sequence order,  
     \* hence there is a single queue for all payload types. \*/  
    block\_t \*\*pp = &src->blocks;  
    for (block\_t \*prev = \*pp; prev != NULL; prev = \*pp)  //将接收到的数据插入到队列中  
    \{  
        int16\_t delta\_seq = seq - rtp\_seq (prev);  
        if (delta\_seq < 0)  
            break;  
        if (delta\_seq == 0)  
        \{  
            msg\_Dbg (demux, "duplicate packet (sequence: %"PRIu16")", seq);  
            goto drop; /\* duplicate \*/  
        \}  
        pp = &prev->p\_next;  
    \}  
    block->p\_next = \*pp;  
    \*pp = block;

/\*rtp\_decode (demux, session, src);\*/  
    return;

drop:  
    block\_Release (block);  
\}

vlc调用过程：

/\*\*  
 \* @file input.c  
 \* @brief RTP packet input  
 \*/  
/\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*  
 \* Copyright 漏 2008 R茅mi Denis-Courmont  
 \*  
 \* This library is free software; you can redistribute it and/or  
 \* modify it under the terms of the GNU Lesser General Public License  
 \* as published by the Free Software Foundation; either version 2.1  
 \* of the License, or (at your option) any later version.  
 \*  
 \* This library is distributed in the hope that it will be useful,  
 \* but WITHOUT ANY WARRANTY; without even the implied warranty of  
 \* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the  
 \* GNU General Public License for more details.  
 \*  
 \* You should have received a copy of the GNU Lesser General Public  
 \* License along with this library; if not, write to the Free Software  
 \* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA  
 \*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*/

\#ifdef HAVE\_CONFIG\_H  
\# include <config.h>  
\#endif

\#include <vlc\_common.h>  
\#include <vlc\_demux.h>  
\#include <vlc\_block.h>  
\#include <vlc\_network.h>

\#include <unistd.h>  
\#ifdef HAVE\_POLL  
\# include <poll.h>  
\#endif

\#include "rtp.h"  
\#ifdef HAVE\_SRTP  
\# include <srtp.h>  
\#endif

static bool fd\_dead (int fd)  
\{  
    struct pollfd ufd = \{ .fd = fd, \};  
    return (poll (&ufd, 1, 0) > 0) && (ufd.revents & POLLHUP);  
\}

/\*\*  
 \* Gets a datagram from the network.  
 \* @param fd datagram file descriptor  
 \* @return a block or NULL on fatal error (socket dead)  
 \*/  
static block\_t \*rtp\_dgram\_recv (vlc\_object\_t \*obj, int fd)  
\{  
    block\_t \*block = block\_Alloc (0xffff);  
    ssize\_t len;

block\_cleanup\_push (block);  
    do  
    \{  
        len = net\_Read (obj, fd, NULL,  
                        block->p\_buffer, block->i\_buffer, false);

if (((len <= 0) && fd\_dead (fd)) || !vlc\_object\_alive (obj))  
        \{   /\* POLLHUP -> permanent (DCCP) socket error \*/  
            block\_Release (block);  
            block = NULL;  
            break;  
        \}  
    \}  
    while (len == -1);  
    vlc\_cleanup\_pop ();

return block ? block\_Realloc (block, 0, len) : NULL;  
\}

/\*\*  
 \* Gets a framed RTP packet.  
 \* @param fd stream file descriptor  
 \* @return a block or NULL in case of fatal error  
 \*/  
static block\_t \*rtp\_stream\_recv (vlc\_object\_t \*obj, int fd)  
\{  
    ssize\_t len = 0;  
    uint8\_t hdr\[2\]; /\* frame header \*/

/\* Receives the RTP frame header \*/  
    do  
    \{  
        ssize\_t val = net\_Read (obj, fd, NULL, hdr + len, 2 - len, false);  
        if (val <= 0)  
            return NULL;  
        len += val;  
    \}  
    while (len < 2);

block\_t \*block = block\_Alloc (GetWBE (hdr));

/\* Receives the RTP packet \*/  
    for (ssize\_t i = 0; i < len;)  
    \{  
        ssize\_t val;

block\_cleanup\_push (block);  
        val = net\_Read (obj, fd, NULL,  
                        block->p\_buffer + i, block->i\_buffer - i, false);  
        vlc\_cleanup\_pop ();

if (val <= 0)  
        \{  
            block\_Release (block);  
            return NULL;  
        \}  
        i += val;  
    \}

return block;  
\}

static block\_t \*rtp\_recv (demux\_t \*demux)  
\{  
    demux\_sys\_t \*p\_sys = demux->p\_sys;

for (block\_t \*block;; block\_Release (block))  
    \{  
        block = p\_sys->framed\_rtp  
                ? rtp\_stream\_recv (VLC\_OBJECT (demux), p\_sys->fd)  
                : rtp\_dgram\_recv (VLC\_OBJECT (demux), p\_sys->fd);  
        if (block == NULL)  
        \{  
            msg\_Err (demux, "RTP flow stopped");  
            break; /\* fatal error \*/  
        \}

if (block->i\_buffer < 2)  
            continue;

/\* FIXME \*/  
        const uint8\_t ptype = rtp\_ptype (block);  
        if (ptype >= 72 && ptype <= 76)  
            continue; /\* Muxed RTCP, ignore for now \*/  
\#ifdef HAVE\_SRTP  
        if (p\_sys->srtp)  
        \{  
            size\_t len = block->i\_buffer;  
            int canc, err;

canc = vlc\_savecancel ();  
            err = srtp\_recv (p\_sys->srtp, block->p\_buffer, &len);  
            vlc\_restorecancel (canc);  
            if (err)  
            \{  
                msg\_Dbg (demux, "SRTP authentication/decryption failed");  
                continue;  
            \}  
            block->i\_buffer = len;  
        \}  
\#endif  
        return block; /\* success! \*/  
    \}  
    return NULL;  
\}

static void timer\_cleanup (void \*timer)  
\{  
    vlc\_timer\_destroy ((vlc\_timer\_t)timer);  
\}

static void rtp\_process (void \*data);

void \*rtp\_thread (void \*data)  
\{  
    demux\_t \*demux = data;  
    demux\_sys\_t \*p\_sys = demux->p\_sys;  
    bool autodetect = true;

if (vlc\_timer\_create (&p\_sys->timer, rtp\_process, data))  
        return NULL;  
    vlc\_cleanup\_push (timer\_cleanup, (void \*)p\_sys->timer);

for (;;)  
    \{  
        block\_t \*block = rtp\_recv (demux);  
        if (block == NULL)  
            break;

if (autodetect)  
        \{   /\* Autodetect payload type, \_before\_ rtp\_queue() \*/  
            /\* No need for lock - the queue is empty. \*/  
            if (rtp\_autodetect (demux, p\_sys->session, block))  
            \{  
                block\_Release (block);  
                continue;  
            \}  
            autodetect = false;  
        \}

int canc = vlc\_savecancel ();  
        vlc\_mutex\_lock (&p\_sys->lock);  
        rtp\_queue (demux, p\_sys->session, block);  
        vlc\_mutex\_unlock (&p\_sys->lock);  
        vlc\_restorecancel (canc);

rtp\_process (demux);  
    \}  
    vlc\_cleanup\_run ();  
    return NULL;  
\}

/\*\*  
 \* Process one RTP packet from the de-jitter queue.  
 \*/  
static void rtp\_process (void \*data)  
\{  
    demux\_t \*demux = data;  
    demux\_sys\_t \*p\_sys = demux->p\_sys;  
    mtime\_t deadline;

vlc\_mutex\_lock (&p\_sys->lock);  
    if (rtp\_dequeue (demux, p\_sys->session, &deadline))  
        vlc\_timer\_schedule (p\_sys->timer, true, deadline, 0);  
    vlc\_mutex\_unlock (&p\_sys->lock);  
\}

顶

踩

*  上一篇[C++面试题][C]
 *  下一篇[多线程参数传递问题][Link 1]
 *

[C]: http://blog.csdn.net/smilestone322/article/details/8474820
[Link 1]: http://blog.csdn.net/smilestone322/article/details/8554080