libVLC 提取视频帧使用OpenGL渲染

在上一节中，我们讲解了如何使用QWidget渲染每一帧视频数据。

由于我们不停的生成的是QImage对象，因此对 CPU 负荷较高。其实在绘制这块我们可以使用 OpenGL去绘制，利用 GPU 减轻 CPU 计算负荷，本节讲解使用OpenGL来绘制每一帧视频数据。

libVLC 提取视频帧使用QWidget渲染-CSDN博客

以下是操作流程：

1.初始化 libVLC 实例。

vlc_base = libvlc_new(0, NULL);

2.创建一个媒体播放器。

    vlc_media = libvlc_media_new_path(vlc_base, filename.toUtf8().data());
    if (!vlc_media) {
        return;
    }
 
    // 创建libvlc实例和媒体播放器
    vlc_mediaPlayer = libvlc_media_player_new_from_media(vlc_media);
    if (!vlc_mediaPlayer) {
        return;
    }

3.设置视频回调。

    libvlc_video_set_format_callbacks(vlc_mediaPlayer, setup, NULL);
    
    // 设置自定义视频输出
    libvlc_video_set_callbacks(vlc_mediaPlayer, lock, unlock, display, NULL);

4.提取视频帧数据，回调给OpenGL显示。

static void unlock(void *opaque, void *picture, void *const *planes) {
	// 这里可以释放视频帧的锁

	uint8_t *buffer = (uint8_t *)*planes; //planes即为帧数据
	//QImage image((unsigned char*)buffer, g_frame->width, g_frame->height, QImage::Format_ARGB32);
	m_this->m_videoFunc(buffer);

	g_frame->mutex.unlock();
}

声明了两个OpenGL接口回调。

//视频每一帧数据回调
typedef std::function<void(uint8_t*)> VideoDataFunc;
//视频宽、高。回调
typedef std::function<void(int, int)> VideoInfoFunc;

设置回调。

	//视频数据
	m_videoFunc = std::bind(&WOpenGLWidget::slotReceiveVideoData, ui.openGLWidget,
		std::placeholders::_1);

	//视频信息
	m_videoInfoFunc = std::bind(&WOpenGLWidget::slotOpenVideo, ui.openGLWidget,
		std::placeholders::_1, std::placeholders::_2);

使用回调。

static void unlock(void *opaque, void *picture, void *const *planes) {
	// 这里可以释放视频帧的锁

	uint8_t *buffer = (uint8_t *)*planes; //planes即为帧数据
	//QImage image((unsigned char*)buffer, g_frame->width, g_frame->height, QImage::Format_ARGB32);
	m_this->m_videoFunc(buffer);

	g_frame->mutex.unlock();
}


static unsigned setup(void **opaque, char *chroma,
	unsigned *width, unsigned *height,
	unsigned *pitches,
	unsigned *lines)
{
	qDebug() << "chroma:" << QString(chroma) << "width:" << *width << ", height:" << *height;

	/* 开辟存放图像数据的内存块 */
	if (g_frame)
	{
		if (g_frame->pixels)
		{
			delete[] g_frame->pixels;
			g_frame->pixels = NULL;
		}

		delete g_frame;
		g_frame = NULL;
	}

	int w = *width;
	int h = *height;
	g_frame = new Frame;
	g_frame->pixels = new uchar[w * h * 4]; // 申请大小也为4通道的像素

	memset(g_frame->pixels, 0, w * h * 4);
	memcpy(chroma, "RV32", 4);
	g_frame->width = w;
	g_frame->height = h;
	*pitches = w * 4;
	*lines = h;
	m_this->m_videoInfoFunc(w, h);
	return 1;
}

 opengl完全没有基础的同学，请先学习以下的几篇文章，我们只需要了解2D图像如何渲染就行。

1.OpenGL简介

2.OpenGL实现第一个窗口-三角形

3.OpenGL 纹理

首先继承QOpenGLWidget类，重写paintGL()、resizeGL()、initializeGL()方法。

本示例演示使用opengles2来渲染，渲染rgba的数据。

以下是封装好的WOpenGLWidget类，使用提升的方式，提升为以下这个类就行了。

#ifndef WOPENGLWIDGET_H
#define WOPENGLWIDGET_H

#include <QOpenGLWidget>
#include <QOpenGLFunctions>
#include <QOpenGLTexture>
#include <QImage>
#include <QOpenGLShaderProgram>
#include <QOpenGLShader>
#include <QDebug>
#include <QImage>
#include <QMouseEvent>
#include <QSet>
#include <QPainter>
#include <QMutex>
#include <QOpenGLBuffer>

class OpenGLDisplayImpl
{
public:
	OpenGLDisplayImpl()
	{
		texture = NULL;
		videoW = 0;
		videoH = 0;
	}

	unsigned char *buffer = {0};

	QOpenGLTexture*         texture;

	GLsizei                 videoW, videoH;
};

class WOpenGLWidget : public QOpenGLWidget,public QOpenGLFunctions
{
    Q_OBJECT
public:
    WOpenGLWidget(QWidget* parent = Q_NULLPTR);
	~WOpenGLWidget();

public:
	void slotOpenVideo(int width,int height);
	void slotReceiveVideoData(uint8_t* buffer);
	void clear();
	void deleteBuffer();

protected:
    virtual void initializeGL();
    virtual void paintGL();
    virtual void resizeGL(int w, int h);

private:
    QOpenGLShaderProgram *m_program = nullptr;          //着色器程序

	QOpenGLBuffer VBO, EBO;

	OpenGLDisplayImpl *m_impl = nullptr;
	bool m_isShowVideo = false;
	QMutex m_mux;
};

#endif // WOPENGLWIDGET_H


#include "WOpenGLWidget.h"
#include <QDebug>
#include <QTimer>
#include <QElapsedTimer>

static float vertices[] = {
//     ---- 位置 ----    - 纹理坐标 -
     1.0f,  1.0f, 0.0f, 1.0f, 1.0f,   // 右上
     1.0f, -1.0f, 0.0f, 1.0f, 0.0f,   // 右下
    -1.0f, -1.0f, 0.0f, 0.0f, 0.0f,   // 左下
    -1.0f,  1.0f, 0.0f, 0.0f, 1.0f    // 左上
};

static unsigned int indices[] = {
    0, 1, 3,
    1, 2, 3
};


WOpenGLWidget::WOpenGLWidget(QWidget* parent)
    : QOpenGLWidget(parent)
	, m_impl(new OpenGLDisplayImpl)
	, EBO(QOpenGLBuffer::IndexBuffer)
{
}

WOpenGLWidget::~WOpenGLWidget()
{
	clear();

	if (m_impl->texture)
	{
		m_impl->texture->destroy();
	}

	delete m_impl;
	m_impl = nullptr;
}

void WOpenGLWidget::slotOpenVideo(int width, int height)
{
	qDebug() << "slotOpenVideo";
	m_mux.lock();
	glClearColor(0.0f, 0.0f, 0.0f, 0.0f);
	glClear(GL_COLOR_BUFFER_BIT);
	m_isShowVideo = false;
	m_impl->videoW = width;
	m_impl->videoH = height;

	deleteBuffer();

	resize(this->width(), this->height());

	m_isShowVideo = true;
	m_mux.unlock();
}

void WOpenGLWidget::slotReceiveVideoData(uint8_t* yuvBuffer)
{
	if (!m_impl)
		return;

	m_mux.lock();
	if(!m_impl->buffer)
		m_impl->buffer = new unsigned char[m_impl->videoW * m_impl->videoH * 4];//y

	memcpy(m_impl->buffer, yuvBuffer, m_impl->videoW * m_impl->videoH * 4);

	update();
    m_mux.unlock();
}


void WOpenGLWidget::clear()
{
	m_mux.lock();

	deleteBuffer();
	m_isShowVideo = false;
	m_mux.unlock();
}

void WOpenGLWidget::deleteBuffer()
{
	if (m_impl)
	{
		if (m_impl->buffer) {
			delete m_impl->buffer;
			m_impl->buffer= nullptr;
		}
	}
}

void WOpenGLWidget::initializeGL()
{
	m_mux.lock();
    initializeOpenGLFunctions();

    m_program = new QOpenGLShaderProgram();
    m_program->addShaderFromSourceFile(QOpenGLShader::Vertex,":/shaders/shaders/shapes.vert");
    m_program->addShaderFromSourceFile(QOpenGLShader::Fragment,":/shaders/shaders/shapes.frag");
	bool success = m_program->link();
    if (!success)
        qDebug() << "ERR:" << m_program->log();


	VBO.create();
    EBO.create();

	VBO.bind();
    EBO.bind();

    VBO.allocate(vertices, 20 * sizeof(float));
    EBO.allocate(indices, 6 * sizeof(unsigned int));

	m_impl->texture = new QOpenGLTexture(QOpenGLTexture::Target2D);
	m_impl->texture->create();
	m_impl->texture->setMinificationFilter(QOpenGLTexture::Nearest);
	m_impl->texture->setMinificationFilter(QOpenGLTexture::Linear);
	m_impl->texture->setWrapMode(QOpenGLTexture::ClampToEdge);

	m_mux.unlock();

	// 启动定时器
    QTimer *ti = new QTimer(this);
    connect(ti, &QTimer::timeout, this, [=] {
        update();
    });
    ti->start(100);
}

void WOpenGLWidget::paintGL()
{
	glClearColor(0.2f, 0.3f, 0.3f, 1.0f);
	glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);

	m_mux.lock();
	if (m_isShowVideo)
	{		
        VBO.bind();
        EBO.bind();
		m_program->bind();

		m_impl->texture->bind(0);

		m_program->setUniformValue("texture", 0);

        int vertexLocation = m_program->attributeLocation("position");
        m_program->enableAttributeArray(vertexLocation);
        m_program->setAttributeBuffer(vertexLocation, GL_FLOAT, 0, 3, 5 * sizeof(float));

        int texcoordLocation = m_program->attributeLocation("texCoord");
        m_program->enableAttributeArray(texcoordLocation);
        m_program->setAttributeBuffer(texcoordLocation, GL_FLOAT, 3 * sizeof(float), 2, 5 * sizeof(float));

        //激活纹理单元0
        glActiveTexture(GL_TEXTURE0);
        glBindTexture(GL_TEXTURE_2D, m_impl->texture->textureId());
        glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
        glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, m_impl->videoW,
            m_impl->videoH, 0, GL_BGRA, GL_UNSIGNED_BYTE, m_impl->buffer);
        //设置纹理环绕方式
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);

        glDrawElements(GL_TRIANGLES,6, GL_UNSIGNED_INT,0);

        if(m_impl->texture)
            m_impl->texture->release();

        m_program->release();
	}
	m_mux.unlock();
}

void WOpenGLWidget::resizeGL(int w, int h)
{
	// 设置视口
	//glViewport(0, 0, w, h);
}

着色器如下所示。

//shapes.vert

#ifdef GL_ES
precision mediump int;
precision mediump float;
#endif

attribute vec3 position;
attribute vec2 texCoord;

varying vec2 outTexCoord;

void main()
{
    gl_Position = vec4(position,1.0);
    outTexCoord = vec2(texCoord.x, 1.0 - texCoord.y);
}

//shapes.frag

#ifdef GL_ES
// Set default precision to medium
precision mediump int;
precision mediump float;
#endif

uniform sampler2D texture;

varying vec2 outTexCoord;

void main()
{
    vec3 rgb = texture2D(texture, outTexCoord);
    gl_FragColor = vec4(rgb, 1);
}

ui界面如下图所示。 

运行截图：

 完整代码：

#pragma once

#include <QtWidgets/QWidget>
#include "ui_showWidget.h"
#include <QMenu>
#include <QActionGroup>
#include <vlc/vlc.h>
#include <QDebug>
#include <QFileDialog>
#include <QThread>
#include <QMouseEvent>
#include <QKeyEvent>

//视频每一帧数据回调
typedef std::function<void(uint8_t*)> VideoDataFunc;
//视频宽、高。回调
typedef std::function<void(int, int)> VideoInfoFunc;

enum Rate
{
	Rate2X,
	Rate1_5X,
	Rate1_25X,
	Rate1_0X,
	Rate0_75X,
	Rate0_5X
};

class showWidget : public QWidget
{
    Q_OBJECT

public:
    showWidget(QWidget *parent = nullptr);
    ~showWidget();

public:
	VideoDataFunc m_videoFunc;
	VideoInfoFunc m_videoInfoFunc;

private slots:
	void slotOpenFile();
	void slotPlay();
	void slotPause();
	void slotStop();
	void slotValueChanged(int value);
	void slotCurrentIndexChanged(int index);

private:
	//事件处理回调
	static void vlcEvents(const libvlc_event_t *ev, void *param);

private:
    Ui::showWidgetClass ui;

private:
	libvlc_instance_t *vlc_base = nullptr;
	libvlc_media_t *vlc_media = nullptr;
	libvlc_media_player_t *vlc_mediaPlayer = nullptr;

	QList<float> m_lstRate;
    QList<QString> m_lstAudioDevice;
};

//=====================================================

#include "showWidget.h"
#include <QTimer>
#include <QTime>
#include <QMutex>
#include <stdlib.h> 

#pragma execution_character_set("utf-8")

static showWidget* m_this = nullptr;

struct Frame 
{
	int     width;
	int     height;
	uchar * pixels;
	QMutex mutex;
};


static Frame *g_frame = nullptr;

// 自定义视频输出模块的回调函数
static void *lock(void *opaque, void **planes) {
	g_frame->mutex.lock();
	*planes = g_frame->pixels;

	return 0;
}

static void unlock(void *opaque, void *picture, void *const *planes) {
	// 这里可以释放视频帧的锁

	uint8_t *buffer = (uint8_t *)*planes; //planes即为帧数据
	//QImage image((unsigned char*)buffer, g_frame->width, g_frame->height, QImage::Format_ARGB32);
	m_this->m_videoFunc(buffer);

	g_frame->mutex.unlock();
}

static void display(void *opaque, void *picture) {
	// 这里可以进行视频帧的显示或其他处理
	(void)opaque;
}


static unsigned setup(void **opaque, char *chroma,
	unsigned *width, unsigned *height,
	unsigned *pitches,
	unsigned *lines)
{
	qDebug() << "chroma:" << QString(chroma) << "width:" << *width << ", height:" << *height;

	/* 开辟存放图像数据的内存块 */
	if (g_frame)
	{
		if (g_frame->pixels)
		{
			delete[] g_frame->pixels;
			g_frame->pixels = NULL;
		}

		delete g_frame;
		g_frame = NULL;
	}

	int w = *width;
	int h = *height;
	g_frame = new Frame;
	g_frame->pixels = new uchar[w * h * 4]; // 申请大小也为4通道的像素

	memset(g_frame->pixels, 0, w * h * 4);
	memcpy(chroma, "RV32", 4);
	g_frame->width = w;
	g_frame->height = h;
	*pitches = w * 4;
	*lines = h;
	m_this->m_videoInfoFunc(w, h);
	return 1;
}

showWidget::showWidget(QWidget *parent)
    : QWidget(parent)
{
    ui.setupUi(this);

	m_this = this;
	this->setWindowTitle("视频播放器");

	vlc_base = libvlc_new(0, NULL);

	ui.cbxRate->setCurrentIndex(Rate1_0X);

	m_lstRate << 2.0 << 1.5 << 1.25 << 1.0 << 0.75 << 0.5;

	ui.btnOpen->setFocusPolicy(Qt::NoFocus);
	ui.btnPlay->setFocusPolicy(Qt::NoFocus);
	ui.btnPause->setFocusPolicy(Qt::NoFocus);
	ui.btnStop->setFocusPolicy(Qt::NoFocus);
	ui.hSliderVolumn->setFocusPolicy(Qt::NoFocus);
	ui.cbxRate->setFocusPolicy(Qt::NoFocus);

	//视频数据
	m_videoFunc = std::bind(&WOpenGLWidget::slotReceiveVideoData, ui.openGLWidget,
		std::placeholders::_1);

	//视频信息
	m_videoInfoFunc = std::bind(&WOpenGLWidget::slotOpenVideo, ui.openGLWidget,
		std::placeholders::_1, std::placeholders::_2);

	connect(ui.btnOpen, &QPushButton::clicked, this, &showWidget::slotOpenFile);
	connect(ui.btnPlay, &QPushButton::clicked, this, &showWidget::slotPlay);
	connect(ui.btnPause, &QPushButton::clicked, this, &showWidget::slotPause);
	connect(ui.btnStop, &QPushButton::clicked, this, &showWidget::slotStop);
	connect(ui.hSliderVolumn, &QSlider::valueChanged, this, &showWidget::slotValueChanged);
	connect(ui.cbxRate,SIGNAL(currentIndexChanged(int)), this, SLOT(slotCurrentIndexChanged(int)));
}

showWidget::~showWidget()
{
	libvlc_release(vlc_base); //减少libvlc实例的引用计数，并销毁
}

void showWidget::slotOpenFile()
{
	/*选择文件*/
	QString filename = QFileDialog::getOpenFileName(this, "选择打开的文件", "D:/", tr("*.*"));
	std::replace(filename.begin(), filename.end(), QChar('/'), QChar('\\'));

	vlc_media = libvlc_media_new_path(vlc_base, filename.toUtf8().data());
	if (!vlc_media) {
		return;
	}

	// 创建libvlc实例和媒体播放器
	vlc_mediaPlayer = libvlc_media_player_new_from_media(vlc_media);
	if (!vlc_mediaPlayer) {
		return;
	}

	libvlc_video_set_format_callbacks(vlc_mediaPlayer, setup, NULL);
	
	// 设置自定义视频输出
	libvlc_video_set_callbacks(vlc_mediaPlayer, lock, unlock, display, NULL);
	
	// 等待元数据加载完成
	libvlc_media_parse(vlc_media);
	
	// 获取各种元数据
	const char *title = libvlc_media_get_meta(vlc_media, libvlc_meta_Title);
	const char *artist = libvlc_media_get_meta(vlc_media, libvlc_meta_Artist);
	const char *album = libvlc_media_get_meta(vlc_media, libvlc_meta_Album);
	const char *url = libvlc_media_get_meta(vlc_media, libvlc_meta_URL);
	const char *date = libvlc_media_get_meta(vlc_media, libvlc_meta_Date);
	const char *lang = libvlc_media_get_meta(vlc_media, libvlc_meta_Language);
	int duration = libvlc_media_get_duration(vlc_media);  // 获取时长（单位：毫秒）

	qDebug("Title: %s", title ? title : "N/A");
	qDebug("Artist: %s", artist ? artist : "N/A");
	qDebug("Album: %s", album ? album : "N/A");
	qDebug("Duration: %d ms", duration);
	qDebug("url: %s", url ? url : "N/A");
	qDebug("date: %s", date ? date : "N/A");
	qDebug("lang: %s", lang ? lang : "N/A");
	
	libvlc_media_track_t **tracks;
	int track_count = libvlc_media_tracks_get(vlc_media,&tracks);
	for (unsigned i = 0; i < track_count; i++) 
	{
		libvlc_media_track_t* track = tracks[i];

		// 显示轨道信息
		printf("Track #%u: %s\n", i, track->psz_description);

		// 这里可以获取到每一个轨道的信息，比如轨道类型 track->i_type
		// 可能是 libvlc_track_video, libvlc_track_audio 或者 libvlc_track_text (字幕)

		if (track->i_type == libvlc_track_video) {
			// 处理视频轨道信息
			qDebug("width = %d",track->video->i_width);
			qDebug("height = %d", track->video->i_height);
			qDebug("rate_num = %d", track->video->i_frame_rate_num);
			qDebug("rate_den = %d", track->video->i_frame_rate_den);
		}
		else if (track->i_type == libvlc_track_audio) {
			// 处理音频轨道信息
			qDebug("channels = %d", track->audio->i_channels);
			qDebug("rate = %d", track->audio->i_rate);
		}
		else if (track->i_type == libvlc_track_text) {
			// 处理字幕轨道信息
		}
	}

	//获取事件管理器
	libvlc_event_manager_t *em = libvlc_media_player_event_manager(vlc_mediaPlayer);

	// 注册事件监听器
	libvlc_event_attach(em, libvlc_MediaPlayerTimeChanged, vlcEvents, this);
	libvlc_event_attach(em, libvlc_MediaPlayerEndReached, vlcEvents, this);
	libvlc_event_attach(em, libvlc_MediaPlayerStopped, vlcEvents, this);
	libvlc_event_attach(em, libvlc_MediaPlayerPlaying, vlcEvents, this);
	libvlc_event_attach(em, libvlc_MediaPlayerPaused, vlcEvents, this);

	QTimer::singleShot(1000, this, &showWidget::slotPlay);
	libvlc_video_filter_list_get(vlc_base);
}

void showWidget::slotPlay()
{
	if (vlc_mediaPlayer)
	{
		libvlc_media_player_play(vlc_mediaPlayer);
	}
}

void showWidget::slotPause()
{
	if (vlc_mediaPlayer)
		libvlc_media_player_pause(vlc_mediaPlayer);
}

void showWidget::slotStop()
{
	if (vlc_mediaPlayer)
		libvlc_media_player_stop(vlc_mediaPlayer);
}

void showWidget::slotValueChanged(int value)
{
	if (vlc_mediaPlayer)
		libvlc_audio_set_volume(vlc_mediaPlayer, value);
}

void showWidget::slotCurrentIndexChanged(int index)
{
	if (vlc_mediaPlayer)
		libvlc_media_player_set_rate(vlc_mediaPlayer, m_lstRate[index]);
}

//事件回调
void showWidget::vlcEvents(const libvlc_event_t *ev, void *param)
{
	showWidget *w = (showWidget*)param;
	//处理不同的事件
	switch (ev->type) {
	case libvlc_MediaPlayerTimeChanged:
	{
		//qDebug() << "VLC媒体播放器时间已更改";
		qint64 len = libvlc_media_player_get_time(w->vlc_mediaPlayer);
		libvlc_time_t lenSec = len / 1000;

		libvlc_time_t totalLen = libvlc_media_player_get_length(w->vlc_mediaPlayer);
		libvlc_time_t totalLenSec = totalLen / 1000;

		int thh, tmm, tss;
		thh = lenSec / 3600;
		tmm = (lenSec % 3600) / 60;
		tss = (lenSec % 60);
		QTime time(thh, tmm, tss);
		w->ui.lbCurTime->setText(time.toString("hh:mm:ss"));

		thh = totalLenSec / 3600;
		tmm = (totalLenSec % 3600) / 60;
		tss = (totalLenSec % 60);
		QTime TotalTime(thh, tmm, tss);
		w->ui.lbTotalTime->setText(TotalTime.toString("hh:mm:ss"));

		double pos = (double)lenSec / totalLenSec * 100;
		w->ui.horizontalSlider->setValue(pos);
	}
		break;
	case libvlc_MediaPlayerEndReached:
		qDebug() << "VLC播放完毕.";
		break;
	case libvlc_MediaPlayerStopped:
		qDebug() << "VLC停止播放";
		break;
	case libvlc_MediaPlayerPlaying:
		qDebug() << "VLC开始播放";
		break;
	case libvlc_MediaPlayerPaused:
		qDebug() << "VLC暂停播放";
		break;
	}
}

更多参考：

Qt+FFmpeg+opengl从零制作视频播放器-7.OpenGL播放视频_qt opengl视频播放器-CSDN博客

Qt+FFmpeg+opengl从零制作视频播放器-1.项目介绍_qt opengl视频播放器-CSDN博客 

libVLC 添加图片和文本水印-CSDN博客

libVLC 音频输出设备切换-CSDN博客

libVLC 音频立体声模式切换-CSDN博客