文章目录
- 视频流解码
- 音频流解码
原始的音视频数据数据量很大，为了方便传输和存储，我们会对原始数据进行压缩和编码。h264是常见的视频编码标准之一，AAC是常见的音频编码标准之一。这里介绍一下如何通过FFmpeg库将视频文件中的h264视频流解码成原始YUV视频数据，如何将AAC音频流解码成原始的PCM音频数据。音视频流的基本解码流程如下图所示:
视频流解码
视频文件中视频流的解码流程大体分为以下几步:
1.分配媒体文件上下文并获取流信息
2.查找对应的视频流和解码器
3.读取视频流中的数据包并解码
4.将解码之后的数据输出到对应的文件中
使用FFmpeg库对视频流进行解码的示例如下所示:
(采用h264编码的视频流进行验证通过)
#define _CRT_SECURE_NO_WARNINGS#include <libavutil/imgutils.h>
#include <libavutil/samplefmt.h>
#include <libavutil/timestamp.h>
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>//视频文件上下文
static AVFormatContext *fmt_ctx = NULL;
//视频解码器上下文
static AVCodecContext *video_dec_ctx = NULL;
//图像宽高
static int width, height;
//像素格式
static enum AVPixelFormat pix_fmt;
//视频流
static AVStream *video_stream = NULL;
//输入文件路径
static const char *src_filename = NULL;
//目标视频文件地址
static const char *video_dst_filename = NULL;
static FILE *video_dst_file = NULL;//图像数据缓存
static uint8_t *video_dst_data[4] = { NULL };
static int video_dst_linesize[4];
static int video_dst_bufsize;//数据流索引
static int video_stream_idx = -1;//帧信息和数据包索引
static AVFrame *frame = NULL;
static AVPacket *pkt = NULL;static int output_video_frame(AVFrame *frame)
{//将解码后的数据帧拷贝到对应的内存当中av_image_copy(video_dst_data, video_dst_linesize,(const uint8_t **)(frame->data), frame->linesize,pix_fmt, width, height);//将解析后的数据写入到文件中fwrite(video_dst_data[0], 1, video_dst_bufsize, video_dst_file);return 0;
}
//解码数据包
static int decode_packet(AVCodecContext *dec, const AVPacket *pkt)
{int ret = 0;ret = avcodec_send_packet(dec, pkt);if (ret < 0) {fprintf(stderr, "Error submitting a packet for decoding (%s)\n", av_err2str(ret));return ret;}//对数据进行解码输出while (ret >= 0) {ret = avcodec_receive_frame(dec, frame);if (ret < 0) {if (ret == AVERROR_EOF || ret == AVERROR(EAGAIN))return 0;return ret;}//将数据内容输出到文件if (dec->codec->type == AVMEDIA_TYPE_VIDEO)ret = output_video_frame(frame);av_frame_unref(frame);if (ret < 0)return ret;}return 0;
}//查找视频流的解码器
static int open_codec_context(int *stream_idx,AVCodecContext **dec_ctx, AVFormatContext *fmt_ctx, enum AVMediaType type)
{int ret, stream_index;AVStream *st;const AVCodec *dec = NULL;//查找流ret = av_find_best_stream(fmt_ctx, type, -1, -1, NULL, 0);if (ret < 0) {fprintf(stderr, "Could not find %s stream in input file '%s'\n",av_get_media_type_string(type), src_filename);return ret;}else {stream_index = ret;st = fmt_ctx->streams[stream_index];//通过数据流查找对应的解码器dec = avcodec_find_decoder(st->codecpar->codec_id);if (!dec) {fprintf(stderr, "Failed to find %s codec\n",av_get_media_type_string(type));return AVERROR(EINVAL);}//创建解码器的上下文*dec_ctx = avcodec_alloc_context3(dec);if (!*dec_ctx) {fprintf(stderr, "Failed to allocate the %s codec context\n",av_get_media_type_string(type));return AVERROR(ENOMEM);}//从输入流中拷贝对应的参数到解码器当中if ((ret = avcodec_parameters_to_context(*dec_ctx, st->codecpar)) < 0) {fprintf(stderr, "Failed to copy %s codec parameters to decoder context\n",av_get_media_type_string(type));return ret;}//初始化解码器if ((ret = avcodec_open2(*dec_ctx, dec, NULL)) < 0) {fprintf(stderr, "Failed to open %s codec\n",av_get_media_type_string(type));return ret;}*stream_idx = stream_index;}return 0;
}int main(int argc, char **argv)
{int ret = 0;if (argc != 3) {av_log(NULL, AV_LOG_DEBUG, "input argumets number is not valid\n");return 1;}src_filename = argv[1];video_dst_filename = argv[2];//打开视频文件分配上下文if (avformat_open_input(&fmt_ctx, src_filename, NULL, NULL) < 0) {av_log(NULL, AV_LOG_DEBUG, "open media file failed\n");return 2;}//获取流信息if (avformat_find_stream_info(fmt_ctx, NULL) < 0) {av_log(NULL, AV_LOG_DEBUG, "get stream info failed\n");return 3;}//获取视频流的索引和解码器if (open_codec_context(&video_stream_idx, &video_dec_ctx, fmt_ctx, AVMEDIA_TYPE_VIDEO) >= 0) {video_stream = fmt_ctx->streams[video_stream_idx];video_dst_file = fopen(video_dst_filename, "wb");if (!video_dst_file) {fprintf(stderr, "Could not open destination file %s\n", video_dst_filename);ret = 1;goto end;}//创建图像容器(像素宽高和图片格式)width = video_dec_ctx->width;height = video_dec_ctx->height;pix_fmt = video_dec_ctx->pix_fmt;ret = av_image_alloc(video_dst_data, video_dst_linesize,width, height, pix_fmt, 1);if (ret < 0) {fprintf(stderr, "Could not allocate raw video buffer\n");goto end;}video_dst_bufsize = ret;}//分配帧frame = av_frame_alloc();if (!frame) {fprintf(stderr, "Could not allocate frame\n");ret = AVERROR(ENOMEM);goto end;}//分配数据包pkt = av_packet_alloc();if (!pkt) {fprintf(stderr, "Could not allocate packet\n");ret = AVERROR(ENOMEM);goto end;}if (video_stream)printf("Demuxing video from file '%s' into '%s'\n", src_filename, video_dst_filename);//读取视频流中的数据包,并解码while (av_read_frame(fmt_ctx, pkt) >= 0) {if (pkt->stream_index == video_stream_idx)ret = decode_packet(video_dec_ctx, pkt);av_packet_unref(pkt);if (ret < 0)break;}//刷新解码器if (video_dec_ctx)decode_packet(video_dec_ctx, NULL);//输出原始视频文件的播放格式if (video_stream) {printf("Play the output video file with the command:\n""ffplay -f rawvideo -pix_fmt %s -video_size %dx%d %s\n",av_get_pix_fmt_name(pix_fmt), width, height,video_dst_filename);}//清理释放对应的数据信息
end:avcodec_free_context(&video_dec_ctx);avformat_close_input(&fmt_ctx);if (video_dst_file)fclose(video_dst_file);av_packet_free(&pkt);av_frame_free(&frame);av_free(video_dst_data[0]);getchar();return ret < 0;
}
音频流解码
视频文件中的AAC音频流数据解码成PCM原始音频数据的流程和视频流的解码流程大体类似，分为以下几步:
1.分配媒体文件上下文，获取流信息
2.查找对应的音频流和解码器
3.读取音频流中的数据包并解码
4.将解码之后的数据输出到对应的文件中
使用FFmpeg库对音频流进行解码的示例如下所示:
(采用AAC编码的音频流进行验证通过)
#define _CRT_SECURE_NO_WARNINGS#include <libavutil/imgutils.h>
#include <libavutil/samplefmt.h>
#include <libavutil/timestamp.h>
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>//视频文件上下文
static AVFormatContext *fmt_ctx = NULL;
//音频解码器上下文
static AVCodecContext *audio_dec_ctx = NULL;
//音频流
static AVStream *audio_stream = NULL;
//输入文件路径
static const char *src_filename = NULL;
//目标音频文件地址
static const char *audio_dst_filename = NULL;
static FILE *audio_dst_file = NULL;//数据流索引
static int audio_stream_idx = -1;//帧信息和数据包索引
static AVFrame *frame = NULL;
static AVPacket *pkt = NULL;//输出音频信息
static int output_audio_frame(AVFrame *frame)
{size_t unpadded_linesize = frame->nb_samples * av_get_bytes_per_sample(frame->format);fwrite(frame->extended_data[0], 1, unpadded_linesize, audio_dst_file);return 0;
}//获取采样时的数据位类型
static int get_format_from_sample_fmt(const char **fmt,enum AVSampleFormat sample_fmt)
{int i;struct sample_fmt_entry {enum AVSampleFormat sample_fmt; const char *fmt_be, *fmt_le;} sample_fmt_entries[] = {{ AV_SAMPLE_FMT_U8, "u8", "u8" },{ AV_SAMPLE_FMT_S16, "s16be", "s16le" },{ AV_SAMPLE_FMT_S32, "s32be", "s32le" },{ AV_SAMPLE_FMT_FLT, "f32be", "f32le" },{ AV_SAMPLE_FMT_DBL, "f64be", "f64le" },};*fmt = NULL;for (i = 0; i < FF_ARRAY_ELEMS(sample_fmt_entries); i++) {struct sample_fmt_entry *entry = &sample_fmt_entries[i];if (sample_fmt == entry->sample_fmt) {*fmt = AV_NE(entry->fmt_be, entry->fmt_le);return 0;}}fprintf(stderr,"sample format %s is not supported as output format\n",av_get_sample_fmt_name(sample_fmt));return -1;
}//解码数据包
static int decode_packet(AVCodecContext *dec, const AVPacket *pkt)
{int ret = 0;ret = avcodec_send_packet(dec, pkt);if (ret < 0) {fprintf(stderr, "Error submitting a packet for decoding (%s)\n", av_err2str(ret));return ret;}//对数据进行解码输出while (ret >= 0) {ret = avcodec_receive_frame(dec, frame);if (ret < 0) {if (ret == AVERROR_EOF || ret == AVERROR(EAGAIN))return 0;return ret;}//将数据内容输出到文件if (dec->codec->type == AVMEDIA_TYPE_AUDIO)ret = output_audio_frame(frame);av_frame_unref(frame);if (ret < 0)return ret;}return 0;
}//获取编码器
static int open_codec_context(int *stream_idx,AVCodecContext **dec_ctx, AVFormatContext *fmt_ctx, enum AVMediaType type)
{int ret, stream_index;AVStream *st;const AVCodec *dec = NULL;//查找流ret = av_find_best_stream(fmt_ctx, type, -1, -1, NULL, 0);if (ret < 0) {fprintf(stderr, "Could not find %s stream in input file '%s'\n",av_get_media_type_string(type), src_filename);return ret;}else {stream_index = ret;st = fmt_ctx->streams[stream_index];//通过数据流查找对应的解码器dec = avcodec_find_decoder(st->codecpar->codec_id);if (!dec) {fprintf(stderr, "Failed to find %s codec\n",av_get_media_type_string(type));return AVERROR(EINVAL);}//创建解码器的上下文*dec_ctx = avcodec_alloc_context3(dec);if (!*dec_ctx) {fprintf(stderr, "Failed to allocate the %s codec context\n",av_get_media_type_string(type));return AVERROR(ENOMEM);}//从输入流中拷贝对应的参数到解码器当中if ((ret = avcodec_parameters_to_context(*dec_ctx, st->codecpar)) < 0) {fprintf(stderr, "Failed to copy %s codec parameters to decoder context\n",av_get_media_type_string(type));return ret;}//初始化解码器if ((ret = avcodec_open2(*dec_ctx, dec, NULL)) < 0) {fprintf(stderr, "Failed to open %s codec\n",av_get_media_type_string(type));return ret;}*stream_idx = stream_index;}return 0;
}int main(int argc, char **argv)
{int ret = 0;if (argc != 3) {av_log(NULL, AV_LOG_DEBUG, "input argumets number is not valid\n");return 1;}src_filename = argv[1];audio_dst_filename = argv[2];//打开视频文件分配上下文if (avformat_open_input(&fmt_ctx, src_filename, NULL, NULL) < 0) {av_log(NULL, AV_LOG_DEBUG, "open media file failed\n");return 2;}//获取流信息if (avformat_find_stream_info(fmt_ctx, NULL) < 0) {av_log(NULL, AV_LOG_DEBUG, "get stream info failed\n");return 3;}//获取音频流的索引和解码器if (open_codec_context(&audio_stream_idx, &audio_dec_ctx, fmt_ctx, AVMEDIA_TYPE_AUDIO) >= 0) {audio_stream = fmt_ctx->streams[audio_stream_idx];audio_dst_file = fopen(audio_dst_filename, "wb");if (!audio_dst_file) {fprintf(stderr, "Could not open destination file %s\n", audio_dst_filename);ret = 1;goto end;}}//分配帧frame = av_frame_alloc();if (!frame) {fprintf(stderr, "Could not allocate frame\n");ret = AVERROR(ENOMEM);goto end;}//分配数据包pkt = av_packet_alloc();if (!pkt) {fprintf(stderr, "Could not allocate packet\n");ret = AVERROR(ENOMEM);goto end;}if (audio_stream)printf("Demuxing audio from file '%s' into '%s'\n", src_filename, audio_dst_filename);//读取音频流的数据包,并解码while (av_read_frame(fmt_ctx, pkt) >= 0) {if (pkt->stream_index == audio_stream_idx)ret = decode_packet(audio_dec_ctx, pkt);av_packet_unref(pkt);if (ret < 0)break;}//刷新解码器if (audio_dec_ctx)decode_packet(audio_dec_ctx, NULL);//输出PCM原始音频文件的播放格式if (audio_dec_ctx) {//采样格式和通道数量enum AVSampleFormat sfmt = audio_dec_ctx->sample_fmt;int n_channels = audio_dec_ctx->channels;const char *fmt;//获取PCM音频的数据类型if (av_sample_fmt_is_planar(sfmt)) {const char *packed = av_get_sample_fmt_name(sfmt);printf("Warning: the sample format the decoder produced is planar ""(%s). This example will output the first channel only.\n",packed ? packed : "?");sfmt = av_get_packed_sample_fmt(sfmt);n_channels = 1;}if ((ret = get_format_from_sample_fmt(&fmt, sfmt)) < 0)goto end;//输出播放原始数据的命令printf("Play the output audio file with the command:\n""ffplay -f %s -ac %d -ar %d %s\n",fmt, n_channels, audio_dec_ctx->sample_rate,audio_dst_filename);}//清理释放对应的数据信息
end:avcodec_free_context(&audio_dec_ctx);avformat_close_input(&fmt_ctx);if (audio_dst_file)fclose(audio_dst_file);av_packet_free(&pkt);av_frame_free(&frame);getchar();return ret < 0;
}
![[Unity] 获取UI组件的屏幕坐标（打包手机端使用前置摄像头）](https://img-blog.csdnimg.cn/223afd5653bc43bbaffaf45e7d715532.png)
