1.页面效果
Web 采用 flask+vue 开发，效果图如下
2.后端
import sys
import subprocess
import os
from PIL import Image
from datetime import datetime
from ASR_metrics import utils as metricsfrom werkzeug.wrappers import Request, Response
from flask import Flask, render_template, request, jsonifysys.path.append('/home/nvidia/7th_CV')
sys.path.append('/home/nvidia/7th_ASR')# ASR 路径
pathASR = "/home/nvidia/7th_ASR"# 项目路径
pathSky = '/home/nvidia'app = Flask(__name__, static_folder='')# 上传路径
uploadPath = 'uploads/'
try_model_1 = None# 主页
@app.route('/')
def index():return render_template('sky7.html', template_folder='templates')# ------------------ASR------------------
# ASR 模型加载
@app.route('/asr/load')
def asrLoad():global try_model_1if try_model_1 == None:import nemo.collections.asr as nemo_asrprint('Loading Nemo')# 加载模型try_model_1 = nemo_asr.models.EncDecCTCModel.restore_from("/home/nvidia/7th_ASR/7th_asr_model.nemo")print('Done loading Nemo')return 'ok'# POST 请求上传音频
@app.route('/asr/upload', methods=['POST'])
def asrUpload():if request.method == 'POST':f = request.files['file']if(f.headers.get('Content-Type') != 'audio/wav'):return '音频格式有错误', 400else:fileName = f'{uploadPath}audio.wav'f.save(fileName)dt = datetime.now()ts = str(int(datetime.timestamp(dt)))return jsonify(f'/{uploadPath}audio.wav?t={ts}')# 识别上传的音频
@app.route('/asr/identify', methods=['GET', 'POST'])
def asrIdentify():global try_model_1if try_model_1 == None:return '模型无效，请重新加载', 500try:asr_result = try_model_1.transcribe(paths2audio_files=["uploads/audio.wav"])s1 = request.form.get('defaultText')s2 = " ".join(asr_result)#识别结果result = {"asr_result": asr_result,"word_error_rate": metrics.calculate_cer(s1,s2),"word_accuracy_rate":1-metrics.calculate_cer(s1,s2)}return jsonify(result)except Exception as e:return '无法识别', 400# ------------------CV------------------# POST 请求上传图片
@app.route("/cv/upload", methods=['POST'])
def cvUpload():if request.method == 'POST':f = request.files['file']print('image', f, f.filename)if not 'image' in f.headers.get('Content-Type'):return '图片有错误', 400original = f'{uploadPath}original.jpg'try:# Convert image to jpegim = Image.open(f)rgb_im = im.convert('RGB')rgb_im.save(original)# Add timestampdt = datetime.now()ts = str(int(datetime.timestamp(dt)))return jsonify(original+'?t='+ts)except Exception as e:return '有错误', 400# 检测图片
@app.route("/api/detect/image")
def detectImage():cv_results = subprocess.Popen('python3 /home/nvidia/7th_CV/detection_image.py /home/nvidia/uploads/original.jpg', shell = True, stdout = subprocess.PIPE, stderr = subprocess.PIPE)print('code', cv_results.returncode)cv_results = str(cv_results.stdout.read()).split('\\n')[-2]dt = datetime.now()ts = str(int(datetime.timestamp(dt)))result = {"detection_result_image_path": f'/uploads/result.jpg?t={ts}'}return jsonify(result)# 获取 FPS，以 Json 格式返回前端
@app.route("/api/detect/fps")
def detectFPS():# Code herefps_results = subprocess.Popen('python3 /home/nvidia/7th_CV/cv_fps.py', shell = True, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)fps_results = str(fps_results.stdout.read()).split('\\n')[-2]fps_results = fps_results.split(" ")[-1]result = {"detection_FPS": fps_results,}return jsonify(result)# 获取 mAP
@app.route("/api/detect/map")
def detectMAP():# Code heremap_results = subprocess.Popen('python3 /home/nvidia/7th_CV/cv_map.py', shell = True, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)map_results = str(map_results.stdout.read())bytes(map_results, encoding="utf-8").decode()map_results = map_results[-9:-3]result = {"detection_mAP": map_results,}return jsonify(result)if __name__ == "__main__":app.run(debug=True)
3.前端
3.1 html
<html><meta http-equiv="content-type" content="text/html; charset=UTF-8"><head><script src="https://unpkg.com/vue@3"></script><script src="https://unpkg.com/axios/dist/axios.min.js"></script><link rel="stylesheet" href="//unpkg.com/layui@2.6.8/dist/css/layui.css"><script src="//unpkg.com/layui@2.6.8/dist/layui.js"></script><link rel="stylesheet" type="text/css" href="/style.css"></head><body id="app"><!-- 模型加载中 --><div class="loading" v-if="loading!=''"><div class="pad">%%loading%%</div></div><h1>7th Sky Hackathon</h1><h5>team：早八睡不醒</h5><div class="content"><div class="layui-tab layui-tab-brief" lay-filter="docDemoTabBrief"><ul class="layui-tab-title"><li class="layui-this"><i class="layui-icon panel-title layui-icon"> ASR</i></li><li><i class="layui-icon panel-title layui-icon"> CV</i></li></ul><div class="layui-tab-content"><!-- ASR 开始--><div class="layui-tab-item layui-show"><div class="layui-anim layui-anim-up"><fieldset class="asr"><legend><span class="panel-title">ASR</span></legend><div class="layui-container"><!-- 1 开始 --><div class="layui-row"><div class="layui-col-md4">1.请加载语音识别模型</div><div class="layui-col-md4"><button class="layui-btn" @click="loadModel()" v-if="!modelLoaded"><iclass="layui-icon"> 加载</i></button><div v-if="modelLoaded" class="modelLoaded">模型加载成功</div></div></div><!-- 1 结束 --><!-- 2、3 开始 --><div class="field file"><div class="newFileUpload"><!-- 2 开始 --><div class="layui-row"><div class="layui-col-md4"><label for="file">2.请选择音频文件</label><div class="note"> 仅支持 .wav 和单声道格式</div></div><div class="layui-col-md4"><div class="userdefined-file"><input type="text" name="userdefinedFile"id="userdefinedFileAudio" value="未选择任何文件" /><button type="button">选择</button></div><input type="file" name="file" id="fileAudio"@change="handleFileUploadAudio($event)" /></div></div><!-- 2 结束 --></div><!-- 3 开始 --><div class="layui-row"><div class="layui-col-md4"> 3.请上传音频文件</div><div class="layui-col-md4"><button class="layui-btn" @click="submitFile('asr')"><iclass="layui-icon"> 上传</i></button></div></div><!-- 3 结束 --></div><!-- 2、3 结束 --><!-- 4 开始 --><div class="layui-row"><div class="field"><div class="layui-col-md4"><label>4.请试听上传语音并输入正确答案</label></div><div class="layui-col-md4"><input id="answer" type="text" name="defaultText"v-model="defaultText" /></div></div></div><!-- 4 结束 --><div class="field" v-if="asrStatus=='uploaded' || asrStatus=='identified'"><!-- 试听 开始 --><div class="layui-row"><div class="audio"><div class="layui-col-md4"> 试听 </div><div class="layui-col-md4"><audio controls :src="audioOriginal"></audio></div></div></div><!-- 试听 结束 --><!-- 5 开始 --><div class="layui-row"><div class="layui-col-md4">5.识别语音</div><div class="layui-col-md4"><div class="action"><button class="layui-btn" @click="identifyAudio()"><iclass="layui-icon"> 识别</i></button></div></div></div><!-- 5 结束 --></div><!-- 6 开始 --><div class="layui-row"><div class="field result asr" v-if="asrStatus=='identified'"><div class="layui-col-md4">6.指标</div><div class="layui-col-md4"><ul><li v-for="(value, key) in asrResult">%%key%%: %%value%%</li></ul></div></div></div><!-- 6 结束 --></div></fieldset></div></div><!-- ASR 结束 --><!-- CV 开始 --><div class="layui-tab-item"><div class="layui-anim layui-anim-up"><fieldset class="cv"><legend><span class="panel-title">CV</span></legend><div class="layui-container"><!-- 1 开始 --><div class="field"><div class="layui-row"><div class="layui-col-md4"><p>1. 获取 FPS</p><div class="item result"> FPS: %%cvFps%%</div></div><div class="layui-col-md4"><div class="item action"><button @click="getFps()" class="inline layui-btn"><iclass="layui-icon"> 获取</i></button></div></div></div></div><!-- 1 结束 --><!-- 2 开始 --><div class="field"><div class="layui-row"><div class="layui-col-md4"><p>2. 获取 mAP</p><div class="item result"> mAP: %%cvMap%%</div></div><div class="layui-col-md4"><div class="item"><button @click="getMap()" class="inline layui-btn"><iclass="layui-icon"> 获取</i></button></div></div></div></div><!-- 2 结束 --><!-- 3、4 开始 --><div class="field file"><!-- 3 开始 --><div class="newFileUpload"><div class="layui-row"><div class="layui-col-md4"><label for="file">3.请选择图像文件 </label><div class="note"> </div></div><div class="layui-col-md4"><div class="userdefined-file"><input type="text" name="userdefinedFile"id="userdefinedFileImage" value="未选择任何文件" /><button type="button">选择</button></div><input type="file" name="file" ref="file" id="fileImage"@change="handleFileUploadImage($event)" /></div></div></div><!-- 3 结束 --><!-- 4 开始 --><div class="layui-row"><div class="layui-col-md4">4.请上传图像文件 </div><div class="layui-col-md4"><button @click="submitFile('cv')" class="layui-btn"><iclass="layui-icon"> 上传</i></button></div></div><!-- 4 结束 --></div><!-- 3、4 结束 --><div class="action" v-if="imageOriginal!=''"><div class="layui-row"><div class="layui-col-md4">5.识别图片</div><div class="layui-col-md4"><button class="layui-btn" @click="identifyImage()"><iclass="layui-icon"> 识别</i></button><br></div></div></div><div class="field"><div class="layui-row"><div class="image original" v-if="imageOriginal!=''"><div class="layui-col-md1"><div class="label">原图</div></div><div class="layui-col-md3"><image :src="imageOriginal" /></div></div><div class="image result cv" v-if="imageResult!=''"><div class="layui-col-md1"><div class="label">结果图</div></div><div class="layui-col-md3"><image :src="imageResult" /></div></div></div></div></div></fieldset></div></div><!-- CV 结束 --></div></div></div></body><script>const {createApp} = VuecreateApp({data() {return {file: '',defaultText: '请检测出纸箱、瓶子和果皮',modelLoaded: false,imageOriginal: '',imageResult: '',audioOriginal: '',error: '',asrResult: {},cvMap: '',cvFps: '',loading: '',asrStatus: 'pending',cvStatus: 'pending'}},// Avoid conflict with Flask delimiterscompilerOptions: {delimiters: ["%%", "%%"]},methods: {async loadModel() {if (this.loading != '') return showError('在运行中，无法执行')this.loading = '加载模型中，请耐心等待...'this.modelLoaded = falsetry {var {data,status} = await axios.get('/asr/load')if (status == 200) {this.modelLoaded = true}} catch (err) {showError(err.response.data)}this.loading = ''},async submitFile(fileType) {let formData = new FormData()formData.append('file', this.file)if (this.file == "") {showError("请选择文件");return false;}statusType = fileType + 'Status'this.loading = '上传中...'try {var {data,status} = await axios.post('/' + fileType + '/upload', formData, {headers: {'Content-Type': 'multipart/form-data'}})if (status == 200) {this[statusType] = 'uploaded'if (fileType == 'cv') {this.imageOriginal = data} else {this.audioOriginal = data}}} catch (err) {showError(err.response.data)}this.loading = ''},handleFileUploadAudio(event) {document.getElementById("userdefinedFileAudio").value = document.getElementById("fileAudio").value;this.file = event.target.files[0];},handleFileUploadImage(event) {document.getElementById("userdefinedFileImage").value = document.getElementById("fileImage").value;this.file = event.target.files[0];},async identifyAudio(event) {// if (this.loading != '') return showError('在运行中，无法执行')this.loading = '识别中...'try {let formData = new FormData()formData.append('defaultText', this.defaultText)console.log('t', this.defaultText)var result = await axios.post('/asr/identify', formData)this['asrStatus'] = 'identified'this.asrResult = result.data} catch (err) {if (err.response.status == 500) this.modelLoaded = falseshowError(err.response.data)}this.loading = ''},async identifyImage(event) {if (this.loading != '') return showError('在运行中，无法执行')this.loading = '识别中...'this.cvStatus = 'pending'try {var {data} = await axios.get('/api/detect/image')this.imageResult = data['detection_result_image_path']} catch (err) {showError(err.response.data)}this.loading = ''},async getFps(event) {if (this.loading != '') return showError('在运行中，无法执行')this.loading = '获取 FPS...'try {var {data} = await axios.get('/api/detect/fps')this.cvFps = data['detection_FPS']} catch (err) {showError(err.response.data)}this.loading = ''},async getMap(event) {// 接口路径: /api/detect/map// 方式: GETif (this.loading != '') return showError('在运行中，无法执行')this.loading = '获取 mAP...'try {var {data} = await axios.get('/api/detect/map')this.cvMap = data['detection_mAP']} catch (err) {showError(err.response.data)}this.loading = ''}}}).mount('#app')function showError(msg) {layer.msg(msg || '错误')}</script>
</html>
3.2 CSS
body {font-size: 20px;margin: 0;
}.content {padding: 10px;margin: 40px;
}fieldset {border: 1px solid #ccc;padding: 10px;
}.modelLoaded {color: green;
}.note {color: #999;margin: 5px 0;font-size: 12px;
}.field {margin: 10px 0;
}.action {margin-top: 10px;
}.loading {position: fixed;top: 0;background: #E8F9D9;text-align: center;width: 100%;
}.pad {padding: 5px;
}.inline {display: inline-block;
}.field .item {margin: 5px 0;
}.image {display: inline-block;margin-right: 10px;
}.image img {max-height: 600px;
}/* 标题 */
h1,
h5 {margin: 40px;
}/* 队名 */
h5 {margin-left: 210px;
}/* 面板title */
.panel-title {font-size: 25px;
}/* 答案输入框 */
#answer {height: 40px;width: 200px;font-size: 14px;display: inline-block;vertical-align: middle;padding-right: 14px;padding-left: 14px;
}/* 音频文件选择 */
.newFileUpload {position: relative;height: 40px;line-height: 40px;
}.newFileUpload label {display: inline-block;
}.userdefined-file {position: absolute;top: 0;/* left: 200px; */z-index: 2;width: 300px;height: 40px;line-height: 40px;font-size: 0;/*应对子元素为 inline-block 引起的外边距*/
}.userdefined-file input[type="text"] {display: inline-block;vertical-align: middle;padding-right: 14px;padding-left: 14px;width: 220px;box-sizing: border-box;border: 1px solid #ccc;height: 40px;line-height: 40px;font-size: 14px;overflow: hidden;text-overflow: ellipsis;white-space: nowrap;
}.userdefined-file button {display: inline-block;vertical-align: middle;width: 80px;text-align: center;height: 40px;line-height: 40px;font-size: 14px;background-color: #009688;/* background-color: #f54; */border: none;color: #fff;cursor: pointer;
}.newFileUpload input[type="file"] {position: absolute;top: 0;/* left: 200px; */z-index: 3;opacity: 0;width: 300px;height: 40px;line-height: 40px;cursor: pointer;
}
