重构代码，解决输出的音符时间错乱问题

happyme531 · Dec 8, 2021 · 526b353 · 526b353
1 parent e879be0
commit 526b353
Show file tree

Hide file tree

Showing 9 changed files with 102 additions and 24 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.16.0)
-project(GenshinImpactPianoExtract VERSION 0.1.2)
+project(GenshinImpactPianoExtract VERSION 0.1.3)
 
 include(CTest)
 enable_testing()

diff --git a/src/buildconfig.h b/src/buildconfig.h
@@ -1,6 +1,6 @@
 //buildconfig.h 由 cmake 从 buildconfig.h.in 自动生成
 #pragma once
-#define LversionStr  L"0.1.2"
-#define versionStr "0.1.2"
+#define LversionStr  L"0.1.3"
+#define versionStr "0.1.3"
 constexpr int DEFAULT_KEY_THRESHOLD = 6;
 constexpr int DEFAULT_LOG_LEVEL = 4;
diff --git a/src/keyposfinder.cpp b/src/keyposfinder.cpp
@@ -15,6 +15,7 @@ void KeyPosFinder::begin(string filePath){
     double fps = video.get(CAP_PROP_FPS);
     int videoHeight = video.get(CAP_PROP_FRAME_HEIGHT);
     int videoWidth = video.get(CAP_PROP_FRAME_WIDTH);
+
     wstringstream strBuf;
     strBuf << L"视频高度:" << videoHeight << L", 宽度:" << videoWidth << L", fps:" << fps;
     ui.logI(strBuf.str());
@@ -24,9 +25,13 @@ void KeyPosFinder::begin(string filePath){
     Mat frame;
     Mat resizedFrame;
     if (logLevel == 5) namedWindow("frames");
+    bool detectedGUIOpen = false;
     // #pragma omp parallel
     while (minFramesCnt != vaildFramesCnt){
+        //针对某些视频一开始根本没有打开界面的问题，在没有检测到界面时每隔5帧读取一帧，节省时间
         if(!video.read(frame)) break;
+        if(!detectedGUIOpen) video.set(CAP_PROP_POS_FRAMES, video.get(CAP_PROP_POS_FRAMES) + 4); // 跳过4帧
+
         resize(frame,resizedFrame,Size(),0.5,0.5);   //宽高缩小一半
         resizedFrame = resizedFrame(Rect(Point(0,videoHeight/4),Point(videoWidth/2-1,videoHeight/2-1))); //切去上半部分, 保留下半部分
         pyrMeanShiftFiltering(resizedFrame,resizedFrame,15*scaleFactor,100*scaleFactor);
@@ -36,6 +41,10 @@ void KeyPosFinder::begin(string filePath){
         vector<Vec3f> circles;
         HoughCircles(resizedFrame,circles, HOUGH_GRADIENT,2,50*scaleFactor,200,55,0,60*scaleFactor);
         ui.updateMsg(L"找到的按键数量:" + to_wstring(circles.size()));
+        if (!detectedGUIOpen && circles.size() > 10 ){
+            detectedGUIOpen = true;
+            startFrame = video.get(CAP_PROP_POS_FRAMES);
+        }   
 
         if (logLevel == 5) {
           for (auto c : circles)

diff --git a/src/keyposfinder.h b/src/keyposfinder.h
@@ -16,13 +16,19 @@ class KeyPosFinder {
   vector<vector<Vec3f>> vaildCircles; //3x7 = 21个按键
   array<Vec3f,21> result;
 
+  //视频中界面被打开时的帧数
+  int startFrame = 0;
+
 
 public:
 
     void begin(string filePath);
     array<Vec3f,21>&& getResult(){
       return move(result);
     }
+    int getStartFrame(){
+      return startFrame;
+    }
     KeyPosFinder(TermUI& _ui): ui(_ui){};
     ~KeyPosFinder(){};
 };
diff --git a/src/main.cpp b/src/main.cpp
@@ -99,8 +99,9 @@ ps:我不玩原神)123");
   string filepath = parser.get<string>("input");
   KeyPosFinder finder(ui); 
   finder.begin(filepath);
+  int startFrame = finder.getStartFrame();
   NoteDetector detector(ui);
-  detector.begin(filepath,finder.getResult(),parser.get<int>("--note-threshold"));
+  detector.begin(filepath,finder.getResult(),parser.get<int>("--note-threshold"),startFrame);
   MidiWriter writer;
 
   //let the output file to be the input file except the extension changed to mid 

diff --git a/src/midiwriter.cpp b/src/midiwriter.cpp
@@ -46,15 +46,15 @@ static int keyToPitch(int key){
 void MidiWriter::toMidiFile(vector<noteData> data, string fileName){
     MidiFile midiFile;
     midiFile.addTimbre(0,0,0,0);
-    midiFile.setTPQ(120);
+    midiFile.setTPQ(240);
     midiFile.absoluteTicks();
     midiFile.addTempo(0, 0, 60);
     for (unsigned int i = 0; i < data.size(); i++) {
       if(data[i].beginTime < 1) continue; //跳过无效的音符
       // int startTick = data[i].beginTime * 1000;
       // int endTick = startTick + 200;
-      int startTick = data[i].beginTime * 120;
-      int endTick = startTick + 30;
+      int startTick = data[i].beginTime * 240;
+      int endTick = startTick + 60;
       midiFile.addNoteOn(0, startTick, 0, keyToPitch(data[i].key), 100);
       midiFile.addNoteOff(0, endTick, 0, keyToPitch(data[i].key));
     }

diff --git a/src/notedetector.cpp b/src/notedetector.cpp
@@ -1,4 +1,5 @@
 #include "notedetector.h"
+#include "pulsedetector.h"
 
 #include <opencv2/opencv.hpp>
 #include "utils.h"
@@ -31,7 +32,7 @@ static void decoderLoop(VideoCapture& video) {
   }
 }
 
-void NoteDetector::begin(string filePath, array<Vec3f, 21> keyPos, int threshold) {
+void NoteDetector::begin(string filePath, array<Vec3f, 21> keyPos, int threshold, int startFrame) {
     ui.addTask(L"获取按键数据");
     VideoCapture video(filePath);
     if (!video.isOpened()) {
@@ -41,6 +42,8 @@ void NoteDetector::begin(string filePath, array<Vec3f, 21> keyPos, int threshold
     double frameTime = 1.0/fps;
     int videoHeight = video.get(CAP_PROP_FRAME_HEIGHT);
     int videoWidth = video.get(CAP_PROP_FRAME_WIDTH);
+    video.set(CAP_PROP_POS_FRAMES, startFrame);
+    int curFrame = startFrame;
     //Mat frame;
     Mat resizedFrame;
     if (logLevel == 5)
@@ -49,6 +52,12 @@ void NoteDetector::begin(string filePath, array<Vec3f, 21> keyPos, int threshold
     pressedKeys.fill(false);
     decoderThread = new thread(decoderLoop, ref(video));
     decoderThread->detach();
+
+    vector<PulseDetector<10>*> detectors;
+    for (int i = 0; i < 21; i++) {
+      detectors.push_back(new PulseDetector<10>(threshold));
+    }
+
     while (!videoEnded){    //处理到视频结尾
         if(video.get(CAP_PROP_POS_FRAMES) <= 2) pressedKeys.fill(false);
         while (frameQueue.size() == 0) {
@@ -75,18 +84,15 @@ void NoteDetector::begin(string filePath, array<Vec3f, 21> keyPos, int threshold
             double avgColor = mean(resizedFrame,mask)[0];
 
             //检测按下
-            if(avgColor - lastColors[i] < -threshold){
-                if(!pressedKeys[i] && video.get(CAP_PROP_POS_FRAMES) > 2){ //防止连续统计并丢弃前几帧不稳定的数据
-
-                  //记录音符
-                  noteData note = {
-                      .key = i,
-                      .beginTime = frameTime * video.get(CAP_PROP_POS_FRAMES)
-                      };
-                  pressedKeys[i] = true;
-                  result.push_back(note);
-                }
-            }else if(avgColor - lastColors[i] > threshold / 3.0f){
+            detectors[i]->addSample(avgColor);
+            if(detectors[i]->havePulse()){
+              //记录音符
+              noteData note = {
+                  .key = i,
+                  .beginTime = frameTime * curFrame - frameTime * startFrame};
+              pressedKeys[i] = true;
+              result.push_back(note);
+            }else{
                 pressedKeys[i] = false;
             }
 
@@ -97,18 +103,25 @@ void NoteDetector::begin(string filePath, array<Vec3f, 21> keyPos, int threshold
           for (int i = 0; i < 21; i++){
             auto c = keyPos[i];
             if (pressedKeys[i]){
-              circle(resizedFrame, Point(c[0], c[1]), c[2] * 0.95, Scalar(0, 255, 255), 1);
+              circle(resizedFrame, Point(c[0], c[1]), c[2] * 0.95, Scalar(0, 255, 255), 2);
               circle(resizedFrame, Point(c[0], c[1]), c[2] * 0.75, Scalar(0, 255, 255), 1);
             }
           }
+            this_thread::sleep_for(chrono::milliseconds(20));
 
           imshow("frames", resizedFrame);
           waitKey(1);   //必须加上,否则预览窗口灰屏卡住
         }
-        ui.updateLastTaskProgress(video.get(CAP_PROP_POS_FRAMES),video.get(CAP_PROP_FRAME_COUNT));
+        curFrame++;
+        ui.updateLastTaskProgress(curFrame,video.get(CAP_PROP_FRAME_COUNT));
         ui.updateMsg(L"音符数量:" + to_wstring(result.size()));
         delete &frame;
     }
-    ui.updateLastTaskProgress(video.get(CAP_PROP_POS_FRAMES),video.get(CAP_PROP_FRAME_COUNT));
+    video.release();
+    delete decoderThread;
+    for (int i = 0; i < 21; i++) {
+      delete detectors[i];
+    }
+    ui.updateLastTaskProgress(curFrame,video.get(CAP_PROP_FRAME_COUNT));
     ui.updateMsg(L"音符数量:" + to_wstring(result.size()));
 }
diff --git a/src/notedetector.h b/src/notedetector.h
@@ -27,7 +27,7 @@ class NoteDetector {
 
 public:
 
-    void begin(string filePath, array<Vec3f,21> keyPos, int threshold);
+    void begin(string filePath, array<Vec3f,21> keyPos, int threshold, int startFrame = 0);
     vector<noteData>&& getResult(){
       return move(result);
     }

diff --git a/src/pulsedetector.h b/src/pulsedetector.h
@@ -0,0 +1,49 @@
+#include <deque>
+#include <algorithm>
+#include <numeric>
+
+template<int BUFFER_SIZE>
+class PulseDetector{
+private:
+    //触发状态, 防止连续触发
+    bool triggered = false;
+    bool pulse = false;
+
+    float threshold = 0.0f;
+    float last_sample = 0;
+
+public:
+    PulseDetector(float threshold_){
+        threshold = threshold_;
+    }
+    ~PulseDetector(){};
+    void addSample(float sample){
+        float delta = -(sample - last_sample);
+        last_sample = sample;
+        if(delta > threshold){
+            if(!triggered){
+                triggered = true;
+            }
+            return;
+        }
+        //复位
+        if(delta < 0){
+            if(triggered){
+                triggered = false;
+                pulse = true;
+            }
+            return;
+        }
+
+    }
+
+    bool havePulse(){
+        if(pulse){
+            pulse = false;
+            return true;
+        }
+        return false;
+    }
+    void reset(){
+    }
+};