Fix publishing apks to huggingface (#1121)

Save APKs for each release in a separate directory. Huggingface requires that each directory cannot contain more than 1000 files. Since we have so many tts models and for each model we need to build APKs of 4 different ABIs, it is a workaround for the huggingface's constraint by placing them into separate directories for different releases.
k2-fsa · Jul 13, 2024 · b5093e2 · b5093e2
1 parent 54e6e96
commit b5093e2
Show file tree

Hide file tree

Showing 36 changed files with 123 additions and 60 deletions.
diff --git a/.github/workflows/apk-asr-2pass.yaml b/.github/workflows/apk-asr-2pass.yaml
@@ -2,6 +2,8 @@ name: apk-asr-2pass
 
 on:
   push:
+    branches:
+      - apk
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+*'
 
@@ -160,14 +162,19 @@ jobs:
             export GIT_LFS_SKIP_SMUDGE=1
             export GIT_CLONE_PROTECTION_ACTIVE=false
 
+            SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt  | cut -d " " -f 2  | cut -d '"' -f 2)
+            echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION"
+
             git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface
             cd huggingface
             git fetch
             git pull
             git merge -m "merge remote" --ff origin main
 
-            mkdir -p asr-2pass
-            cp -v ../apks/*.apk ./asr-2pass/
+            d=asr-2pass/$SHERPA_ONNX_VERSION
+            mkdir -p $d
+
+            cp -v ../apks/*.apk $d/
             git status
             git lfs track "*.apk"
             git add .

diff --git a/.github/workflows/apk-asr.yaml b/.github/workflows/apk-asr.yaml
@@ -2,6 +2,8 @@ name: apk-asr
 
 on:
   push:
+    branches:
+      - apk
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+*'
 
@@ -160,14 +162,18 @@ jobs:
             export GIT_LFS_SKIP_SMUDGE=1
             export GIT_CLONE_PROTECTION_ACTIVE=false
 
+            SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt  | cut -d " " -f 2  | cut -d '"' -f 2)
+            echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION"
+
             git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface
             cd huggingface
             git fetch
             git pull
             git merge -m "merge remote" --ff origin main
 
-            mkdir -p asr
-            cp -v ../apks/*.apk ./asr/
+            d=asr/$SHERPA_ONNX_VERSION
+            mkdir -p $d
+            cp -v ../apks/*.apk $d/
             git status
             git lfs track "*.apk"
             git add .

diff --git a/.github/workflows/apk-audio-tagging-wearos.yaml b/.github/workflows/apk-audio-tagging-wearos.yaml
@@ -2,6 +2,8 @@ name: apk-audio-tagging-wearos
 
 on:
   push:
+    branches:
+      - apk
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+*'
 
@@ -160,14 +162,18 @@ jobs:
             export GIT_LFS_SKIP_SMUDGE=1
             export GIT_CLONE_PROTECTION_ACTIVE=false
 
+            SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt  | cut -d " " -f 2  | cut -d '"' -f 2)
+            echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION"
+
             git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface
             cd huggingface
             git fetch
             git pull
             git merge -m "merge remote" --ff origin main
 
-            mkdir -p audio-tagging-wearos
-            cp -v ../apks/*.apk ./audio-tagging-wearos/
+            d=audio-tagging-wearos/$SHERPA_ONNX_VERSION
+            mkdir -p $d
+            cp -v ../apks/*.apk $d/
             git status
             git lfs track "*.apk"
             git add .

diff --git a/.github/workflows/apk-audio-tagging.yaml b/.github/workflows/apk-audio-tagging.yaml
@@ -2,6 +2,8 @@ name: apk-audio-tagging
 
 on:
   push:
+    branches:
+      - apk
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+*'
 

diff --git a/.github/workflows/apk-kws.yaml b/.github/workflows/apk-kws.yaml
@@ -2,6 +2,8 @@ name: apk-kws
 
 on:
   push:
+    branches:
+      - apk
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+*'
 
@@ -157,14 +159,18 @@ jobs:
             export GIT_LFS_SKIP_SMUDGE=1
             export GIT_CLONE_PROTECTION_ACTIVE=false
 
+            SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt  | cut -d " " -f 2  | cut -d '"' -f 2)
+            echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION"
+
             git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface
             cd huggingface
             git fetch
             git pull
             git merge -m "merge remote" --ff origin main
 
-            mkdir -p kws
-            cp -v ../apks/*.apk ./kws/
+            d=kws/$SHERPA_ONNX_VERSION
+            mkdir -p $d
+            cp -v ../apks/*.apk $d/
             git status
             git lfs track "*.apk"
             git add .

diff --git a/.github/workflows/apk-speaker-identification.yaml b/.github/workflows/apk-speaker-identification.yaml
@@ -2,6 +2,8 @@ name: apk-speaker-identification
 
 on:
   push:
+    branches:
+      - apk
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+*'
 
@@ -98,14 +100,18 @@ jobs:
             export GIT_LFS_SKIP_SMUDGE=1
             export GIT_CLONE_PROTECTION_ACTIVE=false
 
+            SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt  | cut -d " " -f 2  | cut -d '"' -f 2)
+            echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION"
+
             git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface
             cd huggingface
             git fetch
             git pull
             git merge -m "merge remote" --ff origin main
 
-            mkdir -p speaker-identification
-            cp -v ../apks/*.apk ./speaker-identification/
+            d=speaker-identification/$SHERPA_ONNX_VERSION
+            mkdir -p $d/
+            cp -v ../apks/*.apk $d/
             git status
             git lfs track "*.apk"
             git add .

diff --git a/.github/workflows/apk-spoken-language-identification.yaml b/.github/workflows/apk-spoken-language-identification.yaml
@@ -2,6 +2,8 @@ name: apk-slid
 
 on:
   push:
+    branches:
+      - apk
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+*'
 
@@ -160,14 +162,18 @@ jobs:
             export GIT_LFS_SKIP_SMUDGE=1
             export GIT_CLONE_PROTECTION_ACTIVE=false
 
+            SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt  | cut -d " " -f 2  | cut -d '"' -f 2)
+            echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION"
+
             git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface
             cd huggingface
             git fetch
             git pull
             git merge -m "merge remote" --ff origin main
 
-            mkdir -p slid
-            cp -v ../apks/*.apk ./slid/
+            d=slid/$SHERPA_ONNX_VERSION
+            mkdir -p $d/
+            cp -v ../apks/*.apk $d/
             git status
             git lfs track "*.apk"
             git add .

diff --git a/.github/workflows/apk-tts-engine.yaml b/.github/workflows/apk-tts-engine.yaml
@@ -2,6 +2,8 @@ name: apk-tts-engine
 
 on:
   push:
+    branches:
+      - apk
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+*'
 
@@ -161,14 +163,18 @@ jobs:
             export GIT_LFS_SKIP_SMUDGE=1
             export GIT_CLONE_PROTECTION_ACTIVE=false
 
+            SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt  | cut -d " " -f 2  | cut -d '"' -f 2)
+            echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION"
+
             git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface
             cd huggingface
             git fetch
             git pull
             git merge -m "merge remote" --ff origin main
 
-            mkdir -p tts-engine-2
-            cp -v ../apks/*.apk ./tts-engine-2/
+            d=tts-engine-new/$SHERPA_ONNX_VERSION
+            mkdir -p $d
+            cp -v ../apks/*.apk $d/
             git status
             git lfs track "*.apk"
             git add .

diff --git a/.github/workflows/apk-tts.yaml b/.github/workflows/apk-tts.yaml
@@ -2,6 +2,8 @@ name: apk-tts
 
 on:
   push:
+    branches:
+      - apk
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+*'
 
@@ -161,14 +163,18 @@ jobs:
             export GIT_LFS_SKIP_SMUDGE=1
             export GIT_CLONE_PROTECTION_ACTIVE=false
 
+            SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt  | cut -d " " -f 2  | cut -d '"' -f 2)
+            echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION"
+
             git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface
             cd huggingface
             git fetch
             git pull
             git merge -m "merge remote" --ff origin main
 
-            mkdir -p tts-2
-            cp -v ../apks/*.apk ./tts-2/
+            d=tts-new/$SHERPA_ONNX_VERSION
+            mkdir -p $d
+            cp -v ../apks/*.apk $d/
             git status
             git lfs track "*.apk"
             git add .

diff --git a/.github/workflows/apk-vad-asr.yaml b/.github/workflows/apk-vad-asr.yaml
@@ -2,6 +2,8 @@ name: apk-vad-asr
 
 on:
   push:
+    branches:
+      - apk
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+*'
 
@@ -160,14 +162,18 @@ jobs:
             export GIT_LFS_SKIP_SMUDGE=1
             export GIT_CLONE_PROTECTION_ACTIVE=false
 
+            SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt  | cut -d " " -f 2  | cut -d '"' -f 2)
+            echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION"
+
             git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface
             cd huggingface
             git fetch
             git pull
             git merge -m "merge remote" --ff origin main
 
-            mkdir -p vad-asr
-            cp -v ../apks/*.apk ./vad-asr/
+            d=vad-asr/$SHERPA_ONNX_VERSION
+            mkdir -p $d
+            cp -v ../apks/*.apk $d/
             git status
             git lfs track "*.apk"
             git add .

diff --git a/.github/workflows/apk-vad.yaml b/.github/workflows/apk-vad.yaml
@@ -2,6 +2,8 @@ name: apk-vad
 
 on:
   push:
+    branches:
+      - apk
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+*'
 
@@ -157,14 +159,18 @@ jobs:
             export GIT_LFS_SKIP_SMUDGE=1
             export GIT_CLONE_PROTECTION_ACTIVE=false
 
+            SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt  | cut -d " " -f 2  | cut -d '"' -f 2)
+            echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION"
+
             git clone https://huggingface.co/csukuangfj/sherpa-onnx-apk huggingface
             cd huggingface
             git fetch
             git pull
             git merge -m "merge remote" --ff origin main
 
-            mkdir -p vad
-            cp -v ../apks/*.apk ./vad/
+            d=vad/SHERPA_ONNX_VERSION
+            mkdir -p $d
+            cp -v ../apks/*.apk $d/
             git status
             git lfs track "*.apk"
             git add .

diff --git a/go-api-examples/vad-asr-paraformer/main.go b/go-api-examples/vad-asr-paraformer/main.go
@@ -15,7 +15,7 @@ func main() {
 	config := sherpa.VadModelConfig{}
 
 	// Please download silero_vad.onnx from
-	// https://github.com/snakers4/silero-vad/blob/master/files/silero_vad.onnx
+	// https://github.com/snakers4/silero-vad/raw/master/src/silero_vad/data/silero_vad.onnx
 
 	config.SileroVad.Model = "./silero_vad.onnx"
 	config.SileroVad.Threshold = 0.5

diff --git a/go-api-examples/vad-asr-paraformer/run.sh b/go-api-examples/vad-asr-paraformer/run.sh
@@ -3,7 +3,7 @@
 set -ex
 
 if [ ! -f ./silero_vad.onnx ]; then
-  curl -SL -O https://github.com/snakers4/silero-vad/raw/master/files/silero_vad.onnx
+  curl -SL -O https://github.com/snakers4/silero-vad/raw/master/src/silero_vad/data/silero_vad.onnx
 fi
 
 if [ ! -f ./sherpa-onnx-paraformer-trilingual-zh-cantonese-en/model.int8.onnx ]; then

diff --git a/go-api-examples/vad-asr-whisper/main.go b/go-api-examples/vad-asr-whisper/main.go
@@ -15,7 +15,7 @@ func main() {
 	config := sherpa.VadModelConfig{}
 
 	// Please download silero_vad.onnx from
-	// https://github.com/snakers4/silero-vad/blob/master/files/silero_vad.onnx
+	// https://github.com/snakers4/silero-vad/raw/master/src/silero_vad/data/silero_vad.onnx
 
 	config.SileroVad.Model = "./silero_vad.onnx"
 	config.SileroVad.Threshold = 0.5

diff --git a/go-api-examples/vad-asr-whisper/run.sh b/go-api-examples/vad-asr-whisper/run.sh
@@ -3,7 +3,7 @@
 set -ex
 
 if [ ! -f ./silero_vad.onnx ]; then
-  curl -SL -O https://github.com/snakers4/silero-vad/raw/master/files/silero_vad.onnx
+  curl -SL -O https://github.com/snakers4/silero-vad/raw/master/src/silero_vad/data/silero_vad.onnx
 fi
 
 if [ ! -f ./sherpa-onnx-whisper-tiny.en/tiny.en-encoder.int8.onnx ]; then

diff --git a/go-api-examples/vad-speaker-identification/main.go b/go-api-examples/vad-speaker-identification/main.go
@@ -89,7 +89,7 @@ func createVad() *sherpa.VoiceActivityDetector {
 	config := sherpa.VadModelConfig{}
 
 	// Please download silero_vad.onnx from
-	// https://github.com/snakers4/silero-vad/blob/master/files/silero_vad.onnx
+	// https://github.com/snakers4/silero-vad/raw/master/src/silero_vad/data/silero_vad.onnx
 
 	config.SileroVad.Model = "./silero_vad.onnx"
 	config.SileroVad.Threshold = 0.5

diff --git a/go-api-examples/vad-speaker-identification/run.sh b/go-api-examples/vad-speaker-identification/run.sh
@@ -11,7 +11,7 @@ if [ ! -f ./sr-data/enroll/fangjun-sr-1.wav ]; then
 fi
 
 if [ ! -f ./silero_vad.onnx ]; then
-  curl -SL -O https://github.com/snakers4/silero-vad/raw/master/files/silero_vad.onnx
+  curl -SL -O https://github.com/snakers4/silero-vad/raw/master/src/silero_vad/data/silero_vad.onnx
 fi
 
 go mod tidy

diff --git a/go-api-examples/vad-spoken-language-identification/main.go b/go-api-examples/vad-spoken-language-identification/main.go
@@ -15,7 +15,7 @@ func main() {
 	config := sherpa.VadModelConfig{}
 
 	// Please download silero_vad.onnx from
-	// https://github.com/snakers4/silero-vad/blob/master/files/silero_vad.onnx
+	// https://github.com/snakers4/silero-vad/raw/master/src/silero_vad/data/silero_vad.onnx
 
 	config.SileroVad.Model = "./silero_vad.onnx"
 	config.SileroVad.Threshold = 0.5

diff --git a/go-api-examples/vad-spoken-language-identification/run.sh b/go-api-examples/vad-spoken-language-identification/run.sh
@@ -3,7 +3,7 @@
 set -ex
 
 if [ ! -f ./silero_vad.onnx ]; then
-  curl -SL -O https://github.com/snakers4/silero-vad/raw/master/files/silero_vad.onnx
+  curl -SL -O https://github.com/snakers4/silero-vad/raw/master/src/silero_vad/data/silero_vad.onnx
 fi
 
 if [ ! -f ./sherpa-onnx-whisper-tiny/tiny-encoder.int8.onnx ]; then

diff --git a/go-api-examples/vad/main.go b/go-api-examples/vad/main.go
@@ -13,7 +13,7 @@ func main() {
 	config := sherpa.VadModelConfig{}
 
 	// Please download silero_vad.onnx from
-	// https://github.com/snakers4/silero-vad/blob/master/files/silero_vad.onnx
+	// https://github.com/snakers4/silero-vad/raw/master/src/silero_vad/data/silero_vad.onnx
 
 	config.SileroVad.Model = "./silero_vad.onnx"
 	config.SileroVad.Threshold = 0.5

diff --git a/go-api-examples/vad/run.sh b/go-api-examples/vad/run.sh
@@ -3,7 +3,7 @@
 set -ex
 
 if [ ! -f ./silero_vad.onnx ]; then
-  curl -SL -O https://github.com/snakers4/silero-vad/raw/master/files/silero_vad.onnx
+  curl -SL -O https://github.com/snakers4/silero-vad/raw/master/src/silero_vad/data/silero_vad.onnx
 fi
 
 go mod tidy
-Original file line number
+Diff line change
@@ Expand Up / @@ -2,6 +2,8 @@ name: apk-audio-tagging @@
     on:
       push:
+        branches:
+          - apk
         tags:
           - 'v[0-9]+.[0-9]+.[0-9]+*'
@@ Expand Down @@