java audio embedding

java 音频嵌入

Java 中的音频嵌入

当前项目尝试开发一个纯Java音频编码器，可以在纯Java或Android程序中使用。这样的音频编码器可用于音乐流派分类或音乐搜索或音乐推荐器。

当前项目包含两个深度学习网络，采用自：

• 资源网
• 西法尔

这两个模型的训练和验证如下图所示：

用法

在 Keras 中训练音频分类器

Java中的机器学习包是tensorflow，它加载预先训练的音频分类器模型（.pb格式）。音频分类器模型最初是使用 Python 中的 Keras 实现和训练的。然后将训练好的分类器模型（.h5 格式）转换为 .pb 模型文件，该文件可以直接由 Java 中的 Tensorflow 加载。

音频分类器模型的keras训练可以在README_Training.md中找到

在 Java 中运行音频分类器

下面的示例代码展示了如何使用 cifar 音频分类器来预测音乐流派：

 import com . github . chen0040 . tensorflow . classifiers . models . cifar10 . Cifar10AudioClassifier ;
import com . github . chen0040 . tensorflow . classifiers . utils . ResourceUtils ;
import org . slf4j . Logger ;
import org . slf4j . LoggerFactory ;

import java . io . File ;
import java . io . IOException ;
import java . io . InputStream ;
import java . util . ArrayList ;
import java . util . Collections ;
import java . util . List ;

public class Demo {
    public static void main ( String [] args ) {
        InputStream inputStream = ResourceUtils . getInputStream ( "tf_models/cifar10.pb" );
        Cifar10AudioClassifier classifier = new Cifar10AudioClassifier ();
        classifier . load_model ( inputStream );
        
        List < String > paths = getAudioFiles ();
        
        Collections . shuffle ( paths );
        
        for ( String path : paths ) {
            System . out . println ( "Predicting " + path + " ..." );
            File f = new File ( path );
            String label = classifier . predict_audio ( f );
        
            System . out . println ( "Predicted: " + label );
        }
    }
}

下面的示例代码展示了如何使用 resnet v2 音频分类器来预测音乐流派：

 import com . github . chen0040 . tensorflow . classifiers . resnet_v2 . ResNetV2AudioClassifier ;
import com . github . chen0040 . tensorflow . classifiers . utils . ResourceUtils ;
import org . slf4j . Logger ;
import org . slf4j . LoggerFactory ;

import java . io . File ;
import java . io . IOException ;
import java . io . InputStream ;
import java . util . ArrayList ;
import java . util . Collections ;
import java . util . List ;

public class Demo {
    public static void main ( String [] args ) {
        InputStream inputStream = ResourceUtils . getInputStream ( "tf_models/resnet-v2.pb" );
        ResNetV2AudioClassifier classifier = new ResNetV2AudioClassifier ();
        classifier . load_model ( inputStream );
        
        List < String > paths = getAudioFiles ();
        
        Collections . shuffle ( paths );
        
        for ( String path : paths ) {
            System . out . println ( "Predicting " + path + " ..." );
            File f = new File ( path );
            String label = classifier . predict_audio ( f );
        
            System . out . println ( "Predicted: " + label );
        }
    }
}

Java中从音频中提取特征

下面的示例代码展示了如何使用 cifar 音频分类器将音频文件编码为浮点数组：

 import com . github . chen0040 . tensorflow . classifiers . models . cifar10 . Cifar10AudioClassifier ;
import com . github . chen0040 . tensorflow . classifiers . utils . ResourceUtils ;
import org . slf4j . Logger ;
import org . slf4j . LoggerFactory ;

import java . io . File ;
import java . io . IOException ;
import java . io . InputStream ;
import java . util . ArrayList ;
import java . util . Collections ;
import java . util . List ;

public class Demo {
    public static void main ( String [] args ){
        InputStream inputStream = ResourceUtils . getInputStream ( "tf_models/cifar10.pb" );
        Cifar10AudioClassifier classifier = new Cifar10AudioClassifier ();
        classifier . load_model ( inputStream );
        
        List < String > paths = getAudioFiles ();
        
        Collections . shuffle ( paths );
        
        for ( String path : paths ) {
            System . out . println ( "Encoding " + path + " ..." );
            File f = new File ( path );
            float [] encoded_audio = classifier . encode_audio ( f );
        
            System . out . println ( "Encoded: " + Arrays . toString ( encoded_audio ));
        }
    }
}

下面的示例代码展示了如何使用 resnet v2 音频分类器将音频文件编码为浮点数组：

 import com . github . chen0040 . tensorflow . classifiers . resnet_v2 . ResNetV2AudioClassifier ;
import com . github . chen0040 . tensorflow . classifiers . utils . ResourceUtils ;
import org . slf4j . Logger ;
import org . slf4j . LoggerFactory ;

import java . io . File ;
import java . io . IOException ;
import java . io . InputStream ;
import java . util . ArrayList ;
import java . util . Collections ;
import java . util . List ;

public class Demo {
    public static void main ( String [] args ) {
        InputStream inputStream = ResourceUtils . getInputStream ( "tf_models/resnet-v2.pb" );
        ResNetV2AudioClassifier classifier = new ResNetV2AudioClassifier ();
        classifier . load_model ( inputStream );
        
        List < String > paths = getAudioFiles ();
        
        Collections . shuffle ( paths );
        
        for ( String path : paths ) {
            System . out . println ( "Encoding " + path + " ..." );
            File f = new File ( path );
            float [] encoded_audio = classifier . encode_audio ( f );
        
            System . out . println ( "Encoded: " + Arrays . toString ( encoded_audio ));
        }
    }
}

音频搜索引擎

下面的示例代码展示了如何使用 AudioSearchEngine 类索引和搜索音频文件：

 import com . github . chen0040 . tensorflow . search . models . AudioSearchEngine ;
import com . github . chen0040 . tensorflow . search . models . AudioSearchEntry ;

import java . io . File ;
import java . util . List ;

public class Demo {
    public static void main ( String [] args ){
        AudioSearchEngine searchEngine = new AudioSearchEngine ();
        if (! searchEngine . loadIndexDbIfExists ()) {
            searchEngine . indexAll ( FileUtils . getAudioFiles ());
            searchEngine . saveIndexDb ();
        }
        
        int pageIndex = 0 ;
        int pageSize = 20 ;
        boolean skipPerfectMatch = true ;
        File f = new File ( "mp3_samples/example.mp3" );
        System . out . println ( "querying similar music to " + f . getName ());
        List < AudioSearchEntry > result = searchEngine . query ( f , pageIndex , pageSize , skipPerfectMatch );
        for ( int i = 0 ; i < result . size (); ++ i ){
            System . out . println ( "# " + i + ": " + result . get ( i ). getPath () + " (distSq: " + result . get ( i ). getDistance () + ")" );
        }
    }
}

音乐推荐器

下面的示例代码展示了如何使用 KnnAudioRecommender 类根据用户的音乐历史推荐音乐：

 import com . github . chen0040 . tensorflow . classifiers . utils . FileUtils ;
import com . github . chen0040 . tensorflow . recommenders . models . AudioUserHistory ;
import com . github . chen0040 . tensorflow . recommenders . models . KnnAudioRecommender ;
import com . github . chen0040 . tensorflow . search . models . AudioSearchEntry ;

import java . io . File ;
import java . util . Collections ;
import java . util . List ;

public class Demo {
    public static void main ( String [] args ){
        AudioUserHistory userHistory = new AudioUserHistory ();
        
        List < String > audioFiles = FileUtils . getAudioFilePaths ();
        Collections . shuffle ( audioFiles );
        
        for ( int i = 0 ; i < 40 ; ++ i ){
            String filePath = audioFiles . get ( i );
            userHistory . logAudio ( filePath );
            try {
                Thread . sleep ( 100L );
            } catch ( InterruptedException e ) {
                e . printStackTrace ();
            }
        }
        
        KnnAudioRecommender recommender = new KnnAudioRecommender ();
        if (! recommender . loadIndexDbIfExists ()) {
            recommender . indexAll ( new File ( "music_samples" ). listFiles ( a -> a . getAbsolutePath (). toLowerCase (). endsWith ( ".au" )));
            recommender . saveIndexDb ();
        }
        
        System . out . println ( userHistory . head ( 10 ));
        
        int k = 10 ;
        List < AudioSearchEntry > result = recommender . recommends ( userHistory . getHistory (), k );
        
        for ( int i = 0 ; i < result . size (); ++ i ){
            AudioSearchEntry entry = result . get ( i );
            System . out . println ( "Search Result #" + ( i + 1 ) + ": " + entry . getPath ());
        }
    }
}