实时音频波形图

Question

我正在创建一个应用程序来绘制音频信号（随时间变化的均方根振幅）。我有两个输入音频数据的选项：

1. 选择音频文件
2. 从麦克风录音并实时绘制图表。

我已经完成了选项 1 的实施，但在选项 2 上遇到了麻烦——我最终以比我希望的更简单的方式实施它：

单独的类来处理设置接口以记录麦克风，返回一个字节数组：

import javax.sound.sampled.AudioFormat;
import javax.sound.sampled.AudioSystem;
import javax.sound.sampled.LineUnavailableException;
import javax.sound.sampled.TargetDataLine;

public class MicRecorder {
    private AudioFormat fmt;

    public MicRecorder(AudioFormat fmt) {
        this.fmt = fmt;
    }

    public byte[] start(int durationInSeconds) throws LineUnavailableException {
        // initializing new microphone input line
        TargetDataLine line = AudioSystem.getTargetDataLine(fmt);

        // starts recording from microphone
        line.open(fmt);
        line.start();

        // reads audio data into a byte array
        int numBytes = (int) (fmt.getFrameRate() * fmt.getFrameSize() * durationInSeconds);
        byte[] bytes = new byte[numBytes];
        int numBytesRead = line.read(bytes, 0, numBytes);

        // stops recording and releases resources
        line.stop();
        line.close();

        // returns audio data as a byte array
        return bytes;
    }
}

这是我的控制器的简化版本，前 4 个方法是最重要的：

import com.xsanez.dsp_tool.core.audio.AudioTranslator;
import com.xsanez.dsp_tool.core.audio.MicRecorder;
import static com.xsanez.dsp_tool.core.util.UtilMethods.rms;

import javafx.animation.*;
import javafx.concurrent.Task;
import javafx.fxml.FXML;
import javafx.fxml.Initializable;
import javafx.scene.chart.LineChart;
import javafx.scene.chart.XYChart;
import javafx.scene.control.*;
import javafx.scene.layout.VBox;
import javafx.stage.FileChooser;
import javafx.stage.Stage;
import javafx.util.Duration;
import org.controlsfx.control.ToggleSwitch;

import javax.sound.sampled.*;

import java.io.File;
import java.io.IOException;
import java.util.*;
import java.net.URL;

public class MenuController implements Initializable {

    @FXML private VBox chooseFileBox, recordMicBox;
    @FXML private Button recordButton;
    @FXML private LineChart<Number, Number> visualizer;
    @FXML private ToggleSwitch toggleSwitch;
    @FXML private ProgressBar recordingProgressBar;

    private File selectedFile;
    private double[][] samples;

    @Override
    public void initialize(URL url, ResourceBundle resourceBundle) {
        // switches between methods of inputting sound data
        toggleSwitch.selectedProperty().addListener((observableValue, oldValue, newValue) -> {
            if (newValue) {
                chooseFileBox.setVisible(false);
                recordMicBox.setVisible(true);
            } else {
                recordMicBox.setVisible(false);
                chooseFileBox.setVisible(true);
            }
        });
    }

    @FXML // OPTION 1
    public void selectFile() {
        // creates a new window for user to choose a file
        FileChooser fileChooser = new FileChooser();
        fileChooser.getExtensionFilters().add(
            new FileChooser.ExtensionFilter("Audio Files", "*.wav", "*.au", "*.aiff"));
        selectedFile = fileChooser.showOpenDialog(new Stage());

        if (selectedFile != null) {
            // try-with-resources statement to release system resources acquired
            try (AudioInputStream ais = AudioSystem.getAudioInputStream(selectedFile)) {
                samples = AudioTranslator.decode(
                    ais.readAllBytes(), ais.getFormat(), (int) ais.getFrameLength());
            } catch (UnsupportedAudioFileException | IOException e) {
                // creates a new popup window which shows the error encountered
                Alert alert = new Alert(Alert.AlertType.ERROR);
                alert.setHeaderText(null);
                alert.setContentText(e.getMessage());
                alert.showAndWait();
                return;     // return here to exit method and prevent attempt of graphing audio
            }
            graphAudio();
        }
    }

    @FXML // OPTION 2 - records mic for 10 seconds and graphs it once done
    public void recordMic() {
        animateProgressBar(recordingProgressBar, Duration.seconds(10));

        int sampleRate = 44100, sampleSize = 16, numChannels = 1;
        boolean signed = true, bigEndian = true;

        // Initialization a MicRecorder using obtained AudioFormat
        AudioFormat fmt = new AudioFormat(sampleRate, sampleSize, numChannels, signed, bigEndian);
        MicRecorder recorder = new MicRecorder(fmt);

        // New task to run the recording + decoding processes in the background
        Task<double[][]> task = new Task<>() {
            @Override
            protected double[][] call() throws Exception {
                byte[] bytes = recorder.start(10);

                // obtaining samples from byte array "data"
                int frameSize = sampleSize * numChannels / 8;
                int frameLength = bytes.length / frameSize;
                return AudioTranslator.decode(bytes, fmt, frameLength);
            }
        };

        task.setOnSucceeded(event -> {
            samples = task.getValue();
            graphAudio();
        });

        new Thread(task).start();
    }


    // displays the audio as an amplitude-time graph
    private void graphAudio() {
        visualizer.getData().clear();

        int numChannels = samples.length;   // number of rows in 2D array
        int numSamples = samples[0].length;

        for (int channel = 0; channel < numChannels; channel++) {

            var currentSeries = new XYChart.Series<Number, Number>();
            currentSeries.setName("Channel " + channel);

            int increment = numSamples / 1024;     // arbitrary division by 2^10
            for (int i = 0; i < numSamples; i += increment) {

                double[] buffer = Arrays.copyOfRange(samples[channel], i, i + increment - 1);
                double rmsValue = rms(buffer);

                // adding samples to the XY series
                currentSeries.getData().add(new XYChart.Data<>(i, rmsValue));
                currentSeries.getData().add(new XYChart.Data<>(i, -rmsValue));
            }
            // adding series to the XY chart
            visualizer.getData().add(currentSeries);
        }
    }

    private void animateProgressBar(ProgressBar progressBar, Duration duration) {
        Timeline timeline = new Timeline(
                new KeyFrame(Duration.ZERO, new KeyValue(progressBar.progressProperty(), 0)),
                new KeyFrame(duration, new KeyValue(progressBar.progressProperty(), 1))
        );
        timeline.play();
    }
    
    @FXML
    public void resetRecording() {
        recordingProgressBar.setProgress(0.0);
        visualizer.getData().clear();
        samples = null;
    }
    
    @FXML
    public void clearQueue() {
        visualizer.getData().clear();
        selectedFile = null;
        samples = null;
    }
}

（有关 AudioTranslator 的工作原理，请参阅 https://stackoverflow.com/a/26824664，我不想分享我的实现，但如果有必要，请询问）

对应的FXML文件如下：

<?xml version="1.0" encoding="UTF-8"?>

<?import javafx.geometry.Insets?>
<?import javafx.scene.chart.LineChart?>
<?import javafx.scene.chart.NumberAxis?>
<?import javafx.scene.control.Button?>
<?import javafx.scene.control.Label?>
<?import javafx.scene.control.ProgressBar?>
<?import javafx.scene.layout.HBox?>
<?import javafx.scene.layout.Region?>
<?import javafx.scene.layout.StackPane?>
<?import javafx.scene.layout.VBox?>
<?import org.controlsfx.control.ToggleSwitch?>

<VBox alignment="CENTER" prefHeight="800.0" prefWidth="1280.0" spacing="50.0" xmlns="http://javafx.com/javafx/17"
      xmlns:fx="http://javafx.com/fxml/1" fx:controller="com.xsanez.dsp_tool.ui.MenuController">
    <Region prefHeight="75.0" prefWidth="1280.0"/>
    <HBox alignment="CENTER" prefHeight="300.0" prefWidth="1280.0">
        <Region prefWidth="200.0" HBox.hgrow="SOMETIMES"/>
        <LineChart fx:id="visualizer" createSymbols="false" HBox.hgrow="ALWAYS">
            <xAxis>
                <NumberAxis minorTickVisible="false" tickLabelsVisible="false"/>
            </xAxis>
            <yAxis>
                <NumberAxis minorTickVisible="false" tickLabelsVisible="false"/>
            </yAxis>
        </LineChart>
        <Region prefWidth="200.0" HBox.hgrow="SOMETIMES"/>
    </HBox>
    <HBox alignment="CENTER" prefHeight="250.0" prefWidth="1280.0">
        <Region prefWidth="340.0" HBox.hgrow="NEVER"/>
        <VBox alignment="CENTER" prefHeight="200.0" prefWidth="600.0">
            <HBox alignment="CENTER" prefHeight="40.0" prefWidth="200.0" spacing="20.0">
                <padding>
                    <Insets bottom="10.0" top="10.0"/>
                </padding>
                <Label text="Choose File"/>
                <ToggleSwitch fx:id="toggleSwitch" prefHeight="18.0" prefWidth="32.0"/>
                <Label text="Record Mic"/>
            </HBox>
            <StackPane prefWidth="200.0" VBox.vgrow="ALWAYS">
                <VBox fx:id="chooseFileBox" alignment="CENTER" prefHeight="200.0" prefWidth="100.0" spacing="10.0">
                    <HBox alignment="CENTER" prefHeight="40.0" prefWidth="600.0" spacing="20.0">
                        <Button mnemonicParsing="false" onAction="#selectFile" prefHeight="35.0" text="  Select file  "/>
                        <Button mnemonicParsing="false" onAction="#clearQueue" prefHeight="35.0" text="Clear queue"/>
                    </HBox>
                    <HBox alignment="CENTER" prefWidth="600.0" VBox.vgrow="ALWAYS">
                    </HBox>
                </VBox>
                <VBox fx:id="recordMicBox" alignment="CENTER" prefHeight="200.0" prefWidth="100.0" spacing="10.0" visible="false">
                    <Label text="Record"/>
                    <HBox alignment="TOP_CENTER" prefHeight="63.0" prefWidth="600.0" spacing="20.0">
                        <Button fx:id="recordButton" mnemonicParsing="false" onAction="#recordMic" prefHeight="32.0" prefWidth="80.0" text="Start"/>
                        <Button mnemonicParsing="false" onAction="#resetRecording" prefHeight="32.0" prefWidth="80.0" text="Reset"/>
                    </HBox>
                    <ProgressBar fx:id="recordingProgressBar" prefHeight="18.0" prefWidth="230.0" progress="0.0"/>
                </VBox>
            </StackPane>
        </VBox>
        <Region prefWidth="340.0" HBox.hgrow="NEVER"/>
    </HBox>
    <HBox alignment="CENTER" prefHeight="150.0" prefWidth="1280.0"/>
</VBox>

我有几个问题：

• 我希望它实时绘制图表，例如 - 通过每 0.1 秒左右添加一个新点来更新图表（这个新点应该是 0.1 秒内记录的所有样本的有效值）。
  这需要同时解码 TargetDataLine (mic) 提供的字节。目前，它只会在 10 秒后绘制整个图形。
• 我希望能够随时开始和停止录制，而不是录制一段设定的持续时间（在这种情况下，我将其设置为录制 10 秒）

我尝试使用回调函数和 lambda 来解决这些问题，但我对如何使用它们知之甚少，也不知道我在做什么……

import javax.sound.sampled.*;
import java.util.function.Consumer;

public class MicRecorder {

    private AudioFormat fmt;
    private TargetDataLine line;

    private Thread thread;
    private boolean running;

    public MicRecorder(AudioFormat fmt) throws LineUnavailableException {
        this.fmt = fmt;
        this.line = AudioSystem.getTargetDataLine(fmt);
    }

    public void start(Consumer<byte[]> onData) {
        running = true;

        thread = new Thread(() -> {
            try {
                line.open(fmt);
                line.start();
            } catch (LineUnavailableException e) {
                e.printStackTrace();
            }

            byte[] buffer = new byte[line.getBufferSize()];
            while (running) {
                int numBytesRead = line.read(buffer, 0, line.getBufferSize());
                if (numBytesRead == -1) {
                    break;
                }
                onData.accept(buffer);
            }
            line.stop();
            line.close();
        });
    }

    public void stop() throws InterruptedException {
        running = false;
        thread.join();;
    }
}

MenuController 中的更新方法：

@FXML
public void recordMic() throws LineUnavailableException {
    animateProgressBar(recordingProgressBar, Duration.seconds(10));

    int sampleRate = 44100, sampleSize = 16, numChannels = 1;
    boolean signed = true, bigEndian = true;

    // Initialization a MicRecorder using obtained AudioFormat
    AudioFormat fmt = new AudioFormat(sampleRate, sampleSize, numChannels, signed, bigEndian);
    MicRecorder recorder = new MicRecorder(fmt);

    int frameSize = sampleSize * numChannels / 8;

    // New task to run the recording + decoding processes in the background
    Task<double[][]> task = new Task<>() {
        @Override
        protected double[][] call() throws Exception {
            recorder.start(bytes -> {
                AudioTranslator.decode(bytes, fmt, (bytes.length/frameSize));
            });
        }
    };

    task.setOnSucceeded(event -> {
        samples = task.getValue();
        graphAudio();
    });

    new Thread(task).start();
}

不太确定从这里去哪里，我肯定没有正确使用 start 方法——我也可能在错误的地方解码它？我需要在某处返回一个 double[][] 数组。

另外我认为我可能需要制作两种不同的方法来绘制音频（一种用于文件，一种用于麦克风），但理想情况下我想要一种方法接受样本流并在数据到达时不断更新图形，这可以由选项 1 和 2 使用。不确定是否可能。