From 06ccbd37291a74fa2cab4ba76c2758f85dcd2ffb Mon Sep 17 00:00:00 2001 From: ChomeNS <95471003+ChomeNS@users.noreply.github.com> Date: Fri, 6 Oct 2023 17:11:08 +0700 Subject: [PATCH] one step from getting it working (BUT that step will never be done) --- build.gradle | 6 + .../voiceChat/mic/JavaOpusDecoder.java | 70 ++++++++ .../voiceChat/mic/JavaOpusEncoder.java | 78 +++++++++ .../voiceChat/mic/JavaOpusEncoder2.java | 72 ++++++++ .../chomens_bot/voiceChat/mic/MicThread.java | 155 ++++++++++++++++++ .../voiceChat/mic/OpusManager.java | 26 +++ 6 files changed, 407 insertions(+) create mode 100644 src/main/java/land/chipmunk/chayapak/chomens_bot/voiceChat/mic/JavaOpusDecoder.java create mode 100644 src/main/java/land/chipmunk/chayapak/chomens_bot/voiceChat/mic/JavaOpusEncoder.java create mode 100644 src/main/java/land/chipmunk/chayapak/chomens_bot/voiceChat/mic/JavaOpusEncoder2.java create mode 100644 src/main/java/land/chipmunk/chayapak/chomens_bot/voiceChat/mic/MicThread.java create mode 100644 src/main/java/land/chipmunk/chayapak/chomens_bot/voiceChat/mic/OpusManager.java diff --git a/build.gradle b/build.gradle index 9796df3..b2fa8d3 100644 --- a/build.gradle +++ b/build.gradle @@ -30,6 +30,10 @@ repositories { maven { url = uri('https://repo.maven.apache.org/maven2/') } + + maven { + url = uri('https://maven.maxhenkel.de/repository/public') + } } dependencies { @@ -46,6 +50,8 @@ dependencies { implementation 'net.kyori:adventure-text-serializer-legacy:4.13.1' implementation 'org.apache.logging.log4j:log4j-slf4j-impl:2.20.0' implementation 'io.socket:socket.io-client:2.1.0' + implementation 'de.maxhenkel.opus4j:opus4j:2.0.2' + implementation 'org.concentus:Concentus:1.0-SNAPSHOT' } jar { diff --git a/src/main/java/land/chipmunk/chayapak/chomens_bot/voiceChat/mic/JavaOpusDecoder.java b/src/main/java/land/chipmunk/chayapak/chomens_bot/voiceChat/mic/JavaOpusDecoder.java new file mode 100644 index 0000000..938d76b --- /dev/null +++ b/src/main/java/land/chipmunk/chayapak/chomens_bot/voiceChat/mic/JavaOpusDecoder.java @@ -0,0 +1,70 @@ +package land.chipmunk.chayapak.chomens_bot.voiceChat.mic; + +import org.concentus.OpusDecoder; +import org.concentus.OpusException; + +public class JavaOpusDecoder { + protected OpusDecoder opusDecoder; + protected short[] buffer; + protected int sampleRate; + protected int frameSize; + protected int maxPayloadSize; + + public JavaOpusDecoder (int sampleRate, int frameSize, int maxPayloadSize) { + this.sampleRate = sampleRate; + this.frameSize = frameSize; + this.maxPayloadSize = maxPayloadSize; + this.buffer = new short[4096]; + open(); + } + + private void open() { + if (opusDecoder != null) { + return; + } + try { + opusDecoder = new OpusDecoder(sampleRate, 1); + } catch (OpusException e) { + throw new IllegalStateException("Opus decoder error " + e.getMessage()); + } + } + + public short[] decode(byte[] data) { + if (isClosed()) { + throw new IllegalStateException("Decoder is closed"); + } + int result; + + try { + if (data == null || data.length == 0) { + result = opusDecoder.decode(null, 0, 0, buffer, 0, frameSize, false); + } else { + result = opusDecoder.decode(data, 0, data.length, buffer, 0, frameSize, false); + } + } catch (Exception e) { + throw new RuntimeException("Failed to decode audio data: " + e.getMessage()); + } + + short[] audio = new short[result]; + System.arraycopy(buffer, 0, audio, 0, result); + return audio; + } + + public boolean isClosed() { + return opusDecoder == null; + } + + public void close() { + if (opusDecoder == null) { + return; + } + opusDecoder = null; + } + + public void resetState() { + if (isClosed()) { + throw new IllegalStateException("Decoder is closed"); + } + opusDecoder.resetState(); + } +} diff --git a/src/main/java/land/chipmunk/chayapak/chomens_bot/voiceChat/mic/JavaOpusEncoder.java b/src/main/java/land/chipmunk/chayapak/chomens_bot/voiceChat/mic/JavaOpusEncoder.java new file mode 100644 index 0000000..c54dfa0 --- /dev/null +++ b/src/main/java/land/chipmunk/chayapak/chomens_bot/voiceChat/mic/JavaOpusEncoder.java @@ -0,0 +1,78 @@ +package land.chipmunk.chayapak.chomens_bot.voiceChat.mic; + +import org.concentus.OpusApplication; +import org.concentus.OpusEncoder; + +public class JavaOpusEncoder { + public OpusEncoder opusEncoder; + public byte[] buffer; + public int sampleRate; + public int frameSize; + public de.maxhenkel.opus4j.OpusEncoder.Application application; + + public JavaOpusEncoder(int sampleRate, int frameSize, int maxPayloadSize, de.maxhenkel.opus4j.OpusEncoder.Application application) { + this.sampleRate = sampleRate; + this.frameSize = frameSize; + this.application = application; + this.buffer = new byte[maxPayloadSize]; + open(); + } + + private void open() { + if (opusEncoder != null) { + return; + } + try { + opusEncoder = new OpusEncoder(sampleRate, 1, getApplication(application)); + } catch (Exception e) { + throw new IllegalStateException("Failed to create Opus encoder", e); + } + } + + public byte[] encode(short[] rawAudio) { + if (isClosed()) { + throw new IllegalStateException("Encoder is closed"); + } + + int result; + try { + result = opusEncoder.encode(rawAudio, 0, frameSize, buffer, 0, buffer.length); + } catch (Exception e) { + throw new RuntimeException("Failed to encode audio", e); + } + + if (result < 0) { + throw new RuntimeException("Failed to encode audio data"); + } + + byte[] audio = new byte[result]; + System.arraycopy(buffer, 0, audio, 0, result); + return audio; + } + + public void resetState() { + if (isClosed()) { + throw new IllegalStateException("Encoder is closed"); + } + opusEncoder.resetState(); + } + + public boolean isClosed() { + return opusEncoder == null; + } + + public void close() { + if (isClosed()) { + return; + } + opusEncoder = null; + } + + public static OpusApplication getApplication(de.maxhenkel.opus4j.OpusEncoder.Application application) { + return switch (application) { + default -> OpusApplication.OPUS_APPLICATION_VOIP; + case AUDIO -> OpusApplication.OPUS_APPLICATION_AUDIO; + case LOW_DELAY -> OpusApplication.OPUS_APPLICATION_RESTRICTED_LOWDELAY; + }; + } +} diff --git a/src/main/java/land/chipmunk/chayapak/chomens_bot/voiceChat/mic/JavaOpusEncoder2.java b/src/main/java/land/chipmunk/chayapak/chomens_bot/voiceChat/mic/JavaOpusEncoder2.java new file mode 100644 index 0000000..f907885 --- /dev/null +++ b/src/main/java/land/chipmunk/chayapak/chomens_bot/voiceChat/mic/JavaOpusEncoder2.java @@ -0,0 +1,72 @@ +package land.chipmunk.chayapak.chomens_bot.voiceChat.mic; + +import org.concentus.OpusApplication; +import org.concentus.OpusEncoder; + +public class JavaOpusEncoder2 { + public OpusEncoder opusEncoder; + public byte[] buffer; + public int sampleRate; + public int frameSize; + public int maxPayloadSize; + public OpusApplication application; + + public JavaOpusEncoder2 (int sampleRate, int frameSize, int maxPayloadSize, OpusApplication application) { + this.sampleRate = sampleRate; + this.frameSize = frameSize; + this.maxPayloadSize = maxPayloadSize; + this.application = application; + this.buffer = new byte[maxPayloadSize]; + open(); + } + + private void open() { + if (opusEncoder != null) { + return; + } + try { + opusEncoder = new OpusEncoder(sampleRate, 1, application); + } catch (Exception e) { + throw new IllegalStateException("Opus encoder error " + e.getMessage()); + } + } + + public byte[] encode(short[] rawAudio) { + if (isClosed()) { + throw new IllegalStateException("Encoder is closed"); + } + + int result; + try { + result = opusEncoder.encode(rawAudio, 0, frameSize, buffer, 0, buffer.length); + } catch (Exception e) { + throw new RuntimeException("Failed to encode audio data: " + e.getMessage()); + } + + if (result < 0) { + throw new RuntimeException("Failed to encode audio data"); + } + + byte[] audio = new byte[result]; + System.arraycopy(buffer, 0, audio, 0, result); + return audio; + } + + public void resetState() { + if (isClosed()) { + throw new IllegalStateException("Encoder is closed"); + } + opusEncoder.resetState(); + } + + public boolean isClosed() { + return opusEncoder == null; + } + + public void close() { + if (isClosed()) { + return; + } + opusEncoder = null; + } +} diff --git a/src/main/java/land/chipmunk/chayapak/chomens_bot/voiceChat/mic/MicThread.java b/src/main/java/land/chipmunk/chayapak/chomens_bot/voiceChat/mic/MicThread.java new file mode 100644 index 0000000..205fd80 --- /dev/null +++ b/src/main/java/land/chipmunk/chayapak/chomens_bot/voiceChat/mic/MicThread.java @@ -0,0 +1,155 @@ +package land.chipmunk.chayapak.chomens_bot.voiceChat.mic; + +public class MicThread extends Thread { + // this will probably never be finished +// private Microphone mic; +// private boolean running; +// private boolean microphoneLocked; +// private boolean wasWhispering; +// private final JavaOpusEncoder2 encoder; +// +// public MicThread() { +// this.running = true; +// this.encoder = OpusManager.createEncoder(); +// +// setDaemon(true); +// setName("Simple Voice Chat Microphone Thread"); +// } +// +// @Override +// public void run() { +// Microphone mic = getMic(); +// if (mic == null) { +// return; +// } +// +// while (running) { +// short[] audio = pollMic(); +// if (audio == null) { +// continue; +// } +// +// voice(audio); +// } +// } +// +// public short[] pollMic() { +// Microphone mic = getMic(); +// if (mic == null) { +// throw new IllegalStateException("No microphone available"); +// } +// if (!mic.isStarted()) { +// mic.start(); +// } +// +// if (mic.available() < SoundManager.FRAME_SIZE) { +// Utils.sleep(5); +// return null; +// } +// short[] buff = mic.read(); +// volumeManager.adjustVolumeMono(buff, VoicechatClient.CLIENT_CONFIG.microphoneAmplification.get().floatValue()); +// return denoiseIfEnabled(buff); +// } +// +// private Microphone getMic() { +// if (!running) { +// return null; +// } +// if (mic == null) { +// try { +// mic = MicrophoneManager.createMicrophone(); +// } catch (MicrophoneException e) { +// running = false; +// return null; +// } +// } +// return mic; +// } +// +// private volatile boolean activating; +// private volatile int deactivationDelay; +// private volatile short[] lastBuff; +// +// private void voice(short[] audio) { +// sendAudioPacket(audio); +// lastBuff = audio; +// } +// +// private void flush() { +// sendStopPacket(); +// if (!encoder.isClosed()) { +// encoder.resetState(); +// } +// } +// +// public boolean isTalking() { +// return !microphoneLocked && (activating || wasPTT); +// } +// +// public boolean isWhispering() { +// return isTalking() && wasWhispering; +// } +// +// public void close() { +// if (!running) { +// return; +// } +// running = false; +// +// if (Thread.currentThread() != this) { +// try { +// join(100); +// } catch (InterruptedException e) { +// e.printStackTrace(); +// } +// } +// +// mic.close(); +// encoder.close(); +// flush(); +// } +// +// private final AtomicLong sequenceNumber = new AtomicLong(); +// private volatile boolean stopPacketSent = true; +// +// private void sendAudioPacket(short[] data) { +// short[] audio = PluginManager.instance().onClientSound(data, whispering); +// if (audio == null) { +// return; +// } +// +// try { +// if (connection != null && connection.isInitialized()) { +// byte[] encoded = encoder.encode(audio); +// connection.sendToServer(new NetworkMessage(new MicPacket(encoded, whispering, sequenceNumber.getAndIncrement()))); +// stopPacketSent = false; +// } +// } catch (Exception e) { +// e.printStackTrace(); +// } +// try { +// if (client != null && client.getRecorder() != null) { +// client.getRecorder().appendChunk(Minecraft.getInstance().getUser().getGameProfile().getId(), System.currentTimeMillis(), PositionalAudioUtils.convertToStereo(audio)); +// } +// } catch (IOException e) { +// Voicechat.LOGGER.error("Failed to record audio", e); +// client.setRecording(false); +// } +// } +// +// private void sendStopPacket() { +// if (stopPacketSent) { +// return; +// } +// +// if (connection == null || !connection.isInitialized()) { +// return; +// } +// try { +// connection.sendToServer(new NetworkMessage(new MicPacket(new byte[0], false, sequenceNumber.getAndIncrement()))); +// stopPacketSent = true; +// } catch (Exception e) { +// e.printStackTrace(); +// } +// } +} diff --git a/src/main/java/land/chipmunk/chayapak/chomens_bot/voiceChat/mic/OpusManager.java b/src/main/java/land/chipmunk/chayapak/chomens_bot/voiceChat/mic/OpusManager.java new file mode 100644 index 0000000..c8423ef --- /dev/null +++ b/src/main/java/land/chipmunk/chayapak/chomens_bot/voiceChat/mic/OpusManager.java @@ -0,0 +1,26 @@ +package land.chipmunk.chayapak.chomens_bot.voiceChat.mic; + +import org.concentus.OpusApplication; + +public class OpusManager { + public static final int SAMPLE_RATE = 48000; + public static final int FRAME_SIZE = (SAMPLE_RATE / 1000) * 20; + + public static JavaOpusEncoder2 createEncoder(int sampleRate, int frameSize, int maxPayloadSize, OpusApplication application) { + return new JavaOpusEncoder2(sampleRate, frameSize, maxPayloadSize, application); + } + + public static JavaOpusEncoder2 createEncoder() { + OpusApplication application = OpusApplication.OPUS_APPLICATION_AUDIO; + + return createEncoder(SAMPLE_RATE, FRAME_SIZE, 1024, application); + } + + public static JavaOpusDecoder createDecoder(int sampleRate, int frameSize, int maxPayloadSize) { + return new JavaOpusDecoder(sampleRate, frameSize, maxPayloadSize); + } + + public static JavaOpusDecoder createDecoder() { + return createDecoder(SAMPLE_RATE, FRAME_SIZE, 1024); + } +}