From c28ae23774f757ca0f348710099076c6f1e3769f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Przemys=C5=82aw=20Pluta?= Date: Fri, 20 Mar 2020 15:33:07 +0100 Subject: [PATCH] Create working scaffolding for sound synthesizing module: smnp.audio.synth --- modules/synth/build.gradle | 3 + modules/synth/gradle.properties | 7 ++ .../main/kotlin/io/smnp/ext/SynthModule.kt | 20 ++++++ .../io/smnp/ext/function/SynthFunction.kt | 20 ++++++ .../io/smnp/ext/function/WaveFunction.kt | 29 ++++++++ .../kotlin/io/smnp/ext/synth/AdsrEnvelope.kt | 26 +++++++ .../io/smnp/ext/synth/ConstantEnvelope.kt | 7 ++ .../main/kotlin/io/smnp/ext/synth/Envelope.kt | 39 ++++++++++ .../kotlin/io/smnp/ext/synth/Synthesizer.kt | 27 +++++++ .../src/main/kotlin/io/smnp/ext/synth/Wave.kt | 71 +++++++++++++++++++ .../kotlin/io/smnp/ext/synth/WaveCompiler.kt | 52 ++++++++++++++ settings.gradle | 1 + 12 files changed, 302 insertions(+) create mode 100644 modules/synth/build.gradle create mode 100644 modules/synth/gradle.properties create mode 100644 modules/synth/src/main/kotlin/io/smnp/ext/SynthModule.kt create mode 100644 modules/synth/src/main/kotlin/io/smnp/ext/function/SynthFunction.kt create mode 100644 modules/synth/src/main/kotlin/io/smnp/ext/function/WaveFunction.kt create mode 100644 modules/synth/src/main/kotlin/io/smnp/ext/synth/AdsrEnvelope.kt create mode 100644 modules/synth/src/main/kotlin/io/smnp/ext/synth/ConstantEnvelope.kt create mode 100644 modules/synth/src/main/kotlin/io/smnp/ext/synth/Envelope.kt create mode 100644 modules/synth/src/main/kotlin/io/smnp/ext/synth/Synthesizer.kt create mode 100644 modules/synth/src/main/kotlin/io/smnp/ext/synth/Wave.kt create mode 100644 modules/synth/src/main/kotlin/io/smnp/ext/synth/WaveCompiler.kt diff --git a/modules/synth/build.gradle b/modules/synth/build.gradle new file mode 100644 index 0000000..728cd22 --- /dev/null +++ b/modules/synth/build.gradle @@ -0,0 +1,3 @@ +dependencies { + implementation group: 'org.knowm.xchart', name: 'xchart', version: '3.6.2' +} \ No newline at end of file diff --git a/modules/synth/gradle.properties b/modules/synth/gradle.properties new file mode 100644 index 0000000..79998bc --- /dev/null +++ b/modules/synth/gradle.properties @@ -0,0 +1,7 @@ +version=0.0.1 + +pluginVersion=0.1 +pluginId=smnp.audio.synth +pluginClass= +pluginProvider=Bartłomiej Pluta +pluginDependencies= \ No newline at end of file diff --git a/modules/synth/src/main/kotlin/io/smnp/ext/SynthModule.kt b/modules/synth/src/main/kotlin/io/smnp/ext/SynthModule.kt new file mode 100644 index 0000000..6183a9a --- /dev/null +++ b/modules/synth/src/main/kotlin/io/smnp/ext/SynthModule.kt @@ -0,0 +1,20 @@ +package io.smnp.ext + +import io.smnp.environment.Environment +import io.smnp.ext.function.SynthFunction +import io.smnp.ext.function.WaveFunction +import io.smnp.ext.synth.Synthesizer +import org.pf4j.Extension + +@Extension +class SynthModule : NativeModuleProvider("smnp.audio.synth") { + override fun functions() = listOf(WaveFunction(), SynthFunction()) + + override fun onModuleLoad(environment: Environment) { + Synthesizer.init() + } + + override fun beforeModuleDisposal(environment: Environment) { + Synthesizer.dispose() + } +} \ No newline at end of file diff --git a/modules/synth/src/main/kotlin/io/smnp/ext/function/SynthFunction.kt b/modules/synth/src/main/kotlin/io/smnp/ext/function/SynthFunction.kt new file mode 100644 index 0000000..3445959 --- /dev/null +++ b/modules/synth/src/main/kotlin/io/smnp/ext/function/SynthFunction.kt @@ -0,0 +1,20 @@ +package io.smnp.ext.function + +import io.smnp.callable.function.Function +import io.smnp.callable.function.FunctionDefinitionTool +import io.smnp.callable.signature.Signature.Companion.simple +import io.smnp.ext.synth.Synthesizer +import io.smnp.ext.synth.Wave +import io.smnp.type.enumeration.DataType.INT +import io.smnp.type.matcher.Matcher.Companion.listOf +import io.smnp.type.model.Value + +class SynthFunction : Function("synth") { + override fun define(new: FunctionDefinitionTool) { + new function simple(listOf(INT)) body { _, (wave) -> + val bytes = (wave.value as List).map { (it.value as Int).toByte() }.toByteArray() + Synthesizer.synth(Wave(bytes)) + Value.void() + } + } +} \ No newline at end of file diff --git a/modules/synth/src/main/kotlin/io/smnp/ext/function/WaveFunction.kt b/modules/synth/src/main/kotlin/io/smnp/ext/function/WaveFunction.kt new file mode 100644 index 0000000..285898e --- /dev/null +++ b/modules/synth/src/main/kotlin/io/smnp/ext/function/WaveFunction.kt @@ -0,0 +1,29 @@ +package io.smnp.ext.function + +import io.smnp.callable.function.Function +import io.smnp.callable.function.FunctionDefinitionTool +import io.smnp.callable.signature.Signature +import io.smnp.ext.synth.AdsrEnvelope +import io.smnp.ext.synth.Synthesizer +import io.smnp.ext.synth.WaveCompiler +import io.smnp.type.enumeration.DataType +import io.smnp.type.matcher.Matcher +import io.smnp.type.model.Value + +class WaveFunction : Function("wave") { + override fun define(new: FunctionDefinitionTool) { + new function Signature.vararg(Matcher.listOf(DataType.NOTE, DataType.INT, DataType.STRING)) body { _, (vararg) -> + val compiler = WaveCompiler( + listOf(1.0, 0.5), + AdsrEnvelope(0.1, 0.3, 0.7, 0.8), + 120, + 440.0, + Synthesizer.SAMPLING_RATE + ) + + val wave = compiler.compileLines(vararg.unwrapCollections() as List>) + + Value.list(wave.bytes.map { Value.int(it.toInt()) }.toList()) + } + } +} \ No newline at end of file diff --git a/modules/synth/src/main/kotlin/io/smnp/ext/synth/AdsrEnvelope.kt b/modules/synth/src/main/kotlin/io/smnp/ext/synth/AdsrEnvelope.kt new file mode 100644 index 0000000..b42c23e --- /dev/null +++ b/modules/synth/src/main/kotlin/io/smnp/ext/synth/AdsrEnvelope.kt @@ -0,0 +1,26 @@ +package io.smnp.ext.synth + +class AdsrEnvelope(private val p1: Double, private val p2: Double, private val p3: Double, private val s: Double) : Envelope() { + + override fun eval(x: Double, length: Int): Double { + val p1x = length * p1 + val p2x = length * p2 + val p3x = length * p3 + val f1 = quad(0.0 to 0.0, p1x to 1.0, 2.0 * p1x to 0.0) + val f2 = quad(p1x to 1.0, p2x to s, p1x + 2.0 * (p2x - p1x) to 1.0) + val f3 = quad(p3x to s, length.toDouble() to 0.0, p3x + 2.0 * (length - p3x) to s) + + return if (x >= 0 && x < p1x) { + f1(x) + } else if (x >= p1x && x < p2x) { + f2(x) + } else if (x >= p2x && x < p3x) { + s + } else { + f3(x) + } + + } + + override fun name() = "ADSR" +} \ No newline at end of file diff --git a/modules/synth/src/main/kotlin/io/smnp/ext/synth/ConstantEnvelope.kt b/modules/synth/src/main/kotlin/io/smnp/ext/synth/ConstantEnvelope.kt new file mode 100644 index 0000000..e49252a --- /dev/null +++ b/modules/synth/src/main/kotlin/io/smnp/ext/synth/ConstantEnvelope.kt @@ -0,0 +1,7 @@ +package io.smnp.ext.synth + +class ConstantEnvelope : Envelope() { + override fun name() = "Constant" + + override fun eval(x: Double, length: Int) = 1.0 +} \ No newline at end of file diff --git a/modules/synth/src/main/kotlin/io/smnp/ext/synth/Envelope.kt b/modules/synth/src/main/kotlin/io/smnp/ext/synth/Envelope.kt new file mode 100644 index 0000000..133793f --- /dev/null +++ b/modules/synth/src/main/kotlin/io/smnp/ext/synth/Envelope.kt @@ -0,0 +1,39 @@ +package io.smnp.ext.synth + +import org.knowm.xchart.QuickChart +import org.knowm.xchart.SwingWrapper + +abstract class Envelope() { + abstract fun eval(x: Double, length: Int): Double + protected abstract fun name(): String + + fun plot() { + val max = 100 + val x = IntRange(0, max).map { it.toDouble() } + val chart = QuickChart.getChart( + name(), + "x", + "y", + "y(x)", + x, + x.map { eval(it, max) } + ) + + SwingWrapper(chart).displayChart() + } + + fun apply(wave: Wave): Wave { + return Wave(wave.bytes.mapIndexed { index, byte -> + (byte * eval( + index.toDouble(), + wave.bytes.size + )).toByte() + }.toByteArray()) + } + + protected fun quad(a: Pair, b: Pair, c: Pair): (Double) -> Double { + return { x -> + (a.second * ((x - b.first) / (a.first - b.first)) * ((x - c.first) / (a.first - c.first))) + (b.second * ((x - a.first) / (b.first - a.first)) * ((x - c.first) / (b.first - c.first))) + (c.second * ((x - a.first) / (c.first - a.first)) * ((x - b.first) / (c.first - b.first))) + } + } +} \ No newline at end of file diff --git a/modules/synth/src/main/kotlin/io/smnp/ext/synth/Synthesizer.kt b/modules/synth/src/main/kotlin/io/smnp/ext/synth/Synthesizer.kt new file mode 100644 index 0000000..0cf0901 --- /dev/null +++ b/modules/synth/src/main/kotlin/io/smnp/ext/synth/Synthesizer.kt @@ -0,0 +1,27 @@ +package io.smnp.ext.synth + +import javax.sound.sampled.AudioFormat +import javax.sound.sampled.AudioSystem +import javax.sound.sampled.SourceDataLine + + +object Synthesizer { + const val SAMPLING_RATE = 44100.0 // [Hz] + private val format = AudioFormat(SAMPLING_RATE.toFloat(), 8, 1, true, false) + private var line: SourceDataLine = AudioSystem.getSourceDataLine(format) + + fun synth(wave: Wave) { + line.write(wave.bytes, 0, wave.bytes.size) + } + + fun init() { + line.open(format) + line.start() + } + + fun dispose() { + line.drain() + line.stop() + line.close() + } +} \ No newline at end of file diff --git a/modules/synth/src/main/kotlin/io/smnp/ext/synth/Wave.kt b/modules/synth/src/main/kotlin/io/smnp/ext/synth/Wave.kt new file mode 100644 index 0000000..4a414b0 --- /dev/null +++ b/modules/synth/src/main/kotlin/io/smnp/ext/synth/Wave.kt @@ -0,0 +1,71 @@ +package io.smnp.ext.synth + +import org.knowm.xchart.QuickChart +import org.knowm.xchart.SwingWrapper +import java.io.ByteArrayOutputStream +import kotlin.math.roundToInt +import kotlin.math.sin + +class Wave(val bytes: ByteArray) { + + fun plot() { + val chart = QuickChart.getChart( + "Sine wave", + "X", + "Y", + "y(x)", + IntRange(0, bytes.size - 1).map { it.toDouble() }.toList(), + bytes.toList() + ) + + SwingWrapper(chart).displayChart() + } + + operator fun plus(wave: Wave): Wave { + val stream = ByteArrayOutputStream() + stream.writeBytes(bytes) + stream.writeBytes(wave.bytes) + return Wave(stream.toByteArray()) + } + + operator fun times(value: Double): Wave { + return Wave(bytes.map { (it * value).toByte() }.toByteArray()) + } + + fun extend(bytes: ByteArray): Wave { + return this + Wave(bytes) + } + + companion object { + val EMPTY = Wave(ByteArray(0)) + + fun merge(vararg waves: Wave): Wave { + val longestWaveSize = waves.maxBy { it.bytes.size }?.let { it.bytes.size } + ?: throw RuntimeException("Empty waves are not supported") + val adjustedWaves = waves.map { it.extend(ByteArray(longestWaveSize - it.bytes.size)) } + + val signal = IntRange(0, longestWaveSize-1).map { index -> + adjustedWaves.map { it.bytes[index].toDouble() }.sum() + } + + val maxValue = Byte.MAX_VALUE.toDouble() / (signal.max() ?: 1.0) + + return Wave(signal.map { (it * maxValue).toByte() }.toByteArray()) + } + + fun sine(frequency: Double, duration: Double, samplingRate: Double): Wave { + val length = (samplingRate * duration).roundToInt() + val buffer = ByteArray(length) + + buffer.forEachIndexed { i, _ -> + buffer[i] = (Byte.MAX_VALUE * sin(i / samplingRate * 2.0 * Math.PI * frequency)).toByte() + } + + return Wave(buffer) + } + + fun zeros(duration: Double, samplingRate: Double): Wave { + return Wave(ByteArray((samplingRate * duration).roundToInt())) + } + } +} \ No newline at end of file diff --git a/modules/synth/src/main/kotlin/io/smnp/ext/synth/WaveCompiler.kt b/modules/synth/src/main/kotlin/io/smnp/ext/synth/WaveCompiler.kt new file mode 100644 index 0000000..0a363d3 --- /dev/null +++ b/modules/synth/src/main/kotlin/io/smnp/ext/synth/WaveCompiler.kt @@ -0,0 +1,52 @@ +package io.smnp.ext.synth + +import io.smnp.data.entity.Note +import io.smnp.data.enumeration.Pitch +import io.smnp.error.CustomException +import io.smnp.math.Fraction +import io.smnp.type.enumeration.DataType +import io.smnp.type.model.Value +import kotlin.math.pow + +class WaveCompiler( + private val overtones: List, + private val envelope: Envelope, + private val bpm: Int, + tuning: Double, + private val samplingRate: Double +) { + private val semitone = 2.0.pow(1.0/12.0) + private val tuningTable = Pitch.values().mapIndexed { index, pitch -> pitch to tuning/semitone.pow(57-index) }.toMap() + + fun compileLines(lines: List>): Wave { + return Wave.merge(*lines.map { compileLine(it) }.toTypedArray()) + } + + fun compileLine(line: List): Wave { + return line.fold(Wave.EMPTY) { acc, value -> + acc + when(value.type) { + DataType.NOTE -> compileNote(value.value as Note) + DataType.INT -> compileRest(value.value as Int) + DataType.STRING -> Wave.EMPTY + else -> throw CustomException("Invalid data type: '${value.typeName}") + } + } + } + + private fun compileNote(note: Note) = sound((tuningTable[note.pitch] ?: error("")) * 2.0.pow(note.octave), duration(note.duration)) + + private fun duration(duration: Fraction) = 60.0 * 4.0 * duration.decimal / bpm + + private fun compileRest(rest: Int) = Wave.zeros(duration(Fraction(1, rest)), samplingRate) + + private fun sound( + frequency: Double, + duration: Double + ): Wave { + val wave = Wave.merge(*overtones.mapIndexed { overtone, ratio -> + Wave.sine(frequency * (overtone + 1), duration, samplingRate) * ratio + }.toTypedArray()) + + return envelope.apply(wave) + } +} \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index c351ae4..4adb5f7 100644 --- a/settings.gradle +++ b/settings.gradle @@ -10,6 +10,7 @@ include 'modules:io' include 'modules:collection' include 'modules:text' include 'modules:midi' +include 'modules:synth' include 'modules:music' include 'modules:math' include 'modules:mic'