Browse Source

added sounds, but some issues

feature/add_sound
Domagoj Zecevic 1 month ago
parent
commit
31115d1bdd
  1. 4
      app/src/main/AndroidManifest.xml
  2. 100
      app/src/main/java/com/dzecevic/babymonitor/AudioRecorder.kt
  3. 7
      app/src/main/java/com/dzecevic/babymonitor/AudioStore.kt
  4. 7
      app/src/main/java/com/dzecevic/babymonitor/FrameStoreAudio.kt
  5. 186
      app/src/main/java/com/dzecevic/babymonitor/MainActivity.kt
  6. 147
      app/src/main/java/com/dzecevic/babymonitor/MjpegServer.kt
  7. 53
      app/src/main/java/com/dzecevic/babymonitor/StreamService.kt

4
app/src/main/AndroidManifest.xml

@ -9,6 +9,8 @@
<uses-permission android:name="android.permission.WAKE_LOCK" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_CAMERA" />
<uses-permission android:name="android.permission.RECORD_AUDIO"/>
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE"/>
<uses-feature android:name="android.hardware.camera.any" />
@ -36,7 +38,7 @@
<service
android:name=".StreamService"
android:exported="false"
android:foregroundServiceType="camera" />
android:foregroundServiceType="camera|microphone" />
</application>
</manifest>

100
app/src/main/java/com/dzecevic/babymonitor/AudioRecorder.kt

@ -0,0 +1,100 @@
package com.dzecevic.babymonitor
import android.media.AudioFormat
import android.media.AudioRecord
import android.media.MediaRecorder
import android.media.audiofx.AcousticEchoCanceler
import android.media.audiofx.AutomaticGainControl
import android.media.audiofx.NoiseSuppressor
import android.util.Log
import java.util.concurrent.atomic.AtomicBoolean
class AudioRecorder(
private val sampleRate: Int = 16000
) {
private var record: AudioRecord? = null
private val running = AtomicBoolean(false)
private var thread: Thread? = null
fun start() {
if (running.getAndSet(true)) return
val minBuf = AudioRecord.getMinBufferSize(
sampleRate,
AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT
)
// Target ~40ms buffer (low latency but stable)
val bufSize = (minBuf * 2).coerceAtLeast(sampleRate / 25 * 2)
record = AudioRecord(
MediaRecorder.AudioSource.VOICE_COMMUNICATION, // MUCH better than MIC
sampleRate,
AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT,
bufSize
)
val r = record ?: return
if (r.state != AudioRecord.STATE_INITIALIZED) {
Log.e("AudioRecorder", "AudioRecord not initialized")
running.set(false)
return
}
r.startRecording()
// Disable latency-causing DSP if possible
try {
NoiseSuppressor.create(r.audioSessionId)?.enabled = false
AutomaticGainControl.create(r.audioSessionId)?.enabled = false
AcousticEchoCanceler.create(r.audioSessionId)?.enabled = false
} catch (_: Exception) {}
thread = Thread {
val buf = ByteArray(1024) // small chunk = low latency
while (running.get()) {
val n = r.read(buf, 0, buf.size)
if (n > 0) {
// Optional debug level meter
/*
var max = 0
var i = 0
while (i + 1 < n) {
val lo = buf[i].toInt() and 0xff
val hi = buf[i + 1].toInt()
val sample = (hi shl 8) or lo
max = max.coerceAtLeast(kotlin.math.abs(sample))
i += 2
}
Log.d("AUDIO", "level=$max")
*/
FrameStoreAudio.latestPcm.set(buf.copyOf(n))
}
}
}.apply {
isDaemon = true
start()
}
}
fun stop() {
running.set(false)
try { thread?.join(300) } catch (_: Exception) {}
thread = null
try {
record?.stop()
record?.release()
} catch (_: Exception) {}
record = null
}
}

7
app/src/main/java/com/dzecevic/babymonitor/AudioStore.kt

@ -0,0 +1,7 @@
package com.dzecevic.babymonitor
import java.util.concurrent.LinkedBlockingQueue
object AudioStore {
val audioQueue = LinkedBlockingQueue<ByteArray>(50)
}

7
app/src/main/java/com/dzecevic/babymonitor/FrameStoreAudio.kt

@ -0,0 +1,7 @@
package com.dzecevic.babymonitor
import java.util.concurrent.atomic.AtomicReference
object FrameStoreAudio {
val latestPcm = AtomicReference<ByteArray?>(null)
}

186
app/src/main/java/com/dzecevic/babymonitor/MainActivity.kt

@ -2,6 +2,7 @@ package com.dzecevic.babymonitor
import android.Manifest
import android.content.Context
import android.content.Intent
import android.content.pm.PackageManager
import android.net.ConnectivityManager
import android.net.NetworkCapabilities
@ -19,88 +20,67 @@ import androidx.compose.foundation.layout.*
import androidx.compose.material3.Button
import androidx.compose.material3.Text
import androidx.compose.runtime.*
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.unit.dp
import androidx.compose.ui.viewinterop.AndroidView
import androidx.core.content.ContextCompat
import androidx.activity.compose.rememberLauncherForActivityResult
import java.net.Inet4Address
import java.net.NetworkInterface
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.atomic.AtomicReference
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.getValue
import androidx.compose.runtime.setValue
import androidx.compose.foundation.layout.statusBarsPadding
import android.content.Intent
import com.dzecevic.babymonitor.FrameStore
import androidx.compose.ui.Alignment
class MainActivity : ComponentActivity() {
private val latestJpeg = AtomicReference<ByteArray?>(null)
private val serverPort = 8080
private var cameraProvider: ProcessCameraProvider? = null
private lateinit var cameraExecutor: ExecutorService
private var cameraProvider: ProcessCameraProvider? = null
private var lensFacing = CameraSelector.LENS_FACING_BACK
private val frameCount = AtomicInteger(0)
private var rebindToken by mutableStateOf(0)
private val requestPermission = registerForActivityResult(
ActivityResultContracts.RequestPermission()
) { granted ->
if (!granted) Toast.makeText(this, "Camera permission required", Toast.LENGTH_LONG).show()
}
private val cameraPermissionLauncher =
registerForActivityResult(ActivityResultContracts.RequestPermission()) { }
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
cameraExecutor = Executors.newSingleThreadExecutor()
// request permission if needed
if (ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA)
!= PackageManager.PERMISSION_GRANTED
) {
requestPermission.launch(Manifest.permission.CAMERA)
cameraPermissionLauncher.launch(Manifest.permission.CAMERA)
}
setContent {
BabyMonitorScreen(
onStartServer = { ip ->
val intent = Intent(this, StreamService::class.java)
ContextCompat.startForegroundService(this, intent)
Toast.makeText(this, "Open: http://$ip:$serverPort/", Toast.LENGTH_LONG).show()
FrameStore.isStreaming = true
ContextCompat.startForegroundService(
this,
Intent(this, StreamService::class.java)
)
Toast.makeText(this, "Open http://$ip:8080/", Toast.LENGTH_LONG).show()
},
onStopServer = {
FrameStore.isStreaming = false
stopService(Intent(this, StreamService::class.java))
},
isServerRunning = { false },
onSwitchCamera = {
lensFacing =
if (lensFacing == CameraSelector.LENS_FACING_BACK) CameraSelector.LENS_FACING_FRONT
else CameraSelector.LENS_FACING_BACK
rebindToken++
},
rebindToken = rebindToken,
if (lensFacing == CameraSelector.LENS_FACING_BACK)
CameraSelector.LENS_FACING_FRONT
else
CameraSelector.LENS_FACING_BACK
bindCamera = { previewView ->
bindCameraUseCases(previewView)
rebindToken++
},
bindCamera = { bindCameraUseCases(it) },
getIp = { getLocalWifiIp() },
getFrames = { frameCount.get() }
rebindToken = rebindToken
)
}
}
@ -111,15 +91,12 @@ class MainActivity : ComponentActivity() {
}
private fun bindCameraUseCases(previewView: PreviewView) {
if (FrameStore.isStreaming) {
return
}
if (ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA)
!= PackageManager.PERMISSION_GRANTED
) return
if (FrameStore.isStreaming) return
val providerFuture = ProcessCameraProvider.getInstance(this)
providerFuture.addListener({
val provider = providerFuture.get()
cameraProvider = provider
provider.unbindAll()
@ -130,22 +107,20 @@ class MainActivity : ComponentActivity() {
val analysis = ImageAnalysis.Builder()
.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
.setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_YUV_420_888)
.build()
analysis.setAnalyzer(cameraExecutor) { image ->
try {
val rotation = image.imageInfo.rotationDegrees
val isFront = (lensFacing == CameraSelector.LENS_FACING_FRONT)
val jpg = YuvToJpegConverter.toJpeg(
image = image,
quality = 70,
rotationDegrees = rotation,
mirrorHorizontally = isFront
image,
70,
image.imageInfo.rotationDegrees,
lensFacing == CameraSelector.LENS_FACING_FRONT
)
FrameStore.latestJpeg.set(jpg)
FrameStore.frameCount.incrementAndGet()
} catch (_: Exception) {
} finally {
image.close()
@ -156,33 +131,28 @@ class MainActivity : ComponentActivity() {
.requireLensFacing(lensFacing)
.build()
try {
provider.bindToLifecycle(this, selector, preview, analysis)
} catch (e: Exception) {
Toast.makeText(this, "Camera bind failed: ${e.message}", Toast.LENGTH_LONG).show()
}
provider.bindToLifecycle(this, selector, preview, analysis)
}, ContextCompat.getMainExecutor(this))
}
private fun getLocalWifiIp(): String? {
if (!isOnWifi()) return null
// Prefer wlan interface addresses
NetworkInterface.getNetworkInterfaces().toList().forEach { nif ->
nif.inetAddresses.toList().forEach { addr ->
if (!addr.isLoopbackAddress && addr is Inet4Address) {
if (nif.name.startsWith("wlan") || nif.name.startsWith("wifi")) {
return addr.hostAddress
}
if (nif.name.startsWith("wlan")) return addr.hostAddress
}
}
}
// Fallback to WifiManager
val wm = applicationContext.getSystemService(Context.WIFI_SERVICE) as WifiManager
val ipInt = wm.connectionInfo.ipAddress
if (ipInt == 0) return null
return "${ipInt and 0xFF}.${(ipInt shr 8) and 0xFF}.${(ipInt shr 16) and 0xFF}.${(ipInt shr 24) and 0xFF}"
val ip = wm.connectionInfo.ipAddress
if (ip == 0) return null
return "${ip and 0xFF}.${(ip shr 8) and 0xFF}.${(ip shr 16) and 0xFF}.${(ip shr 24) and 0xFF}"
}
private fun isOnWifi(): Boolean {
@ -195,21 +165,20 @@ class MainActivity : ComponentActivity() {
@Composable
private fun BabyMonitorScreen(
onStartServer: (ip: String) -> Unit,
onStartServer: (String) -> Unit,
onStopServer: () -> Unit,
isServerRunning: () -> Boolean,
onSwitchCamera: () -> Unit,
bindCamera: (PreviewView) -> Unit,
getIp: () -> String?,
getFrames: () -> Int,
rebindToken: Int
) {
val context = LocalContext.current
var ip by remember { mutableStateOf(getIp()) }
var running by remember { mutableStateOf(isServerRunning()) }
var running by remember { mutableStateOf(false) }
// UI refresh of IP when screen recomposes
LaunchedEffect(Unit) { ip = getIp() }
val audioPermissionLauncher =
rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { }
Box(Modifier.fillMaxSize()) {
@ -219,89 +188,78 @@ private fun BabyMonitorScreen(
}
}
if (FrameStore.isStreaming) {
Column(
modifier = Modifier.fillMaxSize(),
verticalArrangement = Arrangement.Center,
horizontalAlignment = Alignment.CenterHorizontally
) {
Text("Streaming active", color = Color.White)
Text("Frames: ${getFrames()}", color = Color.White)
}
} else {
if (!FrameStore.isStreaming) {
AndroidView(
factory = { previewView },
modifier = Modifier.fillMaxSize()
)
}
LaunchedEffect(rebindToken) {
if (!FrameStore.isStreaming) {
bindCamera(previewView)
}
if (!FrameStore.isStreaming) bindCamera(previewView)
}
Column(
Modifier
.statusBarsPadding()
.padding(top = 8.dp)
.padding(horizontal = 12.dp)
.padding(12.dp)
.background(Color(0x80000000))
.padding(10.dp)
.padding(12.dp)
) {
if (ip != null) {
Text(
text = "URL: http://$ip:8080/",
color = Color.White
)
} else {
Text(
text = "Not connected to Wi-Fi",
color = Color.Red
)
}
Text(
text = ip?.let { "http://$it:8080/" } ?: "No Wi-Fi",
color = Color.White
)
Spacer(Modifier.height(8.dp))
Text(
text = if (running) "Status: STREAMING" else "Status: STOPPED",
text = if (running) "STREAMING" else "STOPPED",
color = if (running) Color.Green else Color.Yellow
)
Spacer(Modifier.height(10.dp))
Spacer(Modifier.height(12.dp))
Row {
Button(onClick = {
onSwitchCamera()
ip = getIp()
}) { Text("Switch camera") }
}) { Text("Switch") }
Spacer(Modifier.width(8.dp))
Button(onClick = {
ip = getIp()
val currentIp = ip
if (currentIp == null) {
Toast.makeText(context, "Connect to Wi-Fi first", Toast.LENGTH_LONG).show()
val currentIp = ip ?: run {
Toast.makeText(context, "Connect Wi-Fi", Toast.LENGTH_LONG).show()
return@Button
}
if (!running) {
if (ContextCompat.checkSelfPermission(
context,
Manifest.permission.RECORD_AUDIO
) != PackageManager.PERMISSION_GRANTED
) {
audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO)
}
onStartServer(currentIp)
running = true
} else {
onStopServer()
running = false
}
}) {
Text(if (!running) "Start streaming" else "Stop streaming")
Text(if (!running) "Start" else "Stop")
}
}
}
}
}
}

147
app/src/main/java/com/dzecevic/babymonitor/MjpegServer.kt

@ -17,53 +17,47 @@ class MjpegServer(
"""
<html>
<head>
<meta name="viewport" content="width=device-width, initial-scale=1">
<style>
body {
margin: 0;
background: black;
height: 100vh;
overflow: hidden;
}
.wrap {
position: relative;
width: 100vw;
height: 100vh;
display: flex;
justify-content: center;
align-items: center;
}
img {
max-width: 100%;
max-height: 100%;
object-fit: contain;
}
.title {
position: absolute;
top: 12px;
left: 12px;
padding: 8px 10px;
background: rgba(0,0,0,0.5);
color: white;
font-family: sans-serif;
font-size: 14px;
border-radius: 8px;
z-index: 10;
text-decoration: none;
}
</style>
<meta name="viewport" content="width=device-width, initial-scale=1">
<style>
body {
margin: 0;
background: black;
color: white;
display: flex;
flex-direction: column;
justify-content: center;
align-items: center;
height: 100vh;
font-family: Arial, sans-serif;
}
img {
max-width: 100%;
max-height: 85vh;
object-fit: contain;
}
.header {
margin-bottom: 10px;
}
</style>
</head>
<body>
<div class="wrap">
<a class="title" href="mailto:domagoj@zecevic-ninja.com">
Baby Monitor by domagoj@zecevic-ninja.com
</a>
<img src="/mjpeg" />
</div>
<div class="header">
<a href="mailto:domagoj@zecevic-ninja.com" style="color:white;text-decoration:none;">
Baby Monitor by domagoj@zecevic-ninja.com
</a>
</div>
<img src="/mjpeg"/>
<audio autoplay controls>
<source src="/audio" type="audio/wav">
</audio>
</body>
</html>
""".trimIndent()
@ -91,6 +85,33 @@ class MjpegServer(
addHeader("Connection", "close")
}
}
"/audio" -> {
val sampleRate = 16000
val pipeIn = java.io.PipedInputStream()
val pipeOut = java.io.PipedOutputStream(pipeIn)
Thread {
try {
pipeOut.write(wavHeaderPcm16Mono(sampleRate))
pipeOut.flush()
while (FrameStore.isStreaming) {
val pcm = FrameStoreAudio.latestPcm.get()
if (pcm != null) {
pipeOut.write(pcm)
pipeOut.flush()
}
Thread.sleep(20)
}
} catch (_: Exception) {
} finally {
try { pipeOut.close() } catch (_: Exception) {}
}
}.start()
newChunkedResponse(Response.Status.OK, "audio/wav", pipeIn)
}
else -> newFixedLengthResponse(Response.Status.NOT_FOUND, "text/plain", "Not found")
}
}
@ -113,4 +134,40 @@ class MjpegServer(
Thread.sleep(50)
return out.toByteArray()
}
private fun wavHeaderPcm16Mono(sampleRate: Int): ByteArray {
val channels = 1
val bits = 16
val byteRate = sampleRate * channels * (bits / 8)
val blockAlign = channels * (bits / 8)
val dataSize = 0x7fffffff // “very large”
val riffSize = 36 + dataSize
fun le32(v: Int) = byteArrayOf(
(v and 0xff).toByte(),
((v shr 8) and 0xff).toByte(),
((v shr 16) and 0xff).toByte(),
((v shr 24) and 0xff).toByte()
)
fun le16(v: Int) = byteArrayOf(
(v and 0xff).toByte(),
((v shr 8) and 0xff).toByte()
)
return byteArrayOf(
'R'.code.toByte(), 'I'.code.toByte(), 'F'.code.toByte(), 'F'.code.toByte(),
*le32(riffSize),
'W'.code.toByte(), 'A'.code.toByte(), 'V'.code.toByte(), 'E'.code.toByte(),
'f'.code.toByte(), 'm'.code.toByte(), 't'.code.toByte(), ' '.code.toByte(),
*le32(16),
*le16(1),
*le16(channels),
*le32(sampleRate),
*le32(byteRate),
*le16(blockAlign),
*le16(bits),
'd'.code.toByte(), 'a'.code.toByte(), 't'.code.toByte(), 'a'.code.toByte(),
*le32(dataSize)
)
}
}

53
app/src/main/java/com/dzecevic/babymonitor/StreamService.kt

@ -16,6 +16,7 @@ import androidx.lifecycle.LifecycleService
import fi.iki.elonen.NanoHTTPD
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import com.dzecevic.babymonitor.AudioRecorder
class StreamService : LifecycleService() {
@ -28,10 +29,19 @@ class StreamService : LifecycleService() {
// start with back camera
private var lensFacing = CameraSelector.LENS_FACING_BACK
private var audioThread: Thread? = null
private val audioRecorder = AudioRecorder()
@Volatile private var audioRunning = false
private val audioQueue = java.util.concurrent.LinkedBlockingQueue<ByteArray>(50)
override fun onCreate() {
super.onCreate()
FrameStore.isStreaming = true
startForeground(1, buildNotification())
startAudioCapture()
audioRecorder.start()
acquireWakeLock()
cameraExecutor = Executors.newSingleThreadExecutor()
@ -50,8 +60,8 @@ class StreamService : LifecycleService() {
} catch (_: Exception) { }
FrameStore.isStreaming = false
server?.stop()
server = null
stopAudioCapture()
audioRecorder.stop()
cameraExecutor.shutdown()
releaseWakeLock()
@ -131,4 +141,43 @@ class StreamService : LifecycleService() {
.setOngoing(true)
.build()
}
private fun startAudioCapture() {
val sampleRate = 16000
val channelConfig = android.media.AudioFormat.CHANNEL_IN_MONO
val audioFormat = android.media.AudioFormat.ENCODING_PCM_16BIT
val minBuf = android.media.AudioRecord.getMinBufferSize(sampleRate, channelConfig, audioFormat)
val recorder = android.media.AudioRecord(
android.media.MediaRecorder.AudioSource.MIC,
sampleRate,
channelConfig,
audioFormat,
minBuf * 2
)
audioRunning = true
recorder.startRecording()
audioThread = Thread {
val buf = ByteArray(2048)
while (audioRunning) {
val read = recorder.read(buf, 0, buf.size)
if (read > 0) {
// copy exact bytes read
val chunk = buf.copyOf(read)
audioQueue.offer(chunk) // drop if full is ok for live stream
}
}
try { recorder.stop() } catch (_: Exception) {}
recorder.release()
}.apply { start() }
}
private fun stopAudioCapture() {
audioRunning = false
audioThread?.join(500)
audioThread = null
audioQueue.clear()
}
}

Loading…
Cancel
Save