swarm-stt: the speech-to-text core (hash cache, shared runs, ffmpeg and whisper-cli engine)
This commit is contained in:
parent
5443c1c9ba
commit
df4b2f1838
1 changed files with 663 additions and 0 deletions
149
swarm-stt/src/cache.rs
Normal file
149
swarm-stt/src/cache.rs
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
//! The transcript cache: one `<sha256>.txt` file per memo in a directory, kept
|
||||
//! for a TTL counted from the file's modification time.
|
||||
|
||||
use std::fmt::Write as _;
|
||||
use std::io::ErrorKind;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use sha2::{Digest as _, Sha256};
|
||||
|
||||
/// The lowercase hex SHA-256 of `bytes`: the cache key.
|
||||
#[must_use]
|
||||
pub fn key(bytes: &[u8]) -> String {
|
||||
Sha256::digest(bytes)
|
||||
.iter()
|
||||
.fold(String::with_capacity(64), |mut hex, byte| {
|
||||
let _ = write!(hex, "{byte:02x}");
|
||||
hex
|
||||
})
|
||||
}
|
||||
|
||||
/// A directory of transcripts that expire `ttl` after they were written.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Cache {
|
||||
dir: PathBuf,
|
||||
ttl: Duration,
|
||||
}
|
||||
|
||||
impl Cache {
|
||||
/// A cache in `dir` (created on first write).
|
||||
#[must_use]
|
||||
pub fn new(dir: impl Into<PathBuf>, ttl: Duration) -> Self {
|
||||
Self {
|
||||
dir: dir.into(),
|
||||
ttl,
|
||||
}
|
||||
}
|
||||
|
||||
fn path(&self, key: &str) -> PathBuf {
|
||||
self.dir.join(format!("{key}.txt"))
|
||||
}
|
||||
|
||||
/// The transcript stored under `key`, unless it is missing or expired. An
|
||||
/// expired file is removed.
|
||||
pub async fn get(&self, key: &str) -> Option<String> {
|
||||
let path = self.path(key);
|
||||
let age = age_of(&path).await?;
|
||||
if age >= self.ttl {
|
||||
let _ = tokio::fs::remove_file(&path).await;
|
||||
return None;
|
||||
}
|
||||
tokio::fs::read_to_string(&path).await.ok()
|
||||
}
|
||||
|
||||
/// Stores `text` under `key`. The file is written beside its name and
|
||||
/// renamed into place, so a reader never sees half a transcript.
|
||||
pub async fn put(&self, key: &str, text: &str) -> std::io::Result<()> {
|
||||
tokio::fs::create_dir_all(&self.dir).await?;
|
||||
let path = self.path(key);
|
||||
let tmp = self.dir.join(format!("{key}.tmp"));
|
||||
tokio::fs::write(&tmp, text).await?;
|
||||
tokio::fs::rename(&tmp, &path).await
|
||||
}
|
||||
|
||||
/// Removes every expired transcript. Returns how many it removed.
|
||||
pub async fn purge(&self) -> usize {
|
||||
let Ok(mut entries) = tokio::fs::read_dir(&self.dir).await else {
|
||||
return 0;
|
||||
};
|
||||
let mut removed = 0;
|
||||
while let Ok(Some(entry)) = entries.next_entry().await {
|
||||
let path = entry.path();
|
||||
if path.extension().is_some_and(|ext| ext == "txt")
|
||||
&& age_of(&path).await.is_some_and(|age| age >= self.ttl)
|
||||
&& tokio::fs::remove_file(&path).await.is_ok()
|
||||
{
|
||||
removed += 1;
|
||||
}
|
||||
}
|
||||
removed
|
||||
}
|
||||
}
|
||||
|
||||
/// How long ago `path` was last written, or `None` when it cannot be told.
|
||||
async fn age_of(path: &Path) -> Option<Duration> {
|
||||
let modified = match tokio::fs::metadata(path).await {
|
||||
Ok(meta) => meta.modified().ok()?,
|
||||
Err(e) if e.kind() == ErrorKind::NotFound => return None,
|
||||
Err(e) => {
|
||||
tracing::warn!(path = %path.display(), error = %e, "reading a cached transcript's age failed");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
// A file dated in the future counts as brand new.
|
||||
Some(
|
||||
SystemTime::now()
|
||||
.duration_since(modified)
|
||||
.unwrap_or_default(),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{Cache, key};
|
||||
use std::time::Duration;
|
||||
|
||||
#[test]
|
||||
fn the_key_is_the_sha256_of_the_bytes() {
|
||||
assert_eq!(
|
||||
key(b"abc"),
|
||||
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_stored_transcript_is_found_until_it_expires() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let fresh = Cache::new(dir.path(), Duration::from_mins(1));
|
||||
fresh.put("k", "hello").await.unwrap();
|
||||
assert_eq!(fresh.get("k").await.as_deref(), Some("hello"));
|
||||
|
||||
let expired = Cache::new(dir.path(), Duration::ZERO);
|
||||
assert_eq!(expired.get("k").await, None);
|
||||
assert!(
|
||||
!dir.path().join("k.txt").exists(),
|
||||
"an expired file is removed"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn purge_removes_only_expired_transcripts() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let cache = Cache::new(dir.path(), Duration::from_mins(1));
|
||||
cache.put("keep", "a").await.unwrap();
|
||||
assert_eq!(cache.purge().await, 0);
|
||||
assert_eq!(cache.get("keep").await.as_deref(), Some("a"));
|
||||
|
||||
let expiring = Cache::new(dir.path(), Duration::ZERO);
|
||||
assert_eq!(expiring.purge().await, 1);
|
||||
assert_eq!(cache.get("keep").await, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_missing_directory_is_a_miss_and_an_empty_purge() {
|
||||
let cache = Cache::new("/nonexistent/swarm-stt-test", Duration::from_mins(1));
|
||||
assert_eq!(cache.get("k").await, None);
|
||||
assert_eq!(cache.purge().await, 0);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue