//! Backup-and-rebuild support for Codex runtime SQLite databases.//!//! Codex keeps several independent runtime SQLite databases under one SQLite//! home. When SQLite reports that one of them is corrupt, automatic recovery//! moves only that database file and its sidecars into a backup folder so the//! other databases keep their data.use std::borrow::Cow;use std::path::Path;use std::path::PathBuf;const BACKUP_DIR_NAME: &str = "db-backups";#[derive(Clone, Debug, Eq, PartialEq)]pub struct RuntimeDbBackup {    /// Path where the runtime database or sidecar lived before it was moved.    pub original_path: PathBuf,    /// Path where the runtime database or sidecar was backed up.    pub backup_path: PathBuf,}#[derive(Debug)]pub(crate) struct RuntimeDbInitError {    label: &'static str,    operation: &'static str,    path: PathBuf,    source: anyhow::Error,}impl RuntimeDbInitError {    pub(crate) fn new(        label: &'static str,        operation: &'static str,        path: &Path,        source: anyhow::Error,    ) -> Self {        Self {            label,            operation,            path: path.to_path_buf(),            source,        }    }    fn path(&self) -> &Path {        self.path.as_path()    }}impl std::fmt::Display for RuntimeDbInitError {    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {        write!(            f,            "failed to {} {} at {}: {}",            self.operation,            self.label,            self.path.display(),            self.source        )    }}impl std::error::Error for RuntimeDbInitError {    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {        Some(self.source.as_ref())    }}/// Move one Codex runtime SQLite database out of the way so that database can/// be recreated without discarding unrelated runtime databases.pub async fn backup_runtime_db_for_fresh_start(    db_path: &Path,) -> std::io::Result<Vec<RuntimeDbBackup>> {    let sqlite_home = db_path.parent().ok_or_else(|| {        std::io::Error::other(format!(            "database path does not have a parent directory: {}",            db_path.display()        ))    })?;    match tokio::fs::metadata(sqlite_home).await {        Ok(metadata) if metadata.is_dir() => backup_runtime_db_files(db_path).await,        Ok(_) => backup_blocking_sqlite_home(sqlite_home).await,        Err(err) if err.kind() == std::io::ErrorKind::NotFound => {            tokio::fs::create_dir_all(sqlite_home).await?;            Err(std::io::Error::other(format!(                "no Codex runtime database files were found to back up for {}",                db_path.display()            )))        }        Err(err) => Err(err),    }}pub fn runtime_db_path_for_corruption_error(err: &anyhow::Error) -> Option<PathBuf> {    if !is_sqlite_corruption_error(err) {        return None;    }    err.chain()        .find_map(|source| source.downcast_ref::<RuntimeDbInitError>())        .map(|err| err.path().to_path_buf())}pub fn is_sqlite_corruption_error(err: &anyhow::Error) -> bool {    err.chain().any(sqlite_error_source_is_corruption)}fn sqlite_error_source_is_corruption(source: &(dyn std::error::Error + 'static)) -> bool {    let Some(err) = source.downcast_ref::<sqlx::Error>() else {        return false;    };    let sqlx::Error::Database(database_error) = err else {        return false;    };    sqlite_error_detail_is_corruption(database_error.message())        || database_error            .code()            .is_some_and(sqlite_database_code_is_corruption)}fn sqlite_database_code_is_corruption(code: Cow<'_, str>) -> bool {    matches!(        code.as_ref().to_ascii_lowercase().as_str(),        "11" | "26" | "sqlite_corrupt" | "sqlite_notadb"    )}pub fn sqlite_error_detail_is_corruption(detail: &str) -> bool {    let detail = detail.to_ascii_lowercase();    detail.contains("database disk image is malformed")        || detail.contains("database schema is malformed")        || detail.contains("database is corrupt")        || detail.contains("file is not a database")        || detail.contains("sqlite_corrupt")        || detail.contains("sqlite_notadb")        || detail.contains("(code: 11)")        || detail.contains("(code: 26)")}pub fn sqlite_error_detail_is_lock(detail: &str) -> bool {    let detail = detail.to_ascii_lowercase();    detail.contains("database is locked") || detail.contains("database is busy")}async fn backup_runtime_db_files(db_path: &Path) -> std::io::Result<Vec<RuntimeDbBackup>> {    let sqlite_home = db_path.parent().ok_or_else(|| {        std::io::Error::other(format!(            "database path does not have a parent directory: {}",            db_path.display()        ))    })?;    backup_sqlite_paths(sqlite_home, sqlite_paths(db_path)).await}async fn backup_sqlite_paths(    sqlite_home: &Path,    paths: impl IntoIterator<Item = PathBuf>,) -> std::io::Result<Vec<RuntimeDbBackup>> {    let backup_dir = create_unique_backup_dir(sqlite_home.join(BACKUP_DIR_NAME).as_path()).await?;    let mut backups = Vec::new();    for path in paths {        if tokio::fs::try_exists(path.as_path()).await? {            let backup_path = backup_dir.join(file_name(path.as_path())?);            tokio::fs::rename(path.as_path(), backup_path.as_path()).await?;            backups.push(RuntimeDbBackup {                original_path: path,                backup_path,            });        }    }    if backups.is_empty() {        let _ = tokio::fs::remove_dir(backup_dir).await;        return Err(std::io::Error::other(            "no Codex runtime database files were found to back up",        ));    }    Ok(backups)}async fn backup_blocking_sqlite_home(sqlite_home: &Path) -> std::io::Result<Vec<RuntimeDbBackup>> {    let parent = sqlite_home.parent().ok_or_else(|| {        std::io::Error::other(format!(            "cannot create a backup folder for {}",            sqlite_home.display()        ))    })?;    let mut backup_dir_name = file_name(sqlite_home)?.to_os_string();    backup_dir_name.push(format!(".{BACKUP_DIR_NAME}"));    let backup_parent = parent.join(backup_dir_name);    let backup_dir = create_unique_backup_dir(backup_parent.as_path()).await?;    let backup_path = backup_dir.join(file_name(sqlite_home)?);    tokio::fs::rename(sqlite_home, backup_path.as_path()).await?;    tokio::fs::create_dir_all(sqlite_home).await?;    Ok(vec![RuntimeDbBackup {        original_path: sqlite_home.to_path_buf(),        backup_path,    }])}fn sqlite_paths(db_path: &Path) -> Vec<PathBuf> {    let mut wal_path = db_path.as_os_str().to_os_string();    wal_path.push("-wal");    let mut shm_path = db_path.as_os_str().to_os_string();    shm_path.push("-shm");    vec![        db_path.to_path_buf(),        PathBuf::from(wal_path),        PathBuf::from(shm_path),    ]}async fn create_unique_backup_dir(backup_parent: &Path) -> std::io::Result<PathBuf> {    tokio::fs::create_dir_all(backup_parent).await?;    let timestamp = std::time::SystemTime::now()        .duration_since(std::time::UNIX_EPOCH)        .map_or(0, |duration| duration.as_secs());    let mut sequence = 0_u32;    loop {        let backup_dir = backup_parent.join(format!("sqlite-{timestamp}-{sequence}"));        match tokio::fs::create_dir(backup_dir.as_path()).await {            Ok(()) => return Ok(backup_dir),            Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => {                sequence += 1;            }            Err(err) => return Err(err),        }    }}fn file_name(path: &Path) -> std::io::Result<&std::ffi::OsStr> {    path.file_name().ok_or_else(|| {        std::io::Error::other(format!(            "cannot create a backup name for {}",            path.display()        ))    })}#[cfg(test)]#[path = "recovery_tests.rs"]mod tests;