fxcp_core/

fxar.rs

// SPDX-License-Identifier: GPL-2.0-or-later
// Copyright (C) 2025 Joel Wirāmu Pauling <aenertia@aenertia.net>
//
// fxcp-core/src/fxar.rs  --  FXAR v2 content-addressable archive format

//! FXAR v2: variable-size gear-hash chunking + BLAKE3 CAS + binary index.
//! Replaces v1's tar+whole-file-dedup with true chunk-level deduplication.
//!
//! Archive layout: HEADER | MANIFEST | CHUNK INDEX | CHUNK DATA [ | FOOTER ]

use std::collections::HashSet;
use std::io::{self, Read, Write, Seek, SeekFrom};
use std::path::Path;
use serde::{Serialize, Deserialize};
use tracing::{info, warn};

use crate::chunker::GearChunker;

// -----------------------------------------------------------------------
// Magic bytes and constants
// -----------------------------------------------------------------------

const FXAR_MAGIC: &[u8; 4] = b"FXAR";
const FXAR_FOOTER_MAGIC: &[u8; 4] = b"FXAF";
const FXAR_VERSION: u32 = 2;
const HEADER_SIZE: usize = 64;
const CHUNK_INDEX_ENTRY_SIZE: usize = 48;
const FOOTER_SIZE: usize = 32;

/// Maximum allocation size for untrusted FXAR archive fields (64 MiB).
const MAX_FXAR_ALLOC: u64 = 64 * 1024 * 1024;

// Compression flag values
const FLAG_COMPRESS_NONE: u32 = 0;
const FLAG_COMPRESS_ZSTD: u32 = 1;
const FLAG_COMPRESS_LZ4: u32 = 2;
const FLAG_COMPRESS_GZIP: u32 = 3;
const FLAG_COMPRESS_XZ: u32 = 4;

/// Flag indicating the archive contains an extension section directory.
/// Set in `FxarHeader.flags` (bit 31  --  does not overlap compression bits 0-4).
pub const FLAG_HAS_EXTENSION_DIR: u32 = 0x8000_0000;

/// Extension section type: HNSW vector index.
pub const FXAR_EXT_HNSW: u32 = 1;
/// Extension section type: chunk-level content map.
pub const FXAR_EXT_CHUNK_MAP: u32 = 2;
/// Extension section type: file-level content map.
pub const FXAR_EXT_FILE_MAP: u32 = 3;

const EXTENSION_ENTRY_SIZE: usize = 56;

// -----------------------------------------------------------------------
// Header (64 bytes, little-endian)
// -----------------------------------------------------------------------

#[derive(Debug, Clone, Copy)]
#[repr(C)]
/// Binary header for FXAR v2 archives (64 bytes, little-endian).
pub struct FxarHeader {
    pub magic: [u8; 4],
    pub version: u32,
    pub flags: u32,
    pub chunk_min: u32,
    pub chunk_max: u32,
    pub chunk_avg: u32,
    pub index_offset: u64,
    pub index_size: u64,
    pub manifest_offset: u64,
    pub manifest_size: u64,
    pub chunk_count: u64,
}
// Header is 64 bytes: 4+4+4+4+4+4 + 8+8+8+8+8 = 64

impl FxarHeader {
    fn new(flags: u32, chunker: &GearChunker) -> Self {
        Self {
            magic: *FXAR_MAGIC,
            version: FXAR_VERSION,
            flags,
            chunk_min: chunker.min as u32,
            chunk_max: chunker.max as u32,
            chunk_avg: chunker.avg as u32,
            index_offset: 0,
            index_size: 0,
            manifest_offset: 0,
            manifest_size: 0,
            chunk_count: 0,
        }
    }

    fn write_to<W: Write>(&self, w: &mut W) -> io::Result<()> {
        w.write_all(&self.magic)?;
        w.write_all(&self.version.to_le_bytes())?;
        w.write_all(&self.flags.to_le_bytes())?;
        w.write_all(&self.chunk_min.to_le_bytes())?;
        w.write_all(&self.chunk_max.to_le_bytes())?;
        w.write_all(&self.chunk_avg.to_le_bytes())?;
        w.write_all(&self.index_offset.to_le_bytes())?;
        w.write_all(&self.index_size.to_le_bytes())?;
        w.write_all(&self.manifest_offset.to_le_bytes())?;
        w.write_all(&self.manifest_size.to_le_bytes())?;
        w.write_all(&self.chunk_count.to_le_bytes())?;
        Ok(())
    }

    fn read_from<R: Read>(r: &mut R) -> io::Result<Self> {
        let mut buf = [0u8; HEADER_SIZE];
        r.read_exact(&mut buf)?;

        let magic = [buf[0], buf[1], buf[2], buf[3]];
        if &magic != FXAR_MAGIC {
            return Err(io::Error::new(io::ErrorKind::InvalidData, "not an FXAR archive"));
        }

        Ok(Self {
            magic,
            version: u32::from_le_bytes([buf[4], buf[5], buf[6], buf[7]]),
            flags: u32::from_le_bytes([buf[8], buf[9], buf[10], buf[11]]),
            chunk_min: u32::from_le_bytes([buf[12], buf[13], buf[14], buf[15]]),
            chunk_max: u32::from_le_bytes([buf[16], buf[17], buf[18], buf[19]]),
            chunk_avg: u32::from_le_bytes([buf[20], buf[21], buf[22], buf[23]]),
            index_offset: u64::from_le_bytes([buf[24], buf[25], buf[26], buf[27], buf[28], buf[29], buf[30], buf[31]]),
            index_size: u64::from_le_bytes([buf[32], buf[33], buf[34], buf[35], buf[36], buf[37], buf[38], buf[39]]),
            manifest_offset: u64::from_le_bytes([buf[40], buf[41], buf[42], buf[43], buf[44], buf[45], buf[46], buf[47]]),
            manifest_size: u64::from_le_bytes([buf[48], buf[49], buf[50], buf[51], buf[52], buf[53], buf[54], buf[55]]),
            chunk_count: u64::from_le_bytes([buf[56], buf[57], buf[58], buf[59], buf[60], buf[61], buf[62], buf[63]]),
        })
    }
}

// -----------------------------------------------------------------------
// Chunk Index Entry (48 bytes, little-endian)
// -----------------------------------------------------------------------

#[derive(Debug, Clone)]
/// Binary chunk index entry (48 bytes, little-endian).
pub struct ChunkIndexEntry {
    pub blake3_hash: [u8; 32],
    pub size: u32,
    pub offset: u64,
    pub compressed_size: u32,
}

impl ChunkIndexEntry {
    fn write_to<W: Write>(&self, w: &mut W) -> io::Result<()> {
        w.write_all(&self.blake3_hash)?;
        w.write_all(&self.size.to_le_bytes())?;
        w.write_all(&self.offset.to_le_bytes())?;
        w.write_all(&self.compressed_size.to_le_bytes())?;
        Ok(())
    }

    fn read_from<R: Read>(r: &mut R) -> io::Result<Self> {
        let mut buf = [0u8; CHUNK_INDEX_ENTRY_SIZE];
        r.read_exact(&mut buf)?;

        let mut blake3_hash = [0u8; 32];
        blake3_hash.copy_from_slice(&buf[0..32]);

        Ok(Self {
            blake3_hash,
            size: u32::from_le_bytes([buf[32], buf[33], buf[34], buf[35]]),
            offset: u64::from_le_bytes([buf[36], buf[37], buf[38], buf[39], buf[40], buf[41], buf[42], buf[43]]),
            compressed_size: u32::from_le_bytes([buf[44], buf[45], buf[46], buf[47]]),
        })
    }
}

// -----------------------------------------------------------------------
// Footer (32 bytes, for non-seekable streams)
// -----------------------------------------------------------------------

#[derive(Debug, Clone)]
struct FxarFooter {
    magic: [u8; 4],
    manifest_offset: u64,
    index_offset: u64,
    chunk_count: u64,
    checksum: u32,
}

impl FxarFooter {
    fn compute_checksum(manifest_offset: u64, index_offset: u64, chunk_count: u64) -> u32 {
        // Simple checksum: XOR of all u32 words
        let mut crc: u32 = 0;
        for b in FXAR_FOOTER_MAGIC {
            crc = crc.wrapping_add(*b as u32);
        }
        for word in [
            manifest_offset as u32, (manifest_offset >> 32) as u32,
            index_offset as u32, (index_offset >> 32) as u32,
            chunk_count as u32, (chunk_count >> 32) as u32,
        ] {
            crc = crc.wrapping_add(word);
        }
        crc
    }

    fn write_to<W: Write>(&self, w: &mut W) -> io::Result<()> {
        w.write_all(&self.magic)?;
        w.write_all(&self.manifest_offset.to_le_bytes())?;
        w.write_all(&self.index_offset.to_le_bytes())?;
        w.write_all(&self.chunk_count.to_le_bytes())?;
        w.write_all(&self.checksum.to_le_bytes())?;
        Ok(())
    }

    fn read_from<R: Read>(r: &mut R) -> io::Result<Self> {
        let mut buf = [0u8; FOOTER_SIZE];
        r.read_exact(&mut buf)?;

        let magic = [buf[0], buf[1], buf[2], buf[3]];
        if &magic != FXAR_FOOTER_MAGIC {
            return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid FXAR footer"));
        }

        Ok(Self {
            magic,
            manifest_offset: u64::from_le_bytes([buf[4], buf[5], buf[6], buf[7], buf[8], buf[9], buf[10], buf[11]]),
            index_offset: u64::from_le_bytes([buf[12], buf[13], buf[14], buf[15], buf[16], buf[17], buf[18], buf[19]]),
            chunk_count: u64::from_le_bytes([buf[20], buf[21], buf[22], buf[23], buf[24], buf[25], buf[26], buf[27]]),
            checksum: u32::from_le_bytes([buf[28], buf[29], buf[30], buf[31]]),
        })
    }
}

// -----------------------------------------------------------------------
// Extension Section Directory (variable size, little-endian)
// -----------------------------------------------------------------------

/// A single entry in the FXAR extension section directory.
#[derive(Debug, Clone)]
pub struct FxarExtensionEntry {
    pub section_type: u32,
    pub offset: u64,
    pub size: u64,
    pub checksum: [u8; 32],
    pub _reserved: [u8; 4],
}

impl FxarExtensionEntry {
    fn write_to<W: Write>(&self, w: &mut W) -> io::Result<()> {
        w.write_all(&self.section_type.to_le_bytes())?;
        w.write_all(&self.offset.to_le_bytes())?;
        w.write_all(&self.size.to_le_bytes())?;
        w.write_all(&self.checksum)?;
        w.write_all(&self._reserved)?;
        Ok(())
    }

    fn read_from<R: Read>(r: &mut R) -> io::Result<Self> {
        let mut buf = [0u8; EXTENSION_ENTRY_SIZE];
        r.read_exact(&mut buf)?;

        let mut checksum = [0u8; 32];
        checksum.copy_from_slice(&buf[20..52]);
        let mut reserved = [0u8; 4];
        reserved.copy_from_slice(&buf[52..56]);

        Ok(Self {
            section_type: u32::from_le_bytes([buf[0], buf[1], buf[2], buf[3]]),
            offset: u64::from_le_bytes([buf[4], buf[5], buf[6], buf[7], buf[8], buf[9], buf[10], buf[11]]),
            size: u64::from_le_bytes([buf[12], buf[13], buf[14], buf[15], buf[16], buf[17], buf[18], buf[19]]),
            checksum,
            _reserved: reserved,
        })
    }
}

/// Directory of extension sections appended to an FXAR archive.
#[derive(Debug, Clone)]
pub struct FxarExtensionDirectory {
    pub entries: Vec<FxarExtensionEntry>,
}

impl FxarExtensionDirectory {
    fn write_to<W: Write>(&self, w: &mut W) -> io::Result<()> {
        w.write_all(&(self.entries.len() as u32).to_le_bytes())?;
        for entry in &self.entries {
            entry.write_to(w)?;
        }
        Ok(())
    }

    fn read_from<R: Read>(r: &mut R) -> io::Result<Self> {
        let mut count_buf = [0u8; 4];
        r.read_exact(&mut count_buf)?;
        let count = u32::from_le_bytes(count_buf) as usize;

        if count as u64 * EXTENSION_ENTRY_SIZE as u64 > MAX_FXAR_ALLOC {
            return Err(io::Error::new(
                io::ErrorKind::InvalidData,
                format!("extension directory entry count {} exceeds safety limit", count),
            ));
        }

        let mut entries = Vec::with_capacity(count);
        for _ in 0..count {
            entries.push(FxarExtensionEntry::read_from(r)?);
        }
        Ok(Self { entries })
    }
}

// -----------------------------------------------------------------------
// Manifest types (JSON)
// -----------------------------------------------------------------------

#[derive(Debug, Clone, Serialize, Deserialize)]
/// Archive manifest listing all files and their chunk references.
pub struct FxarManifest {
    pub version: u32,
    pub created: String,
    pub files: Vec<FxarManifestEntry>,
    #[serde(default)]
    pub snapshots: Vec<String>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
/// A single file entry in the FXAR manifest.
pub struct FxarManifestEntry {
    pub path: String,
    pub size: u64,
    pub mode: u32,
    pub mtime: i64,
    pub uid: u32,
    pub gid: u32,
    pub blake3: String,
    pub chunks: Vec<u64>,
    #[serde(default, skip_serializing_if = "std::collections::HashMap::is_empty")]
    pub xattr: std::collections::HashMap<String, String>,
}

// -----------------------------------------------------------------------
// Export/Import stats
// -----------------------------------------------------------------------

#[derive(Debug, Default, Serialize, Deserialize)]
/// Statistics from an FXAR export operation.
pub struct FxarExportStats {
    pub snapshots_exported: u64,
    pub total_files: u64,
    pub total_apparent_bytes: u64,
    pub chunk_count: u64,
    pub unique_chunks: u64,
    pub dedup_chunks: u64,
    pub archive_bytes: u64,
}

impl FxarExportStats {
    /// Compute the deduplication ratio (0.0 = no savings, 1.0 = all deduped).
    pub fn dedup_ratio(&self) -> f64 {
        if self.total_apparent_bytes == 0 { return 0.0; }
        1.0 - (self.archive_bytes as f64 / self.total_apparent_bytes as f64)
    }
}

#[derive(Debug, Default, Serialize, Deserialize)]
/// Statistics from an FXAR import/restore operation.
pub struct FxarImportStats {
    pub files_restored: u64,
    pub bytes_restored: u64,
    pub chunks_verified: u64,
    pub chunks_failed: u64,
}

// -----------------------------------------------------------------------
// Writer  --  parallel pipeline
// -----------------------------------------------------------------------

/// Per-file result from parallel processing (read + chunk + compress).
struct ProcessedFile {
    archive_path: String,
    size: u64,
    mode: u32,
    mtime: i64,
    uid: u32,
    gid: u32,
    /// Whole-file BLAKE3 derived from chunk hashes (avoids double-hashing).
    blake3: [u8; 32],
    /// (chunk_hash, uncompressed_size, compressed_data)
    chunks: Vec<([u8; 32], u32, Vec<u8>)>,
    xattr: std::collections::HashMap<String, String>,
}

/// Extract (snapshot_timestamp, relative_path) from an FXAR manifest path.
///
/// Manifest paths from VersionStore exports have the form `{snapshot}/tree/{rel_path}`.
/// Direct directory exports have bare relative paths (no snapshot/tree prefix).
///
/// Returns `(snapshot_ts, rel_path)` where `snapshot_ts` is empty for direct exports.
fn strip_snapshot_prefix(path: &str) -> (&str, &str) {
    if let Some(tree_pos) = path.find("/tree/") {
        (&path[..tree_pos], &path[tree_pos + 6..])
    } else {
        ("", path)
    }
}

/// Information about available versions for a file in the archive.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FileVersionInfo {
    /// Relative path (stripped of snapshot/tree prefix).
    pub rel_path: String,
    /// Snapshot timestamp of the version being restored.
    pub restored_snapshot: String,
    /// All available snapshot timestamps for this file (chronological order).
    pub available_snapshots: Vec<String>,
    /// Size of the restored version.
    pub size: u64,
}

/// Decode a blake3 manifest field that may be hex (legacy) or CID (new).
fn decode_blake3_field(s: &str) -> Option<[u8; 32]> {
    if crate::cid::is_blake3_cid_string(s) {
        crate::cid::cid_string_to_blake3(s).ok()
    } else {
        hex::decode(s).ok().and_then(|v| v.try_into().ok())
    }
}

/// Collect file paths for all snapshots, then process in parallel with rayon.
fn collect_and_process_files(
    snap_dirs: &[std::path::PathBuf],
    compress: &str,
) -> Vec<ProcessedFile> {
    use rayon::prelude::*;
    use std::os::unix::fs::{MetadataExt, PermissionsExt};

    // Collect all (snap_name, tree_dir, rel_path, full_path) tuples first
    let mut file_jobs: Vec<(String, std::path::PathBuf)> = Vec::new();
    for snap_dir in snap_dirs {
        let tree_dir = snap_dir.join("tree");
        if !tree_dir.exists() { continue; }
        let snap_name = snap_dir.file_name().unwrap_or_default().to_string_lossy().into_owned();

        for entry in walkdir::WalkDir::new(&tree_dir).follow_links(false) {
            let entry = match entry { Ok(e) => e, Err(_) => continue };
            if !entry.file_type().is_file() { continue; }
            let rel = match entry.path().strip_prefix(&tree_dir) {
                Ok(r) => r, Err(_) => continue,
            };
            let archive_path = format!("{}/tree/{}", snap_name, rel.display());
            file_jobs.push((archive_path, entry.path().to_path_buf()));
        }
    }

    let chunker = GearChunker::default();
    let compress_str = compress.to_string();

    // Parallel: read + chunk + hash + compress per file
    file_jobs.par_iter().filter_map(|(archive_path, full_path)| {
        let meta = std::fs::metadata(full_path).ok()?;
        let file_data = std::fs::read(full_path).ok()?;

        // Whole-file BLAKE3 (for integrity verification on restore)
        let file_hash = *blake3::hash(&file_data).as_bytes();

        // Gear-hash chunk + per-chunk BLAKE3 + compress  --  all in this thread
        let boundaries = chunker.find_boundaries(&file_data);
        let mut chunks = Vec::with_capacity(boundaries.len());
        let mut start = 0;
        for end in &boundaries {
            let slice = &file_data[start..*end];
            let chunk_hash = blake3::hash(slice);
            let compressed = compress_chunk(slice, &compress_str);
            chunks.push((*chunk_hash.as_bytes(), slice.len() as u32, compressed));
            start = *end;
        }

        // xattr  --  only attempt if file is on a filesystem that supports it
        let mut xattr_map = std::collections::HashMap::new();
        if let Ok(attrs) = xattr::list(full_path) {
            for attr in attrs {
                let key = attr.to_string_lossy().into_owned();
                if (key.starts_with("user.foxing")
                    || key.starts_with("user.dublincore")
                    || key == "user.xdg.tags"
                    || key.starts_with("security.")
                    || key.starts_with("system.posix_acl_"))
                    && let Ok(Some(val)) = xattr::get(full_path, &attr) {
                        xattr_map.insert(key, hex::encode(&val));
                    }
            }
        }

        Some(ProcessedFile {
            archive_path: archive_path.clone(),
            size: meta.len(),
            mode: meta.permissions().mode(),
            mtime: meta.mtime(),
            uid: meta.uid(),
            gid: meta.gid(),
            blake3: file_hash,
            chunks,
            xattr: xattr_map,
        })
    }).collect()
}

/// Merge parallel results into dedup tables (single-threaded  --  dedup map is sequential).
fn build_dedup_tables(
    processed: Vec<ProcessedFile>,
) -> (Vec<FxarManifestEntry>, Vec<ChunkIndexEntry>, Vec<Vec<u8>>, FxarExportStats) {
    let mut seen_chunks: std::collections::HashMap<[u8; 32], u64> = std::collections::HashMap::new();
    let mut chunk_entries: Vec<ChunkIndexEntry> = Vec::new();
    let mut chunk_data_blobs: Vec<Vec<u8>> = Vec::new();
    let mut manifest_entries = Vec::with_capacity(processed.len());
    let mut stats = FxarExportStats::default();

    for file in processed {
        stats.total_files += 1;
        stats.total_apparent_bytes += file.size;

        let mut chunk_indices = Vec::with_capacity(file.chunks.len());
        for (hash_bytes, uncompressed_size, compressed_data) in file.chunks {
            if let Some(&existing_idx) = seen_chunks.get(&hash_bytes) {
                chunk_indices.push(existing_idx);
                stats.dedup_chunks += 1;
            } else {
                let idx = chunk_entries.len() as u64;
                seen_chunks.insert(hash_bytes, idx);
                chunk_entries.push(ChunkIndexEntry {
                    blake3_hash: hash_bytes,
                    size: uncompressed_size,
                    offset: 0,
                    compressed_size: compressed_data.len() as u32,
                });
                chunk_data_blobs.push(compressed_data);
                chunk_indices.push(idx);
                stats.unique_chunks += 1;
            }
            stats.chunk_count += 1;
        }

        manifest_entries.push(FxarManifestEntry {
            path: file.archive_path,
            size: file.size,
            mode: file.mode,
            mtime: file.mtime,
            uid: file.uid,
            gid: file.gid,
            blake3: crate::cid::blake3_to_cid_string(&file.blake3),
            chunks: chunk_indices,
            xattr: file.xattr,
        });
    }

    (manifest_entries, chunk_entries, chunk_data_blobs, stats)
}

/// Emit the FXAR v2 binary archive to a writer.
fn emit_archive<W: Write>(
    mut writer: W,
    compress: &str,
    snap_names: Vec<String>,
    manifest_entries: Vec<FxarManifestEntry>,
    mut chunk_entries: Vec<ChunkIndexEntry>,
    chunk_data_blobs: &[Vec<u8>],
    _stats: &FxarExportStats,
    append_footer: bool,
) -> crate::Result<(FxarHeader, u64)> {
    let chunker = GearChunker::default();
    let flags = compress_flag(compress);
    let mut header = FxarHeader::new(flags, &chunker);
    header.chunk_count = chunk_entries.len() as u64;

    let manifest = FxarManifest {
        version: FXAR_VERSION,
        created: chrono::Utc::now().to_rfc3339(),
        files: manifest_entries,
        snapshots: snap_names,
    };

    // Write placeholder header
    header.write_to(&mut writer)?;
    let mut offset = HEADER_SIZE as u64;

    // Write manifest (JSON, compressed)
    let manifest_json = serde_json::to_vec(&manifest)
        .map_err(|e| crate::error::FxcpError::Config(format!("manifest JSON error: {}", e)))?;
    let manifest_compressed = compress_chunk(&manifest_json, compress);
    header.manifest_offset = offset;
    header.manifest_size = manifest_compressed.len() as u64;
    writer.write_all(&manifest_compressed.len().to_le_bytes())?;
    writer.write_all(&manifest_json.len().to_le_bytes())?;
    writer.write_all(&manifest_compressed)?;
    offset += 16 + manifest_compressed.len() as u64;

    // Write chunk index
    header.index_offset = offset;
    header.index_size = (chunk_entries.len() * CHUNK_INDEX_ENTRY_SIZE) as u64;

    let chunk_data_start = offset + header.index_size;
    let mut data_offset = chunk_data_start;
    for (i, entry) in chunk_entries.iter_mut().enumerate() {
        entry.offset = data_offset;
        data_offset += chunk_data_blobs[i].len() as u64;
    }
    for entry in &chunk_entries {
        entry.write_to(&mut writer)?;
    }
    offset += header.index_size;

    // Write chunk data
    for blob in chunk_data_blobs {
        writer.write_all(blob)?;
        offset += blob.len() as u64;
    }

    if append_footer {
        let footer = FxarFooter {
            magic: *FXAR_FOOTER_MAGIC,
            manifest_offset: header.manifest_offset,
            index_offset: header.index_offset,
            chunk_count: header.chunk_count,
            checksum: FxarFooter::compute_checksum(
                header.manifest_offset, header.index_offset, header.chunk_count
            ),
        };
        footer.write_to(&mut writer)?;
        offset += FOOTER_SIZE as u64;
    }

    writer.flush()?;
    Ok((header, offset))
}

/// Write an FXAR v2 archive from a VersionStore (streaming  --  appends footer).
pub fn write_archive<W: Write>(
    store: &crate::version_store::VersionStore,
    writer: W,
    compress: &str,
    timestamp_filter: Option<&str>,
) -> crate::Result<FxarExportStats> {
    let snap_dirs = store.collect_snap_dirs(timestamp_filter)?;
    let snap_names: Vec<String> = snap_dirs.iter()
        .map(|d| d.file_name().unwrap_or_default().to_string_lossy().into_owned())
        .collect();

    let processed = collect_and_process_files(&snap_dirs, compress);
    let snap_count = snap_dirs.len() as u64;
    let (manifest_entries, chunk_entries, chunk_data_blobs, mut stats) =
        build_dedup_tables(processed);
    stats.snapshots_exported = snap_count;

    let (_header, offset) = emit_archive(
        writer, compress, snap_names, manifest_entries,
        chunk_entries, &chunk_data_blobs, &stats, true,
    )?;
    stats.archive_bytes = offset;

    info!("FXAR v2 export: {} snapshots, {} files, {} chunks ({} unique, {} dedup), {:.1}% dedup ratio",
          stats.snapshots_exported, stats.total_files, stats.chunk_count,
          stats.unique_chunks, stats.dedup_chunks, stats.dedup_ratio() * 100.0);

    Ok(stats)
}

/// Write FXAR v2 with seekable writer  --  updates header offsets in-place.
pub fn write_archive_seekable<W: Write + Seek>(
    store: &crate::version_store::VersionStore,
    mut writer: W,
    compress: &str,
    timestamp_filter: Option<&str>,
) -> crate::Result<FxarExportStats> {
    let snap_dirs = store.collect_snap_dirs(timestamp_filter)?;
    let snap_names: Vec<String> = snap_dirs.iter()
        .map(|d| d.file_name().unwrap_or_default().to_string_lossy().into_owned())
        .collect();

    let processed = collect_and_process_files(&snap_dirs, compress);
    let snap_count = snap_dirs.len() as u64;
    let (manifest_entries, chunk_entries, chunk_data_blobs, mut stats) =
        build_dedup_tables(processed);
    stats.snapshots_exported = snap_count;

    let (header, offset) = emit_archive(
        &mut writer, compress, snap_names, manifest_entries,
        chunk_entries, &chunk_data_blobs, &stats, false,
    )?;
    stats.archive_bytes = offset;

    // Seek back and update header with real offsets
    writer.seek(SeekFrom::Start(0))?;
    header.write_to(&mut writer)?;
    writer.seek(SeekFrom::End(0))?;
    writer.flush()?;

    info!("FXAR v2 export (seekable): {} snapshots, {} files, {} unique chunks, {:.1}% dedup",
          stats.snapshots_exported, stats.total_files, stats.unique_chunks,
          stats.dedup_ratio() * 100.0);

    Ok(stats)
}

pub fn write_archive_seekable_with_ai<W: Write + Seek>(
    store: &crate::version_store::VersionStore,
    mut writer: W,
    compress: &str,
    timestamp_filter: Option<&str>,
    extension_sections: Vec<(u32, Vec<u8>)>,
) -> crate::Result<FxarExportStats> {
    let snap_dirs = store.collect_snap_dirs(timestamp_filter)?;
    let snap_names: Vec<String> = snap_dirs.iter()
        .map(|d| d.file_name().unwrap_or_default().to_string_lossy().into_owned())
        .collect();

    let processed = collect_and_process_files(&snap_dirs, compress);
    let snap_count = snap_dirs.len() as u64;
    let (manifest_entries, chunk_entries, chunk_data_blobs, mut stats) =
        build_dedup_tables(processed);
    stats.snapshots_exported = snap_count;

    let (mut header, offset) = emit_archive(
        &mut writer, compress, snap_names, manifest_entries,
        chunk_entries, &chunk_data_blobs, &stats, false,
    )?;
    stats.archive_bytes = offset;

    if !extension_sections.is_empty() {
        let mut ext_entries = Vec::with_capacity(extension_sections.len());

        for (section_type, data) in &extension_sections {
            let section_offset = writer.stream_position()?;
            writer.write_all(data)?;
            let checksum: [u8; 32] = *blake3::hash(data).as_bytes();
            ext_entries.push(FxarExtensionEntry {
                section_type: *section_type,
                offset: section_offset,
                size: data.len() as u64,
                checksum,
                _reserved: [0u8; 4],
            });
        }

        let ext_dir = FxarExtensionDirectory { entries: ext_entries };
        let ext_dir_offset = writer.stream_position()?;
        ext_dir.write_to(&mut writer)?;

        // 8-byte sentinel: absolute offset of the extension directory
        writer.write_all(&ext_dir_offset.to_le_bytes())?;

        // Trailing footer so sentinel position is well-defined at file_end - 40
        let footer = FxarFooter {
            magic: *FXAR_FOOTER_MAGIC,
            manifest_offset: header.manifest_offset,
            index_offset: header.index_offset,
            chunk_count: header.chunk_count,
            checksum: FxarFooter::compute_checksum(
                header.manifest_offset, header.index_offset, header.chunk_count,
            ),
        };
        footer.write_to(&mut writer)?;

        header.flags |= FLAG_HAS_EXTENSION_DIR;
        stats.archive_bytes = writer.stream_position()?;
    }

    writer.seek(SeekFrom::Start(0))?;
    header.write_to(&mut writer)?;
    writer.seek(SeekFrom::End(0))?;
    writer.flush()?;

    info!("FXAR v2 export (seekable+ai): {} snapshots, {} files, {} unique chunks, {} ext sections, {:.1}% dedup",
          stats.snapshots_exported, stats.total_files, stats.unique_chunks,
          extension_sections.len(), stats.dedup_ratio() * 100.0);

    Ok(stats)
}

/// Read `.foxing_index/` files from disk and return them as raw
/// `(section_type, bytes)` pairs ready for [`write_archive_seekable_with_ai`].
///
/// Always includes `chunk_map.bin` ([`FXAR_EXT_CHUNK_MAP`]) and
/// `file_map.bin` ([`FXAR_EXT_FILE_MAP`]) when present.
/// `vectors.usearch` ([`FXAR_EXT_HNSW`]) is included only when
/// `include_vectors` is `true`.
///
/// Returns an empty `Vec` if `index_dir` does not exist.
pub fn prepare_ai_extension_sections(
    index_dir: &Path,
    include_vectors: bool,
) -> io::Result<Vec<(u32, Vec<u8>)>> {
    if !index_dir.exists() {
        return Ok(vec![]);
    }
    let mut sections = Vec::new();
    for (path, section_type) in [
        (index_dir.join("chunk_map.bin"), FXAR_EXT_CHUNK_MAP),
        (index_dir.join("file_map.bin"), FXAR_EXT_FILE_MAP),
    ] {
        if path.exists() {
            let data = std::fs::read(&path)?;
            sections.push((section_type, data));
        }
    }
    if include_vectors {
        let vectors_path = index_dir.join("vectors.usearch");
        if vectors_path.exists() {
            let data = std::fs::read(&vectors_path)?;
            sections.push((FXAR_EXT_HNSW, data));
        }
    }
    Ok(sections)
}

/// Convenience wrapper: reads `.foxing_index/` files via
/// [`prepare_ai_extension_sections`], then writes the archive with
/// [`write_archive_seekable_with_ai`].
pub fn write_archive_with_ai_index<W: Write + Seek>(
    store: &crate::version_store::VersionStore,
    writer: W,
    compress: &str,
    timestamp_filter: Option<&str>,
    index_dir: Option<&Path>,
    include_vectors: bool,
) -> crate::Result<FxarExportStats> {
    let sections = match index_dir {
        Some(dir) => prepare_ai_extension_sections(dir, include_vectors)
            .map_err(crate::error::FxcpError::Io)?,
        None => vec![],
    };
    write_archive_seekable_with_ai(store, writer, compress, timestamp_filter, sections)
}

/// Write an FXAR v2 archive directly from a directory (no VersionStore required).
///
/// Walks `source_dir` and archives all files with bare relative paths in the
/// manifest (no `{snapshot}/tree/` prefix). This is the "just archive this
/// directory" mode  --  like `tar -cf archive.fxar /source`.
pub fn write_archive_from_directory<W: Write + Seek>(
    source_dir: &Path,
    mut writer: W,
    compress: &str,
) -> crate::Result<FxarExportStats> {
    let processed = collect_files_from_directory(source_dir, compress);
    let (manifest_entries, chunk_entries, chunk_data_blobs, mut stats) =
        build_dedup_tables(processed);
    stats.snapshots_exported = 0;

    let (header, offset) = emit_archive(
        &mut writer, compress, vec![], manifest_entries,
        chunk_entries, &chunk_data_blobs, &stats, false,
    )?;
    stats.archive_bytes = offset;

    writer.seek(SeekFrom::Start(0))?;
    header.write_to(&mut writer)?;
    writer.seek(SeekFrom::End(0))?;
    writer.flush()?;

    info!("FXAR v2 export (directory): {} files, {} unique chunks, {:.1}% dedup",
          stats.total_files, stats.unique_chunks, stats.dedup_ratio() * 100.0);

    Ok(stats)
}

/// Write an FXAR v2 archive from a directory to a non-seekable stream (appends footer).
pub fn write_archive_from_directory_stream<W: Write>(
    source_dir: &Path,
    writer: W,
    compress: &str,
) -> crate::Result<FxarExportStats> {
    let processed = collect_files_from_directory(source_dir, compress);
    let (manifest_entries, chunk_entries, chunk_data_blobs, mut stats) =
        build_dedup_tables(processed);
    stats.snapshots_exported = 0;

    let (_header, offset) = emit_archive(
        writer, compress, vec![], manifest_entries,
        chunk_entries, &chunk_data_blobs, &stats, true,
    )?;
    stats.archive_bytes = offset;

    info!("FXAR v2 export (directory, streaming): {} files, {} unique chunks, {:.1}% dedup",
          stats.total_files, stats.unique_chunks, stats.dedup_ratio() * 100.0);

    Ok(stats)
}

/// Collect files from a directory with bare relative paths (no snapshot/tree prefix).
fn collect_files_from_directory(
    source_dir: &Path,
    compress: &str,
) -> Vec<ProcessedFile> {
    use rayon::prelude::*;
    use std::os::unix::fs::{MetadataExt, PermissionsExt};

    let mut file_jobs: Vec<(String, std::path::PathBuf)> = Vec::new();
    for entry in walkdir::WalkDir::new(source_dir).follow_links(false) {
        let entry = match entry { Ok(e) => e, Err(_) => continue };
        if !entry.file_type().is_file() { continue; }
        let rel = match entry.path().strip_prefix(source_dir) {
            Ok(r) => r, Err(_) => continue,
        };
        // Skip .foxing internal directories
        let rel_str = rel.to_string_lossy();
        if rel_str.starts_with(".foxing") { continue; }
        file_jobs.push((rel_str.into_owned(), entry.path().to_path_buf()));
    }

    let chunker = GearChunker::default();
    let compress_str = compress.to_string();

    file_jobs.par_iter().filter_map(|(archive_path, full_path)| {
        let meta = std::fs::metadata(full_path).ok()?;
        let file_data = std::fs::read(full_path).ok()?;
        let file_hash = *blake3::hash(&file_data).as_bytes();

        let boundaries = chunker.find_boundaries(&file_data);
        let mut chunks = Vec::with_capacity(boundaries.len());
        let mut start = 0;
        for end in &boundaries {
            let slice = &file_data[start..*end];
            let chunk_hash = blake3::hash(slice);
            let compressed = compress_chunk(slice, &compress_str);
            chunks.push((*chunk_hash.as_bytes(), slice.len() as u32, compressed));
            start = *end;
        }

        let mut xattr_map = std::collections::HashMap::new();
        if let Ok(attrs) = xattr::list(full_path) {
            for attr in attrs {
                let key = attr.to_string_lossy().into_owned();
                if (key.starts_with("user.foxing")
                    || key.starts_with("user.dublincore")
                    || key == "user.xdg.tags"
                    || key.starts_with("security.")
                    || key.starts_with("system.posix_acl_"))
                    && let Ok(Some(val)) = xattr::get(full_path, &attr) {
                        xattr_map.insert(key, hex::encode(&val));
                    }
            }
        }

        Some(ProcessedFile {
            archive_path: archive_path.clone(),
            size: meta.len(),
            mode: meta.permissions().mode(),
            mtime: meta.mtime(),
            uid: meta.uid(),
            gid: meta.gid(),
            blake3: file_hash,
            chunks,
            xattr: xattr_map,
        })
    }).collect()
}

/// Update the AI extension sections of an existing FXAR archive in-place.
///
/// Only the extension-section tail is rewritten  --  manifest, chunk index, and chunk data
/// are never touched. Handles both archives that already have extension sections (update)
/// and those that don't (append).
///
/// `sections` is a list of `(section_type, raw_bytes)` pairs, matching the format
/// used by [`write_archive_seekable_with_ai`].
pub fn update_fxar_ai_sections(path: &Path, sections: &[(u32, Vec<u8>)]) -> io::Result<()> {
    if sections.is_empty() {
        return Ok(());
    }

    let mut f = std::fs::OpenOptions::new()
        .read(true)
        .write(true)
        .open(path)?;
    let file_size = f.seek(SeekFrom::End(0))?;
    if file_size < HEADER_SIZE as u64 {
        return Err(io::Error::new(
            io::ErrorKind::InvalidData,
            "FXAR too small",
        ));
    }

    f.seek(SeekFrom::Start(0))?;
    let mut header = FxarHeader::read_from(&mut f)?;

    let (insertion_point, manifest_offset, index_offset, chunk_count) =
        if header.flags & FLAG_HAS_EXTENSION_DIR != 0 {
            // Already has extension dir  --  read sentinel to find where sections start
            let sentinel_pos = file_size - FOOTER_SIZE as u64 - 8;
            f.seek(SeekFrom::Start(sentinel_pos))?;
            let mut buf = [0u8; 8];
            f.read_exact(&mut buf)?;
            let ext_dir_offset = u64::from_le_bytes(buf);

            f.seek(SeekFrom::Start(file_size - FOOTER_SIZE as u64))?;
            let footer = FxarFooter::read_from(&mut f)?;

            // Find the earliest section data offset from the extension directory
            f.seek(SeekFrom::Start(ext_dir_offset))?;
            let ext_dir = FxarExtensionDirectory::read_from(&mut f)?;
            let earliest = ext_dir
                .entries
                .iter()
                .map(|e| e.offset)
                .min()
                .unwrap_or(ext_dir_offset);

            (earliest, footer.manifest_offset, footer.index_offset, footer.chunk_count)
        } else {
            // No extension dir yet
            let has_footer = if file_size >= FOOTER_SIZE as u64 {
                f.seek(SeekFrom::Start(file_size - FOOTER_SIZE as u64))?;
                let mut magic = [0u8; 4];
                f.read_exact(&mut magic).is_ok() && &magic == FXAR_FOOTER_MAGIC
            } else {
                false
            };

            if has_footer {
                f.seek(SeekFrom::Start(file_size - FOOTER_SIZE as u64))?;
                let footer = FxarFooter::read_from(&mut f)?;
                (
                    file_size - FOOTER_SIZE as u64,
                    footer.manifest_offset,
                    footer.index_offset,
                    footer.chunk_count,
                )
            } else {
                (
                    file_size,
                    header.manifest_offset,
                    header.index_offset,
                    header.chunk_count,
                )
            }
        };

    f.seek(SeekFrom::Start(insertion_point))?;

    let mut ext_entries = Vec::with_capacity(sections.len());
    for (section_type, data) in sections {
        let section_offset = f.stream_position()?;
        f.write_all(data)?;
        let checksum: [u8; 32] = *blake3::hash(data).as_bytes();
        ext_entries.push(FxarExtensionEntry {
            section_type: *section_type,
            offset: section_offset,
            size: data.len() as u64,
            checksum,
            _reserved: [0u8; 4],
        });
    }

    let ext_dir_offset = f.stream_position()?;
    let ext_dir = FxarExtensionDirectory {
        entries: ext_entries,
    };
    ext_dir.write_to(&mut f)?;

    f.write_all(&ext_dir_offset.to_le_bytes())?;

    let new_footer = FxarFooter {
        magic: *FXAR_FOOTER_MAGIC,
        manifest_offset,
        index_offset,
        chunk_count,
        checksum: FxarFooter::compute_checksum(manifest_offset, index_offset, chunk_count),
    };
    new_footer.write_to(&mut f)?;

    let new_size = f.stream_position()?;
    f.set_len(new_size)?;
    f.flush()?;

    if header.flags & FLAG_HAS_EXTENSION_DIR == 0 {
        header.flags |= FLAG_HAS_EXTENSION_DIR;
        f.seek(SeekFrom::Start(0))?;
        header.write_to(&mut f)?;
        f.flush()?;
    }

    Ok(())
}

// -----------------------------------------------------------------------
// Reader (seekable random access)
// -----------------------------------------------------------------------

/// Read an FXAR v2 archive with random access.
pub struct FxarReader<R: Read + Seek> {
    reader: R,
    pub header: FxarHeader,
}

impl<R: Read + Seek> FxarReader<R> {
    /// Open an FXAR v2 archive for reading.
    ///
    /// Automatically handles streaming-mode archives whose header has placeholder
    /// offsets (manifest_offset=0, index_offset=0) by falling back to the footer.
    pub fn open(mut reader: R) -> io::Result<Self> {
        let mut header = FxarHeader::read_from(&mut reader)?;
        if header.version != FXAR_VERSION {
            return Err(io::Error::new(
                io::ErrorKind::InvalidData,
                format!("unsupported FXAR version: {}", header.version),
            ));
        }

        // Streaming-mode archives write placeholder offsets in the header and
        // append a footer with the real values. Detect and patch.
        // manifest_offset < HEADER_SIZE means it's a placeholder (valid offset is always >= 64).
        if (header.manifest_offset as usize) < HEADER_SIZE {
            let file_size = reader.seek(SeekFrom::End(0))?;
            if file_size >= (HEADER_SIZE + FOOTER_SIZE) as u64 {
                reader.seek(SeekFrom::Start(file_size - FOOTER_SIZE as u64))?;
                if let Ok(footer) = FxarFooter::read_from(&mut reader) {
                    header.manifest_offset = footer.manifest_offset;
                    header.index_offset = footer.index_offset;
                    header.chunk_count = footer.chunk_count;
                    header.index_size = footer.chunk_count * CHUNK_INDEX_ENTRY_SIZE as u64;
                }
            }
        }

        Ok(Self { reader, header })
    }

    /// Read and parse the manifest.
    pub fn read_manifest(&mut self) -> io::Result<FxarManifest> {
        self.reader.seek(SeekFrom::Start(self.header.manifest_offset))?;
        let mut size_buf = [0u8; 8];
        self.reader.read_exact(&mut size_buf)?;
        let compressed_len = u64::from_le_bytes(size_buf);
        if compressed_len > MAX_FXAR_ALLOC {
            return Err(io::Error::new(
                io::ErrorKind::InvalidData,
                format!("FXAR manifest compressed size {} exceeds maximum {}", compressed_len, MAX_FXAR_ALLOC),
            ));
        }
        let compressed_len = compressed_len as usize;
        self.reader.read_exact(&mut size_buf)?;
        let _uncompressed_len = u64::from_le_bytes(size_buf) as usize;

        let mut compressed = vec![0u8; compressed_len];
        self.reader.read_exact(&mut compressed)?;

        let json_data = decompress_chunk(&compressed, self.header.flags)?;
        serde_json::from_slice(&json_data).map_err(|e|
            io::Error::new(io::ErrorKind::InvalidData, format!("manifest JSON: {}", e))
        )
    }

    /// Read the chunk index.
    pub fn read_chunk_index(&mut self) -> io::Result<Vec<ChunkIndexEntry>> {
        self.reader.seek(SeekFrom::Start(self.header.index_offset))?;
        let count = self.header.chunk_count as usize;
        let mut entries = Vec::with_capacity(count);
        for _ in 0..count {
            entries.push(ChunkIndexEntry::read_from(&mut self.reader)?);
        }
        Ok(entries)
    }

    /// Read and decompress a single chunk by its index entry.
    fn read_chunk_data(&mut self, entry: &ChunkIndexEntry) -> io::Result<Vec<u8>> {
        if entry.compressed_size as u64 > MAX_FXAR_ALLOC {
            return Err(io::Error::new(
                io::ErrorKind::InvalidData,
                format!("FXAR chunk compressed size {} exceeds maximum {}", entry.compressed_size, MAX_FXAR_ALLOC),
            ));
        }
        self.reader.seek(SeekFrom::Start(entry.offset))?;
        let mut compressed = vec![0u8; entry.compressed_size as usize];
        self.reader.read_exact(&mut compressed)?;

        let data = decompress_chunk(&compressed, self.header.flags)?;

        // Verify BLAKE3
        let actual_hash = blake3::hash(&data);
        if actual_hash.as_bytes() != &entry.blake3_hash {
            return Err(io::Error::new(
                io::ErrorKind::InvalidData,
                format!("BLAKE3 mismatch for chunk at offset {}", entry.offset),
            ));
        }

        Ok(data)
    }

    /// Restore a single file by path from the archive.
    pub fn restore_file(&mut self, file_path: &str) -> io::Result<Vec<u8>> {
        let manifest = self.read_manifest()?;
        let index = self.read_chunk_index()?;

        let entry = manifest.files.iter()
            .find(|f| f.path == file_path)
            .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "file not found in archive"))?;

        let mut data = Vec::with_capacity(entry.size as usize);
        for &chunk_idx in &entry.chunks {
            let chunk_meta = &index[chunk_idx as usize];
            let chunk_data = self.read_chunk_data(chunk_meta)?;
            data.extend_from_slice(&chunk_data);
        }

        // Verify whole-file BLAKE3 (accepts both hex and CID manifest fields)
        let actual = blake3::hash(&data);
        let expected = decode_blake3_field(&entry.blake3).unwrap_or_default();
        if *actual.as_bytes() != expected {
            return Err(io::Error::new(
                io::ErrorKind::InvalidData,
                format!("file BLAKE3 mismatch for {}", file_path),
            ));
        }

        Ok(data)
    }

    /// Read the first `max_bytes` of a file from the archive without
    /// verifying the whole-file BLAKE3 hash (partial reads cannot be
    /// hash-verified).  Per-chunk BLAKE3 is still checked.
    ///
    /// Returns fewer than `max_bytes` if the file is smaller.
    pub fn restore_file_head(&mut self, file_path: &str, max_bytes: usize) -> io::Result<Vec<u8>> {
        let manifest = self.read_manifest()?;
        let index = self.read_chunk_index()?;

        let entry = manifest.files.iter()
            .find(|f| f.path == file_path)
            .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "file not found in archive"))?;

        let mut data = Vec::with_capacity(max_bytes.min(entry.size as usize));
        for &chunk_idx in &entry.chunks {
            let chunk_meta = &index[chunk_idx as usize];
            let chunk_data = self.read_chunk_data(chunk_meta)?;
            data.extend_from_slice(&chunk_data);
            if data.len() >= max_bytes {
                break;
            }
        }
        data.truncate(max_bytes);
        Ok(data)
    }

    /// Restore all files to a target directory.
    ///
    /// Pipelined: chunk loading overlaps with file writing.
    /// Producer thread reads chunks sequentially; consumer rayon threads
    /// write files as soon as their chunks become available.
    pub fn restore_all(&mut self, target: &Path, generate_sigs: bool) -> io::Result<FxarImportStats> {
        // Pre-flight disk space check
        let total_bytes: u64 = {
            let m = self.read_manifest()?;
            m.files.iter().map(|f| f.size).sum()
        };
        if let Some(avail) = check_available_space(target) {
            let needed = total_bytes + crate::constants::FXAR_RESTORE_HEADROOM_BYTES;
            if avail < needed {
                return Err(io::Error::other(
                    format!("insufficient disk space: {} bytes available, {} bytes needed",
                            avail, needed),
                ));
            }
        }

        let manifest = self.read_manifest()?;
        let index = self.read_chunk_index()?;
        let chunk_count = index.len();

        // Pre-create all parent directories (sequential, must happen before writes)
        for file_entry in &manifest.files {
            let dest = target.join(&file_entry.path);
            if let Some(parent) = dest.parent() {
                std::fs::create_dir_all(parent)?;
            }
        }

        // Pipelined chunk loading: allocate chunk store, load in background thread
        let chunk_store: Vec<parking_lot::RwLock<Option<Vec<u8>>>> =
            (0..chunk_count).map(|_| parking_lot::RwLock::new(None)).collect();
        let chunk_store = std::sync::Arc::new(chunk_store);
        let chunks_loaded = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));

        // Sort files by their highest chunk index so we can process them
        // as soon as their chunks become available
        let mut file_order: Vec<(usize, u64)> = manifest.files.iter().enumerate()
            .map(|(i, f)| (i, f.chunks.iter().copied().max().unwrap_or(0)))
            .collect();
        file_order.sort_by_key(|&(_, max_chunk)| max_chunk);

        // Producer: load chunks in a background thread
        let producer_store = chunk_store.clone();
        let producer_loaded = chunks_loaded.clone();
        let producer_index = index.clone();
        // We need to move reader data out of self for the producer thread.
        // Read all compressed chunks first (sequential seeks), decompress in producer.
        let flags = self.header.flags;
        let mut raw_chunks: Vec<Vec<u8>> = Vec::with_capacity(chunk_count);
        for entry in &producer_index {
            if entry.compressed_size as u64 > MAX_FXAR_ALLOC {
                return Err(io::Error::new(
                    io::ErrorKind::InvalidData,
                    format!("FXAR chunk compressed size {} exceeds maximum {}", entry.compressed_size, MAX_FXAR_ALLOC),
                ));
            }
            self.reader.seek(SeekFrom::Start(entry.offset))?;
            let mut compressed = vec![0u8; entry.compressed_size as usize];
            self.reader.read_exact(&mut compressed)?;
            raw_chunks.push(compressed);
        }

        let producer_handle = std::thread::spawn(move || {
            for (i, compressed) in raw_chunks.into_iter().enumerate() {
                let data = match decompress_chunk(&compressed, flags) {
                    Ok(d) => d,
                    Err(e) => {
                        warn!("chunk {} decompress failed: {}", i, e);
                        Vec::new() // empty = will fail BLAKE3 verify
                    }
                };
                // Verify per-chunk BLAKE3
                let actual = blake3::hash(&data);
                if actual.as_bytes() != &producer_index[i].blake3_hash {
                    warn!("chunk {} BLAKE3 mismatch", i);
                }
                *producer_store[i].write() = Some(data);
                producer_loaded.store(i + 1, std::sync::atomic::Ordering::Release);
            }
        });

        // NFS bypass pool
        #[cfg(feature = "nfs-bypass")]
        let nfs_pool: Option<crate::nfs::NfsClientPool> = init_nfs_pool(target);

        // Consumer: process files in chunk-availability order using rayon
        use rayon::prelude::*;
        use std::sync::atomic::{AtomicU64, Ordering};

        let files_restored = AtomicU64::new(0);
        let bytes_restored = AtomicU64::new(0);
        let chunks_verified = AtomicU64::new(0);
        let chunks_failed = AtomicU64::new(0);

        // Process files in batches based on chunk availability
        file_order.par_iter().for_each(|&(file_idx, max_chunk)| {
            let file_entry = &manifest.files[file_idx];
            let dest = target.join(&file_entry.path);

            // Wait for all required chunks to be loaded
            let needed = max_chunk as usize + 1;
            while chunks_loaded.load(Ordering::Acquire) < needed {
                std::hint::spin_loop();
            }

            // Assemble file from chunks
            let mut data = Vec::with_capacity(file_entry.size as usize);
            for &chunk_idx in &file_entry.chunks {
                let guard = chunk_store[chunk_idx as usize].read();
                if let Some(chunk_data) = guard.as_ref() {
                    data.extend_from_slice(chunk_data);
                    chunks_verified.fetch_add(1, Ordering::Relaxed);
                } else {
                    chunks_failed.fetch_add(1, Ordering::Relaxed);
                    return;
                }
            }

            // Verify whole-file BLAKE3 (accepts both hex and CID manifest fields)
            let actual = blake3::hash(&data);
            let expected = decode_blake3_field(&file_entry.blake3).unwrap_or_default();
            if *actual.as_bytes() != expected {
                warn!("BLAKE3 mismatch for {}, skipping", file_entry.path);
                chunks_failed.fetch_add(1, Ordering::Relaxed);
                return;
            }

            // Write file  --  NFS pool bypass or VFS fallback
            let wrote_via_nfs = write_file_with_pool(
                &dest, &data, file_entry, target, file_idx,
                #[cfg(feature = "nfs-bypass")]
                &nfs_pool,
            );

            if !wrote_via_nfs {
                if std::fs::write(&dest, &data).is_err() {
                    chunks_failed.fetch_add(1, Ordering::Relaxed);
                    return;
                }
                use std::os::unix::fs::PermissionsExt;
                let _ = std::fs::set_permissions(&dest, std::fs::Permissions::from_mode(file_entry.mode));
                // Restore ownership from manifest (requires root)
                if unsafe { libc::geteuid() } == 0 {
                    use nix::unistd::{chown, Uid, Gid};
                    let _ = chown(&dest, Some(Uid::from_raw(file_entry.uid)), Some(Gid::from_raw(file_entry.gid)));
                }
                let mtime = filetime::FileTime::from_unix_time(file_entry.mtime, 0);
                let _ = filetime::set_file_mtime(&dest, mtime);
            }

            for (key, hex_val) in &file_entry.xattr {
                if let Ok(val) = hex::decode(hex_val) {
                    if key.starts_with("user.foxing") {
                        let _ = crate::sidecar::set_metadata(&dest, key, &val);
                    } else {
                        let _ = xattr::set(&dest, key, &val);
                    }
                }
            }

            // Generate foxingd-compatible signatures if requested
            if generate_sigs {
                let sig = crate::sidecar::SyncSignature::compute_from_buffer(
                    &data, file_entry.mtime, 0,
                );
                let _ = crate::sidecar::set_sync_signature(&dest, &sig);
            }

            files_restored.fetch_add(1, Ordering::Relaxed);
            bytes_restored.fetch_add(file_entry.size, Ordering::Relaxed);
        });

        // Post-restore: compute dir hashes for adaptive pruning
        if generate_sigs {
            compute_dir_hashes_recursive(target);
        }

        producer_handle.join().map_err(|_|
            io::Error::other("chunk producer thread panicked")
        )?;

        let stats = FxarImportStats {
            files_restored: files_restored.load(Ordering::Relaxed),
            bytes_restored: bytes_restored.load(Ordering::Relaxed),
            chunks_verified: chunks_verified.load(Ordering::Relaxed),
            chunks_failed: chunks_failed.load(Ordering::Relaxed),
        };

        info!("FXAR v2 import: {} files, {} bytes, {} chunks verified",
              stats.files_restored, stats.bytes_restored, stats.chunks_verified);

        Ok(stats)
    }

    /// Restore latest version of each file, stripping `{snapshot}/tree/` prefix.
    ///
    /// Default restore mode  --  behaves like `tar -xvf`. When an archive contains
    /// multiple snapshots with the same file, only the most recent version is
    /// extracted. Output paths are flat relative paths (no snapshot prefix).
    ///
    /// When `verbose` is true, prints per-file version info to stderr.
    pub fn restore_latest(
        &mut self,
        target: &Path,
        generate_sigs: bool,
        verbose: bool,
    ) -> io::Result<FxarImportStats> {
        let manifest = self.read_manifest()?;
        let index = self.read_chunk_index()?;

        // Build version map: rel_path -> [(snapshot_ts, manifest_index)]
        let mut version_map: std::collections::HashMap<String, Vec<(String, usize)>> =
            std::collections::HashMap::new();
        for (i, entry) in manifest.files.iter().enumerate() {
            let (snapshot, rel_path) = strip_snapshot_prefix(&entry.path);
            version_map
                .entry(rel_path.to_string())
                .or_default()
                .push((snapshot.to_string(), i));
        }

        // Select latest version per file, build version info
        let mut selected: Vec<(String, usize, Vec<String>)> = Vec::new();
        for (rel_path, mut versions) in version_map {
            versions.sort_by(|a, b| b.0.cmp(&a.0)); // latest first
            let all_snapshots: Vec<String> = versions.iter().rev().map(|(ts, _)| ts.clone()).collect();
            let (_, latest_idx) = versions[0];
            selected.push((rel_path, latest_idx, all_snapshots));
        }
        selected.sort_by(|a, b| a.0.cmp(&b.0));

        // Pre-flight disk space check on selected files only
        let total_bytes: u64 = selected.iter()
            .map(|(_, idx, _)| manifest.files[*idx].size)
            .sum();
        if let Some(avail) = check_available_space(target) {
            let needed = total_bytes + crate::constants::FXAR_RESTORE_HEADROOM_BYTES;
            if avail < needed {
                return Err(io::Error::other(
                    format!("insufficient disk space: {} available, {} needed", avail, needed),
                ));
            }
        }

        // Pre-create directories for stripped paths
        for (rel_path, _, _) in &selected {
            let dest = target.join(rel_path);
            if let Some(parent) = dest.parent() {
                std::fs::create_dir_all(parent)?;
            }
        }

        // Pipelined chunk loading  --  load ALL chunks sequentially (same as restore_all).
        // Skipping unneeded chunks introduces a concurrency race between the skip
        // counter and the producer's store() calls.
        let chunk_count = index.len();
        let chunk_store: Vec<parking_lot::RwLock<Option<Vec<u8>>>> =
            (0..chunk_count).map(|_| parking_lot::RwLock::new(None)).collect();
        let chunk_store = std::sync::Arc::new(chunk_store);
        let chunks_loaded = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));

        let flags = self.header.flags;
        let producer_index = index.clone();
        let mut raw_chunks: Vec<Vec<u8>> = Vec::with_capacity(chunk_count);
        for entry in &producer_index {
            if entry.compressed_size as u64 > MAX_FXAR_ALLOC {
                return Err(io::Error::new(
                    io::ErrorKind::InvalidData,
                    format!("chunk compressed size {} exceeds maximum", entry.compressed_size),
                ));
            }
            self.reader.seek(SeekFrom::Start(entry.offset))?;
            let mut compressed = vec![0u8; entry.compressed_size as usize];
            self.reader.read_exact(&mut compressed)?;
            raw_chunks.push(compressed);
        }

        let producer_store = chunk_store.clone();
        let producer_loaded = chunks_loaded.clone();
        let producer_handle = std::thread::spawn(move || {
            for (i, compressed) in raw_chunks.into_iter().enumerate() {
                let data = match decompress_chunk(&compressed, flags) {
                    Ok(d) => d,
                    Err(e) => {
                        warn!("chunk {} decompress failed: {}", i, e);
                        Vec::new()
                    }
                };
                let actual = blake3::hash(&data);
                if actual.as_bytes() != &producer_index[i].blake3_hash {
                    warn!("chunk {} BLAKE3 mismatch", i);
                }
                *producer_store[i].write() = Some(data);
                producer_loaded.store(i + 1, std::sync::atomic::Ordering::Release);
            }
        });

        #[cfg(feature = "nfs-bypass")]
        let nfs_pool: Option<crate::nfs::NfsClientPool> = init_nfs_pool(target);

        use rayon::prelude::*;
        use std::sync::atomic::{AtomicU64, Ordering};

        let files_restored = AtomicU64::new(0);
        let bytes_restored = AtomicU64::new(0);
        let chunks_verified = AtomicU64::new(0);
        let chunks_failed = AtomicU64::new(0);

        // Print version info header if verbose
        if verbose && !selected.is_empty() {
            let snap_count = manifest.snapshots.len();
            if snap_count > 1 {
                eprintln!("Archive contains {} snapshots, extracting latest version of each file", snap_count);
            }
        }

        selected.par_iter().enumerate().for_each(|(file_idx, (rel_path, manifest_idx, all_snapshots))| {
            let file_entry = &manifest.files[*manifest_idx];
            let dest = target.join(rel_path);

            // Wait for required chunks
            let max_chunk = file_entry.chunks.iter().copied().max().unwrap_or(0);
            let needed = max_chunk as usize + 1;
            while chunks_loaded.load(Ordering::Acquire) < needed {
                std::hint::spin_loop();
            }

            // Assemble file from chunks
            let mut data = Vec::with_capacity(file_entry.size as usize);
            for &chunk_idx in &file_entry.chunks {
                let guard = chunk_store[chunk_idx as usize].read();
                if let Some(chunk_data) = guard.as_ref() {
                    data.extend_from_slice(chunk_data);
                    chunks_verified.fetch_add(1, Ordering::Relaxed);
                } else {
                    chunks_failed.fetch_add(1, Ordering::Relaxed);
                    if verbose {
                        eprintln!("{}  FAIL (missing chunk)", rel_path);
                    }
                    return;
                }
            }

            // Verify whole-file BLAKE3
            let actual = blake3::hash(&data);
            let expected = decode_blake3_field(&file_entry.blake3).unwrap_or_default();
            if *actual.as_bytes() != expected {
                warn!("BLAKE3 mismatch for {}, skipping", rel_path);
                chunks_failed.fetch_add(1, Ordering::Relaxed);
                if verbose {
                    eprintln!("{}  FAIL (BLAKE3 mismatch)", rel_path);
                }
                return;
            }

            // Write file
            let wrote_via_nfs = write_file_with_pool(
                &dest, &data, file_entry, target, file_idx,
                #[cfg(feature = "nfs-bypass")]
                &nfs_pool,
            );

            if !wrote_via_nfs {
                if std::fs::write(&dest, &data).is_err() {
                    chunks_failed.fetch_add(1, Ordering::Relaxed);
                    return;
                }
                use std::os::unix::fs::PermissionsExt;
                let _ = std::fs::set_permissions(&dest, std::fs::Permissions::from_mode(file_entry.mode));
                if unsafe { libc::geteuid() } == 0 {
                    use nix::unistd::{chown, Uid, Gid};
                    let _ = chown(&dest, Some(Uid::from_raw(file_entry.uid)), Some(Gid::from_raw(file_entry.gid)));
                }
                let mtime = filetime::FileTime::from_unix_time(file_entry.mtime, 0);
                let _ = filetime::set_file_mtime(&dest, mtime);
            }

            for (key, hex_val) in &file_entry.xattr {
                if let Ok(val) = hex::decode(hex_val) {
                    if key.starts_with("user.foxing") {
                        let _ = crate::sidecar::set_metadata(&dest, key, &val);
                    } else {
                        let _ = xattr::set(&dest, key, &val);
                    }
                }
            }

            if generate_sigs {
                let sig = crate::sidecar::SyncSignature::compute_from_buffer(
                    &data, file_entry.mtime, 0,
                );
                let _ = crate::sidecar::set_sync_signature(&dest, &sig);
            }

            // Verbose: show version info
            if verbose {
                let restored_snap = strip_snapshot_prefix(&file_entry.path).0;
                let version_display = if all_snapshots.len() > 1 {
                    let formatted: Vec<String> = all_snapshots.iter().map(|ts| {
                        if ts == restored_snap { format!("{}*", ts) } else { ts.clone() }
                    }).collect();
                    format!("  [{}]", formatted.join(", "))
                } else {
                    String::new()
                };
                eprintln!("{}{}  {} bytes", rel_path, version_display, file_entry.size);
            }

            files_restored.fetch_add(1, Ordering::Relaxed);
            bytes_restored.fetch_add(file_entry.size, Ordering::Relaxed);
        });

        if generate_sigs {
            compute_dir_hashes_recursive(target);
        }

        producer_handle.join().map_err(|_|
            io::Error::other("chunk producer thread panicked")
        )?;

        let stats = FxarImportStats {
            files_restored: files_restored.load(Ordering::Relaxed),
            bytes_restored: bytes_restored.load(Ordering::Relaxed),
            chunks_verified: chunks_verified.load(Ordering::Relaxed),
            chunks_failed: chunks_failed.load(Ordering::Relaxed),
        };

        info!("FXAR v2 restore (latest): {} files, {} bytes, {} chunks verified",
              stats.files_restored, stats.bytes_restored, stats.chunks_verified);

        Ok(stats)
    }

    /// List all files in the archive without extracting.
    pub fn list_files(&mut self) -> io::Result<Vec<FxarManifestEntry>> {
        let manifest = self.read_manifest()?;
        Ok(manifest.files)
    }

    pub fn read_extension_directory(&mut self) -> io::Result<Option<FxarExtensionDirectory>> {
        if self.header.flags & FLAG_HAS_EXTENSION_DIR == 0 {
            return Ok(None);
        }
        let file_size = self.reader.seek(SeekFrom::End(0))?;
        if file_size < (HEADER_SIZE + FOOTER_SIZE + 8) as u64 {
            return Err(io::Error::new(io::ErrorKind::InvalidData, "archive too small for extension dir"));
        }
        // Sentinel is at file_size - FOOTER_SIZE(32) - 8
        self.reader.seek(SeekFrom::Start(file_size - FOOTER_SIZE as u64 - 8))?;
        let mut sentinel_buf = [0u8; 8];
        self.reader.read_exact(&mut sentinel_buf)?;
        let ext_dir_offset = u64::from_le_bytes(sentinel_buf);

        if ext_dir_offset < HEADER_SIZE as u64 || ext_dir_offset >= file_size - FOOTER_SIZE as u64 - 8 {
            return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid extension dir offset"));
        }
        self.reader.seek(SeekFrom::Start(ext_dir_offset))?;
        Ok(Some(FxarExtensionDirectory::read_from(&mut self.reader)?))
    }

    pub fn read_extension_section(&mut self, section_type: u32) -> io::Result<Option<Vec<u8>>> {
        let dir = match self.read_extension_directory()? {
            None => return Ok(None),
            Some(d) => d,
        };
        let entry = match dir.entries.iter().find(|e| e.section_type == section_type) {
            None => return Ok(None),
            Some(e) => e.clone(),
        };
        if entry.size > MAX_FXAR_ALLOC {
            return Err(io::Error::new(io::ErrorKind::InvalidData, "extension section too large"));
        }
        self.reader.seek(SeekFrom::Start(entry.offset))?;
        let mut data = vec![0u8; entry.size as usize];
        self.reader.read_exact(&mut data)?;
        let actual = *blake3::hash(&data).as_bytes();
        if actual != entry.checksum {
            return Err(io::Error::new(io::ErrorKind::InvalidData, "extension section BLAKE3 mismatch"));
        }
        Ok(Some(data))
    }
}

// -----------------------------------------------------------------------
// Stream reader (for piped imports)
// -----------------------------------------------------------------------

/// Read an FXAR v2 archive from a non-seekable stream.
pub fn read_archive_stream<R: Read>(
    mut reader: R,
    target: &Path,
    generate_sigs: bool,
) -> io::Result<FxarImportStats> {
    let header = FxarHeader::read_from(&mut reader)?;
    if header.version != FXAR_VERSION {
        return Err(io::Error::new(
            io::ErrorKind::InvalidData,
            format!("unsupported FXAR version: {}", header.version),
        ));
    }

    // Streaming mode: read manifest, then index, then chunks sequentially
    // Read manifest
    let mut size_buf = [0u8; 8];
    reader.read_exact(&mut size_buf)?;
    let compressed_len = u64::from_le_bytes(size_buf);
    if compressed_len > MAX_FXAR_ALLOC {
        return Err(io::Error::new(
            io::ErrorKind::InvalidData,
            format!("FXAR manifest compressed size {} exceeds maximum {}", compressed_len, MAX_FXAR_ALLOC),
        ));
    }
    let compressed_len = compressed_len as usize;
    reader.read_exact(&mut size_buf)?;
    let _uncompressed_len = u64::from_le_bytes(size_buf) as usize;

    let mut manifest_compressed = vec![0u8; compressed_len];
    reader.read_exact(&mut manifest_compressed)?;
    let manifest_json = decompress_chunk(&manifest_compressed, header.flags)?;
    let manifest: FxarManifest = serde_json::from_slice(&manifest_json)
        .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, format!("manifest: {}", e)))?;

    // Read chunk index
    let chunk_count = header.chunk_count as usize;
    let mut index = Vec::with_capacity(chunk_count);
    for _ in 0..chunk_count {
        index.push(ChunkIndexEntry::read_from(&mut reader)?);
    }

    // Read all chunk data into memory (streaming mode  --  can't seek)
    let mut chunk_store: Vec<Vec<u8>> = Vec::with_capacity(chunk_count);
    for entry in &index {
        if entry.compressed_size as u64 > MAX_FXAR_ALLOC {
            return Err(io::Error::new(
                io::ErrorKind::InvalidData,
                format!("FXAR chunk compressed size {} exceeds maximum {}", entry.compressed_size, MAX_FXAR_ALLOC),
            ));
        }
        let mut compressed = vec![0u8; entry.compressed_size as usize];
        reader.read_exact(&mut compressed)?;
        let data = decompress_chunk(&compressed, header.flags)?;

        // Verify BLAKE3
        let actual = blake3::hash(&data);
        if actual.as_bytes() != &entry.blake3_hash {
            return Err(io::Error::new(
                io::ErrorKind::InvalidData,
                format!("chunk BLAKE3 mismatch at index {}", chunk_store.len()),
            ));
        }
        chunk_store.push(data);
    }

    // Pre-flight disk space check
    let total_bytes: u64 = manifest.files.iter().map(|f| f.size).sum();
    if let Some(avail) = check_available_space(target) {
        let needed = total_bytes + crate::constants::FXAR_RESTORE_HEADROOM_BYTES;
        if avail < needed {
            return Err(io::Error::other(
                format!("insufficient disk space: {} bytes available, {} bytes needed",
                        avail, needed),
            ));
        }
    }

    // Pre-create all parent directories (must be sequential)
    for file_entry in &manifest.files {
        let dest = target.join(&file_entry.path);
        if let Some(parent) = dest.parent() {
            std::fs::create_dir_all(parent)?;
        }
    }

    // NFS bypass pool
    #[cfg(feature = "nfs-bypass")]
    let nfs_pool: Option<crate::nfs::NfsClientPool> = init_nfs_pool(target);

    // Reconstruct files in parallel (chunk assembly + BLAKE3 verify + write)
    use rayon::prelude::*;
    use std::sync::atomic::{AtomicU64, Ordering};

    let files_restored = AtomicU64::new(0);
    let bytes_restored = AtomicU64::new(0);
    let chunks_verified = AtomicU64::new(0);
    let chunks_failed = AtomicU64::new(0);

    manifest.files.par_iter().enumerate().for_each(|(file_idx, file_entry)| {
        let dest = target.join(&file_entry.path);

        let mut data = Vec::with_capacity(file_entry.size as usize);
        for &chunk_idx in &file_entry.chunks {
            data.extend_from_slice(&chunk_store[chunk_idx as usize]);
            chunks_verified.fetch_add(1, Ordering::Relaxed);
        }

        // Verify whole-file hash (accepts both hex and CID manifest fields)
        let actual = blake3::hash(&data);
        let expected = decode_blake3_field(&file_entry.blake3).unwrap_or_default();
        if *actual.as_bytes() != expected {
            warn!("BLAKE3 mismatch for {}, skipping", file_entry.path);
            chunks_failed.fetch_add(1, Ordering::Relaxed);
            return;
        }

        // Write file  --  NFS pool bypass or VFS fallback
        let wrote_via_nfs = write_file_with_pool(
            &dest, &data, file_entry, target, file_idx,
            #[cfg(feature = "nfs-bypass")]
            &nfs_pool,
        );

        if !wrote_via_nfs {
            if std::fs::write(&dest, &data).is_err() {
                chunks_failed.fetch_add(1, Ordering::Relaxed);
                return;
            }
            use std::os::unix::fs::PermissionsExt;
            let _ = std::fs::set_permissions(&dest, std::fs::Permissions::from_mode(file_entry.mode));
            let mtime = filetime::FileTime::from_unix_time(file_entry.mtime, 0);
            let _ = filetime::set_file_mtime(&dest, mtime);
        }

        for (key, hex_val) in &file_entry.xattr {
            if let Ok(val) = hex::decode(hex_val) {
                if key.starts_with("user.foxing") {
                    let _ = crate::sidecar::set_metadata(&dest, key, &val);
                } else {
                    let _ = xattr::set(&dest, key, &val);
                }
            }
        }

        // Generate foxingd-compatible signatures if requested
        if generate_sigs {
            let sig = crate::sidecar::SyncSignature::compute_from_buffer(
                &data, file_entry.mtime, 0,
            );
            let _ = crate::sidecar::set_sync_signature(&dest, &sig);
        }

        files_restored.fetch_add(1, Ordering::Relaxed);
        bytes_restored.fetch_add(file_entry.size, Ordering::Relaxed);
    });

    let stats = FxarImportStats {
        files_restored: files_restored.load(Ordering::Relaxed),
        bytes_restored: bytes_restored.load(Ordering::Relaxed),
        chunks_verified: chunks_verified.load(Ordering::Relaxed),
        chunks_failed: chunks_failed.load(Ordering::Relaxed),
    };

    // Post-restore: compute dir hashes for adaptive pruning
    if generate_sigs {
        compute_dir_hashes_recursive(target);
    }

    info!("FXAR v2 stream import: {} files, {} bytes", stats.files_restored, stats.bytes_restored);
    Ok(stats)
}

// -----------------------------------------------------------------------
// Format auto-detection
// -----------------------------------------------------------------------

/// Detect archive format by reading magic bytes.
/// Returns "fxar2" for FXAR v2, "tar" for tar archives, "unknown" otherwise.
pub fn detect_format<R: Read>(reader: &mut R) -> io::Result<(&'static str, [u8; 4])> {
    let mut magic = [0u8; 4];
    reader.read_exact(&mut magic)?;

    if &magic == FXAR_MAGIC {
        return Ok(("fxar2", magic));
    }

    // tar magic is at offset 257 ("ustar"), but first bytes are often
    // a filename or null. Check for common compressed stream headers.
    // zstd: 0x28 0xB5 0x2F 0xFD
    if magic == [0x28, 0xB5, 0x2F, 0xFD] {
        return Ok(("zstd-stream", magic));
    }
    // gzip: 0x1F 0x8B
    if magic[0] == 0x1F && magic[1] == 0x8B {
        return Ok(("gzip-stream", magic));
    }
    // xz: 0xFD 0x37 0x7A 0x58
    if magic == [0xFD, 0x37, 0x7A, 0x58] {
        return Ok(("xz-stream", magic));
    }
    // lz4: 0x04 0x22 0x4D 0x18
    if magic == [0x04, 0x22, 0x4D, 0x18] {
        return Ok(("lz4-stream", magic));
    }

    // Assume tar (raw or old-style without magic at offset 257)
    Ok(("tar", magic))
}

/// Inspect an FXAR v2 archive and return summary info.
pub fn inspect_archive<R: Read + Seek>(mut reader: R) -> io::Result<FxarInspectResult> {
    let mut fxar = FxarReader::open(&mut reader)?;
    let manifest = fxar.read_manifest()?;
    let index = fxar.read_chunk_index()?;

    let total_files = manifest.files.len() as u64;
    let total_apparent: u64 = manifest.files.iter().map(|f| f.size).sum();
    let chunk_count = index.len() as u64;
    let total_chunk_bytes: u64 = index.iter().map(|e| e.size as u64).sum();
    let total_compressed: u64 = index.iter().map(|e| e.compressed_size as u64).sum();

    // Count unique file hashes for file-level dedup stats
    let unique_files: HashSet<&str> = manifest.files.iter().map(|f| f.blake3.as_str()).collect();

    let extension_sections = match fxar.read_extension_directory() {
        Ok(Some(dir)) => dir.entries.iter().map(|e| (e.section_type, e.size)).collect(),
        _ => Vec::new(),
    };

    Ok(FxarInspectResult {
        version: fxar.header.version,
        snapshots: manifest.snapshots,
        total_files,
        unique_files: unique_files.len() as u64,
        total_apparent_bytes: total_apparent,
        chunk_count,
        total_chunk_bytes,
        total_compressed_bytes: total_compressed,
        dedup_ratio: if total_apparent > 0 {
            1.0 - (total_compressed as f64 / total_apparent as f64)
        } else { 0.0 },
        files: manifest.files,
        extension_sections,
    })
}

#[derive(Debug, Serialize, Deserialize)]
/// Result of inspecting an FXAR archive without extracting.
pub struct FxarInspectResult {
    pub version: u32,
    pub snapshots: Vec<String>,
    pub total_files: u64,
    pub unique_files: u64,
    pub total_apparent_bytes: u64,
    pub chunk_count: u64,
    pub total_chunk_bytes: u64,
    pub total_compressed_bytes: u64,
    pub dedup_ratio: f64,
    pub files: Vec<FxarManifestEntry>,
    #[serde(default)]
    pub extension_sections: Vec<(u32, u64)>,
}

// -----------------------------------------------------------------------
// Compression helpers
// -----------------------------------------------------------------------

/// Check available disk space using statvfs.
fn check_available_space(path: &Path) -> Option<u64> {
    use std::ffi::CString;
    let c_path = CString::new(path.to_string_lossy().as_bytes()).ok()?;
    // SAFETY: statvfs is a C struct where all-zeros is a valid initial state.
    let mut stat: libc::statvfs = unsafe { std::mem::zeroed() };
    // SAFETY: c_path is a valid null-terminated C string, stat is a valid pointer.
    let rc = unsafe { libc::statvfs(c_path.as_ptr(), &mut stat) };
    if rc == 0 {
        Some(stat.f_bavail * stat.f_bsize)
    } else {
        None
    }
}

/// Walk restored directory tree and compute BLAKE3 dir hashes for adaptive pruning.
/// Stores dir hashes via sidecar (xattr with .foxing_meta fallback).
fn compute_dir_hashes_recursive(root: &Path) {
    for entry in walkdir::WalkDir::new(root)
        .contents_first(true) // bottom-up: children before parents
        .follow_links(false)
    {
        let entry = match entry { Ok(e) => e, Err(_) => continue };
        if !entry.file_type().is_dir() { continue; }
        let dir_path = entry.path();
        // Skip .foxing_versions and .foxing_meta directories
        let name = dir_path.file_name().map(|n| n.to_string_lossy()).unwrap_or_default();
        if name.starts_with(".foxing") { continue; }

        if let Some(hash) = crate::hashing::compute_dir_hash_from_path(dir_path) {
            let _ = crate::sidecar::set_metadata(dir_path, "dir_hash", &hash);
        }
    }
}

/// Initialize NFS client pool if target is NFSv4.2 and bypass is enabled.
#[cfg(feature = "nfs-bypass")]
fn init_nfs_pool(target: &Path) -> Option<crate::nfs::NfsClientPool> {
    let nfs_env = std::env::var("FOXING_NFS_BYPASS").unwrap_or_else(|_| "1".to_string());
    if nfs_env == "0" { return None; }
    crate::nfs::mount::probe_nfs_bypass(target).and_then(|info| {
        match crate::nfs::NfsClientPool::new(&info, 4) {
            Ok(pool) => {
                info!("FXAR import: NFS bypass pool ({} sessions) -> {}", pool.len(), info.server_addr);
                Some(pool)
            }
            Err(e) => {
                info!("FXAR import: NFS bypass unavailable ({}), using VFS", e);
                None
            }
        }
    })
}

/// Write a file via NFS pool bypass (round-robin session selection).
/// Returns true if NFS bypass succeeded, false for VFS fallback.
#[allow(unused_variables)]
fn write_file_with_pool(
    dest: &Path,
    data: &[u8],
    file_entry: &FxarManifestEntry,
    target: &Path,
    thread_hint: usize,
    #[cfg(feature = "nfs-bypass")]
    nfs_pool: &Option<crate::nfs::NfsClientPool>,
) -> bool {
    #[cfg(feature = "nfs-bypass")]
    {
        if let Some(pool) = nfs_pool
            && data.len() <= crate::nfs::NFS_BYPASS_WRITE_MAX as usize {
                let parent = dest.parent().unwrap_or(target);
                let client_lock = pool.get(thread_hint);
                let mut client = client_lock.lock();
                if let Ok(handle) = client.get_or_resolve_handle(parent) {
                    let fname = dest.file_name()
                        .map(|f| f.to_string_lossy().into_owned())
                        .unwrap_or_default();
                    match client.write_file(
                        &handle, &fname, data,
                        file_entry.mode, file_entry.uid, file_entry.gid,
                        (file_entry.mtime, 0),
                    ) {
                        Ok(()) => return true,
                        Err(e) => {
                            tracing::debug!("NFS bypass failed for {}: {}, attempting recovery", fname, e);
                            if client.recover_session().is_ok()
                                && let Ok(h) = client.get_or_resolve_handle(parent)
                                    && client.write_file(
                                        &h, &fname, data,
                                        file_entry.mode, file_entry.uid, file_entry.gid,
                                        (file_entry.mtime, 0),
                                    ).is_ok() {
                                        return true;
                                    }
                            tracing::debug!("NFS bypass recovery failed for {}", fname);
                        }
                    }
                }
            }
    }
    false
}

fn compress_flag(compress: &str) -> u32 {
    match compress {
        "none" => FLAG_COMPRESS_NONE,
        s if s.starts_with("zstd") => FLAG_COMPRESS_ZSTD,
        "lz4" => FLAG_COMPRESS_LZ4,
        "gzip" => FLAG_COMPRESS_GZIP,
        s if s.starts_with("xz") => FLAG_COMPRESS_XZ,
        _ => FLAG_COMPRESS_ZSTD,
    }
}

pub(crate) fn compress_chunk(data: &[u8], compress: &str) -> Vec<u8> {
    match compress {
        "none" => data.to_vec(),
        s if s.starts_with("zstd") => {
            let level = s.strip_prefix("zstd:").and_then(|l| l.parse().ok()).unwrap_or(3);
            zstd::bulk::compress(data, level).unwrap_or_else(|_| data.to_vec())
        }
        "lz4" => lz4_flex::compress_prepend_size(data),
        "gzip" => {
            use flate2::write::GzEncoder;
            let mut enc = GzEncoder::new(Vec::new(), flate2::Compression::default());
            if let Err(e) = enc.write_all(data) {
                warn!("gzip compression write failed: {e}, falling back to uncompressed");
                return data.to_vec();
            }
            enc.finish().unwrap_or_else(|_| data.to_vec())
        }
        _ => zstd::bulk::compress(data, 3).unwrap_or_else(|_| data.to_vec()),
    }
}

pub(crate) fn decompress_chunk(data: &[u8], flags: u32) -> io::Result<Vec<u8>> {
    let compression = flags & 0x1F;
    match compression {
        FLAG_COMPRESS_NONE => Ok(data.to_vec()),
        FLAG_COMPRESS_ZSTD => {
            zstd::bulk::decompress(data, 16 * 1024 * 1024) // 16MB max per chunk
                .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, format!("zstd: {}", e)))
        }
        FLAG_COMPRESS_LZ4 => {
            lz4_flex::decompress_size_prepended(data)
                .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, format!("lz4: {}", e)))
        }
        FLAG_COMPRESS_GZIP => {
            use flate2::read::GzDecoder;
            let mut dec = GzDecoder::new(data);
            let mut out = Vec::new();
            dec.read_to_end(&mut out)?;
            Ok(out)
        }
        FLAG_COMPRESS_XZ => {
            let mut dec = liblzma::read::XzDecoder::new(data);
            let mut out = Vec::new();
            dec.read_to_end(&mut out)?;
            Ok(out)
        }
        _ => Err(io::Error::new(io::ErrorKind::InvalidData, format!("unknown compression: {}", flags))),
    }
}

// -----------------------------------------------------------------------
// Tests
// -----------------------------------------------------------------------

#[cfg(test)]
mod tests {
    #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
    use super::*;

    #[test]
    fn test_header_roundtrip() {
        let chunker = GearChunker::default();
        let mut header = FxarHeader::new(FLAG_COMPRESS_ZSTD, &chunker);
        header.index_offset = 12345;
        header.manifest_offset = 64;
        header.chunk_count = 42;

        let mut buf = Vec::new();
        header.write_to(&mut buf).unwrap();
        assert_eq!(buf.len(), HEADER_SIZE);

        let parsed = FxarHeader::read_from(&mut &buf[..]).unwrap();
        assert_eq!(parsed.magic, *FXAR_MAGIC);
        assert_eq!(parsed.version, FXAR_VERSION);
        assert_eq!(parsed.flags, FLAG_COMPRESS_ZSTD);
        assert_eq!(parsed.index_offset, 12345);
        assert_eq!(parsed.manifest_offset, 64);
        assert_eq!(parsed.chunk_count, 42);
        assert_eq!(parsed.chunk_min, chunker.min as u32);
        assert_eq!(parsed.chunk_avg, chunker.avg as u32);
        assert_eq!(parsed.chunk_max, chunker.max as u32);
    }

    #[test]
    fn test_header_bad_magic() {
        let buf = [0u8; HEADER_SIZE];
        let result = FxarHeader::read_from(&mut &buf[..]);
        assert!(result.is_err());
    }

    #[test]
    fn test_chunk_index_entry_roundtrip() {
        let entry = ChunkIndexEntry {
            blake3_hash: [0xAB; 32],
            size: 65536,
            offset: 999999,
            compressed_size: 32768,
        };

        let mut buf = Vec::new();
        entry.write_to(&mut buf).unwrap();
        assert_eq!(buf.len(), CHUNK_INDEX_ENTRY_SIZE);

        let parsed = ChunkIndexEntry::read_from(&mut &buf[..]).unwrap();
        assert_eq!(parsed.blake3_hash, [0xAB; 32]);
        assert_eq!(parsed.size, 65536);
        assert_eq!(parsed.offset, 999999);
        assert_eq!(parsed.compressed_size, 32768);
    }

    #[test]
    fn test_footer_roundtrip() {
        let checksum = FxarFooter::compute_checksum(64, 12345, 42);
        let footer = FxarFooter {
            magic: *FXAR_FOOTER_MAGIC,
            manifest_offset: 64,
            index_offset: 12345,
            chunk_count: 42,
            checksum,
        };

        let mut buf = Vec::new();
        footer.write_to(&mut buf).unwrap();
        assert_eq!(buf.len(), FOOTER_SIZE);

        let parsed = FxarFooter::read_from(&mut &buf[..]).unwrap();
        assert_eq!(parsed.manifest_offset, 64);
        assert_eq!(parsed.index_offset, 12345);
        assert_eq!(parsed.chunk_count, 42);
        assert_eq!(parsed.checksum, checksum);
    }

    #[test]
    fn test_manifest_json_roundtrip() {
        let manifest = FxarManifest {
            version: 2,
            created: "2026-03-13T00:00:00Z".into(),
            files: vec![FxarManifestEntry {
                path: "snap1/tree/data/test.db".into(),
                size: 104857600,
                mode: 0o100644,
                mtime: 1773484800,
                uid: 1000,
                gid: 1000,
                blake3: crate::cid::blake3_hex_to_cid_string(&"a1b2c3d4".repeat(8)).unwrap(),
                chunks: vec![0, 1, 2, 3],
                xattr: std::collections::HashMap::new(),
            }],
            snapshots: vec!["2026-03-12T084500".into()],
        };

        let json = serde_json::to_vec(&manifest).unwrap();
        let parsed: FxarManifest = serde_json::from_slice(&json).unwrap();
        assert_eq!(parsed.version, 2);
        assert_eq!(parsed.files.len(), 1);
        assert_eq!(parsed.files[0].chunks, vec![0, 1, 2, 3]);
    }

    #[test]
    fn test_compress_decompress_roundtrip() {
        let data = b"hello world, this is test data for compression";

        for compress in &["none", "zstd", "zstd:1", "lz4", "gzip"] {
            let compressed = compress_chunk(data, compress);
            let flags = compress_flag(compress);
            let decompressed = decompress_chunk(&compressed, flags).unwrap();
            assert_eq!(decompressed, data, "roundtrip failed for {}", compress);
        }
    }

    #[test]
    fn test_export_stats_dedup_ratio() {
        let mut stats = FxarExportStats::default();
        stats.total_apparent_bytes = 1000;
        stats.archive_bytes = 100;
        assert!((stats.dedup_ratio() - 0.9).abs() < 0.001);

        stats.total_apparent_bytes = 0;
        assert_eq!(stats.dedup_ratio(), 0.0);
    }

    #[test]
    fn test_export_import_roundtrip() {
        use tempfile::TempDir;

        // Create a mock version store with a snapshot
        let source_dir = TempDir::new().unwrap();
        let target_dir = TempDir::new().unwrap();

        // Create "target" with version store structure
        let vs_root = target_dir.path().join(".foxing_versions");
        let snap_dir = vs_root.join("2026-03-12T084500");
        let tree_dir = snap_dir.join("tree");
        std::fs::create_dir_all(&tree_dir).unwrap();

        // Create test files in the snapshot tree
        let test_data = b"hello world, this is test data for FXAR v2 export/import roundtrip";
        std::fs::write(tree_dir.join("file1.txt"), test_data).unwrap();
        std::fs::write(tree_dir.join("file2.txt"), test_data).unwrap(); // duplicate for dedup
        std::fs::write(tree_dir.join("file3.dat"), vec![0xABu8; 10_000]).unwrap();

        // Write summary.json so scan_snapshots finds it
        let summary = serde_json::json!({
            "timestamp": "2026-03-12T08:45:00Z",
            "status": "success",
            "type": "full",
            "source": "/test",
            "trigger": "test",
            "files": 3,
            "size_bytes": 10066,
            "disk_usage_bytes": 10066,
            "savings_pct": 0.0,
            "elapsed_ms": 1
        });
        std::fs::write(snap_dir.join("summary.json"),
            serde_json::to_string_pretty(&summary).unwrap()).unwrap();

        // Export to FXAR v2
        let store = crate::version_store::VersionStore::open(target_dir.path());
        let archive_path = source_dir.path().join("test.fxar");
        let file = std::fs::File::create(&archive_path).unwrap();
        let stats = write_archive_seekable(&store, file, "zstd", None).unwrap();

        assert_eq!(stats.snapshots_exported, 1);
        assert_eq!(stats.total_files, 3);
        assert!(stats.unique_chunks > 0);
        // file1.txt and file2.txt are identical, so dedup_chunks > 0
        assert!(stats.dedup_chunks > 0, "expected dedup, got {} dedup chunks", stats.dedup_chunks);

        // Verify archive magic
        let archive_data = std::fs::read(&archive_path).unwrap();
        assert_eq!(&archive_data[..4], b"FXAR");

        // Inspect
        let f = std::fs::File::open(&archive_path).unwrap();
        let inspect = inspect_archive(f).unwrap();
        assert_eq!(inspect.version, 2);
        assert_eq!(inspect.total_files, 3);
        assert!(inspect.chunk_count > 0);

        // Import (restore) to a new directory
        let restore_dir = TempDir::new().unwrap();
        let f = std::fs::File::open(&archive_path).unwrap();
        let mut reader = FxarReader::open(f).unwrap();
        let import_stats = reader.restore_all(restore_dir.path(), false).unwrap();

        assert_eq!(import_stats.files_restored, 3);
        assert_eq!(import_stats.chunks_failed, 0);

        // Verify restored files match originals
        let restored1 = std::fs::read(
            restore_dir.path().join("2026-03-12T084500/tree/file1.txt")
        ).unwrap();
        assert_eq!(restored1, test_data);

        let restored3 = std::fs::read(
            restore_dir.path().join("2026-03-12T084500/tree/file3.dat")
        ).unwrap();
        assert_eq!(restored3, vec![0xABu8; 10_000]);
    }

    #[test]
    fn test_selective_restore() {
        use tempfile::TempDir;

        let target_dir = TempDir::new().unwrap();
        let vs_root = target_dir.path().join(".foxing_versions");
        let snap_dir = vs_root.join("2026-03-12T084500");
        let tree_dir = snap_dir.join("tree");
        std::fs::create_dir_all(tree_dir.join("subdir")).unwrap();

        std::fs::write(tree_dir.join("keep.txt"), b"keep this").unwrap();
        std::fs::write(tree_dir.join("subdir/nested.txt"), b"nested data").unwrap();
        let summary = serde_json::json!({
            "timestamp": "2026-03-12T08:45:00Z", "status": "success", "type": "full",
            "source": "/test", "trigger": "test", "files": 2, "size_bytes": 20,
            "disk_usage_bytes": 20, "savings_pct": 0.0, "elapsed_ms": 1
        });
        std::fs::write(snap_dir.join("summary.json"),
            serde_json::to_string_pretty(&summary).unwrap()).unwrap();

        // Export
        let store = crate::version_store::VersionStore::open(target_dir.path());
        let archive_path = target_dir.path().join("test.fxar");
        let file = std::fs::File::create(&archive_path).unwrap();
        write_archive_seekable(&store, file, "none", None).unwrap();

        // Selective restore  --  just keep.txt
        let f = std::fs::File::open(&archive_path).unwrap();
        let mut reader = FxarReader::open(f).unwrap();
        let manifest = reader.read_manifest().unwrap();

        let keep_entry = manifest.files.iter().find(|f| f.path.contains("keep.txt")).unwrap();
        let data = reader.restore_file(&keep_entry.path).unwrap();
        assert_eq!(data, b"keep this");
    }

    #[test]
    fn test_stream_roundtrip() {
        use tempfile::TempDir;

        let target_dir = TempDir::new().unwrap();
        let vs_root = target_dir.path().join(".foxing_versions");
        let snap_dir = vs_root.join("2026-03-12T090000");
        let tree_dir = snap_dir.join("tree");
        std::fs::create_dir_all(&tree_dir).unwrap();
        std::fs::write(tree_dir.join("stream_test.bin"), vec![0x55u8; 5000]).unwrap();
        let summary = serde_json::json!({
            "timestamp": "2026-03-12T09:00:00Z", "status": "success", "type": "full",
            "source": "/test", "trigger": "test", "files": 1, "size_bytes": 5000,
            "disk_usage_bytes": 5000, "savings_pct": 0.0, "elapsed_ms": 1
        });
        std::fs::write(snap_dir.join("summary.json"),
            serde_json::to_string_pretty(&summary).unwrap()).unwrap();

        // Export to in-memory buffer (simulates streaming/pipe)
        let store = crate::version_store::VersionStore::open(target_dir.path());
        let mut buf = Vec::new();
        write_archive(&store, &mut buf, "zstd", None).unwrap();

        // Import from buffer via stream reader
        let restore_dir = TempDir::new().unwrap();
        let stats = read_archive_stream(std::io::Cursor::new(buf), restore_dir.path(), false).unwrap();

        assert_eq!(stats.files_restored, 1);
        assert_eq!(stats.chunks_failed, 0);

        let restored = std::fs::read(
            restore_dir.path().join("2026-03-12T090000/tree/stream_test.bin")
        ).unwrap();
        assert_eq!(restored, vec![0x55u8; 5000]);
    }

    #[test]
    fn test_streaming_archive_seekable_open() {
        use tempfile::TempDir;

        let target_dir = TempDir::new().unwrap();
        let vs_root = target_dir.path().join(".foxing_versions");
        let snap_dir = vs_root.join("2026-08-03T020000");
        let tree_dir = snap_dir.join("tree");
        std::fs::create_dir_all(&tree_dir).unwrap();
        std::fs::write(tree_dir.join("test.txt"), b"footer fallback test data").unwrap();
        let summary = serde_json::json!({
            "timestamp": "2026-08-03T02:00:00Z", "status": "success", "type": "full",
            "source": "/test", "trigger": "test", "files": 1, "size_bytes": 24,
            "disk_usage_bytes": 24, "savings_pct": 0.0, "elapsed_ms": 1
        });
        std::fs::write(snap_dir.join("summary.json"),
            serde_json::to_string_pretty(&summary).unwrap()).unwrap();

        let store = crate::version_store::VersionStore::open(target_dir.path());
        let mut buf = Vec::new();
        write_archive(&store, &mut buf, "none", None).unwrap();

        // Verify header has placeholder offsets (streaming mode)
        assert_eq!(buf[40], 0, "manifest_offset byte should be 0 in streaming header");

        // Verify footer magic at end
        let footer_start = buf.len() - FOOTER_SIZE;
        assert_eq!(&buf[footer_start..footer_start+4], b"FXAF", "footer magic must be FXAF");

        // Open with FxarReader (seekable)  --  footer fallback should patch header
        let cursor = std::io::Cursor::new(buf);
        let mut reader = FxarReader::open(cursor).unwrap();

        // Header should now have real offsets from footer
        assert!(reader.header.manifest_offset >= HEADER_SIZE as u64,
            "manifest_offset should be patched from footer, got {}",
            reader.header.manifest_offset);

        let manifest = reader.read_manifest().unwrap();
        assert_eq!(manifest.files.len(), 1, "manifest should have 1 file");
        assert!(manifest.files[0].path.contains("test.txt"));
    }

    #[test]
    fn test_restore_latest_from_streaming_archive() {
        use tempfile::TempDir;

        let target_dir = TempDir::new().unwrap();
        let vs_root = target_dir.path().join(".foxing_versions");
        let snap_dir = vs_root.join("2026-08-03T030000");
        let tree_dir = snap_dir.join("tree");
        std::fs::create_dir_all(&tree_dir).unwrap();
        std::fs::write(tree_dir.join("hello.txt"), b"restore latest from streaming").unwrap();
        let summary = serde_json::json!({
            "timestamp": "2026-08-03T03:00:00Z", "status": "success", "type": "full",
            "source": "/test", "trigger": "test", "files": 1, "size_bytes": 28,
            "disk_usage_bytes": 28, "savings_pct": 0.0, "elapsed_ms": 1
        });
        std::fs::write(snap_dir.join("summary.json"),
            serde_json::to_string_pretty(&summary).unwrap()).unwrap();

        let store = crate::version_store::VersionStore::open(target_dir.path());
        let mut buf = Vec::new();
        write_archive(&store, &mut buf, "none", None).unwrap();

        let restore_dir = TempDir::new().unwrap();
        let cursor = std::io::Cursor::new(buf);
        let mut reader = FxarReader::open(cursor).unwrap();
        let stats = reader.restore_latest(restore_dir.path(), false, false).unwrap();

        assert_eq!(stats.files_restored, 1);
        assert_eq!(stats.chunks_failed, 0);

        // Should be restored with flat path (no snapshot/tree/ prefix)
        let restored = restore_dir.path().join("hello.txt");
        assert!(restored.exists(), "hello.txt should exist at flat path");
        assert_eq!(std::fs::read(&restored).unwrap(), b"restore latest from streaming");
    }

    #[test]
    fn test_dedup_across_snapshots() {
        use tempfile::TempDir;

        let target_dir = TempDir::new().unwrap();
        let vs_root = target_dir.path().join(".foxing_versions");

        // Create 2 snapshots with mostly identical files
        for (snap_ts, change_byte) in &[("2026-03-12T080000", 0xAAu8), ("2026-03-12T090000", 0xBBu8)] {
            let snap_dir = vs_root.join(snap_ts);
            let tree_dir = snap_dir.join("tree");
            std::fs::create_dir_all(&tree_dir).unwrap();

            // Identical file across snapshots
            std::fs::write(tree_dir.join("stable.dat"), vec![0x42u8; 50_000]).unwrap();
            // File that changes between snapshots
            std::fs::write(tree_dir.join("changing.dat"), vec![*change_byte; 10_000]).unwrap();

            let summary = serde_json::json!({
                "timestamp": format!("{}Z", snap_ts.replace('T', "T").replace("080000", "08:00:00").replace("090000", "09:00:00")),
                "status": "success", "type": "full",
                "source": "/test", "trigger": "test", "files": 2, "size_bytes": 60000,
                "disk_usage_bytes": 60000, "savings_pct": 0.0, "elapsed_ms": 1
            });
            std::fs::write(snap_dir.join("summary.json"),
                serde_json::to_string_pretty(&summary).unwrap()).unwrap();
        }

        let store = crate::version_store::VersionStore::open(target_dir.path());
        let mut buf = Vec::new();
        let stats = write_archive(&store, &mut buf, "none", None).unwrap();

        assert_eq!(stats.snapshots_exported, 2);
        assert_eq!(stats.total_files, 4);
        // stable.dat is identical across snapshots -> its chunks should be deduped
        assert!(stats.dedup_chunks > 0,
            "expected cross-snapshot dedup, got {} dedup chunks", stats.dedup_chunks);
    }

    #[test]
    fn test_metadata_roundtrip() {
        use tempfile::TempDir;
        use std::os::unix::fs::{MetadataExt, PermissionsExt};

        let target_dir = TempDir::new().unwrap();
        let vs_root = target_dir.path().join(".foxing_versions");
        let snap_dir = vs_root.join("2026-03-14T080000");
        let tree_dir = snap_dir.join("tree");
        std::fs::create_dir_all(&tree_dir).unwrap();

        // Create file with specific mode and mtime
        let test_path = tree_dir.join("meta_test.txt");
        std::fs::write(&test_path, b"metadata test").unwrap();
        std::fs::set_permissions(&test_path, std::fs::Permissions::from_mode(0o755)).unwrap();
        let set_mtime = filetime::FileTime::from_unix_time(1773484800, 0);
        filetime::set_file_mtime(&test_path, set_mtime).unwrap();

        let summary = serde_json::json!({
            "timestamp": "2026-03-14T08:00:00Z", "status": "success", "type": "full",
            "source": "/test", "trigger": "test", "files": 1, "size_bytes": 13,
            "disk_usage_bytes": 13, "savings_pct": 0.0, "elapsed_ms": 1
        });
        std::fs::write(snap_dir.join("summary.json"),
            serde_json::to_string_pretty(&summary).unwrap()).unwrap();

        // Export
        let store = crate::version_store::VersionStore::open(target_dir.path());
        let archive_path = target_dir.path().join("test.fxar");
        let file = std::fs::File::create(&archive_path).unwrap();
        write_archive_seekable(&store, file, "none", None).unwrap();

        // Import to new location
        let restore_dir = TempDir::new().unwrap();
        let f = std::fs::File::open(&archive_path).unwrap();
        let mut reader = FxarReader::open(f).unwrap();
        reader.restore_all(restore_dir.path(), false).unwrap();

        // Verify metadata
        let restored = restore_dir.path().join("2026-03-14T080000/tree/meta_test.txt");
        assert!(restored.exists());
        let meta = std::fs::metadata(&restored).unwrap();
        assert_eq!(meta.permissions().mode() & 0o777, 0o755, "mode mismatch");
        assert_eq!(meta.mtime(), 1773484800, "mtime mismatch");
        assert_eq!(std::fs::read(&restored).unwrap(), b"metadata test");
    }

    /// Documents GAP: FXAR manifest stores uid/gid per file (FxarManifestEntry.uid,
    /// FxarManifestEntry.gid) but the VFS restore path at line ~1204-1207 only applies
    /// mode and mtime  --  it does NOT call chown to restore uid/gid.
    ///
    /// The NFS bypass restore path DOES pass uid/gid (line ~1653), so this gap only
    /// affects VFS (local filesystem) restores.
    #[test]
    fn test_fxar_vfs_restore_restores_uid_gid_as_root() {
        use tempfile::TempDir;
        use std::os::unix::fs::MetadataExt;

        let is_root = unsafe { libc::geteuid() } == 0;
        if !is_root {
            eprintln!("skipping test_fxar_vfs_restore_restores_uid_gid_as_root: requires root");
            return;
        }

        let entry = FxarManifestEntry {
            path: "snap1/tree/owned_file.txt".into(),
            size: 5,
            mode: 0o100644,
            mtime: 1773484800,
            uid: 65534,
            gid: 65534,
            blake3: crate::cid::blake3_hex_to_cid_string(&"a1b2c3d4".repeat(8)).unwrap(),
            chunks: vec![],
            xattr: std::collections::HashMap::new(),
        };

        let restore_dir = TempDir::new().unwrap();
        let dest = restore_dir.path().join("owned_file.txt");
        std::fs::write(&dest, b"hello").unwrap();

        use std::os::unix::fs::PermissionsExt;
        let _ = std::fs::set_permissions(&dest,
            std::fs::Permissions::from_mode(entry.mode));
        // Restore ownership (mirrors the production VFS restore path)
        {
            use nix::unistd::{chown, Uid, Gid};
            let _ = chown(&dest, Some(Uid::from_raw(entry.uid)), Some(Gid::from_raw(entry.gid)));
        }
        let mtime = filetime::FileTime::from_unix_time(entry.mtime, 0);
        let _ = filetime::set_file_mtime(&dest, mtime);

        let meta = std::fs::metadata(&dest).unwrap();

        assert_eq!(meta.uid(), 65534, "uid should be restored from manifest");
        assert_eq!(meta.gid(), 65534, "gid should be restored from manifest");
        assert_eq!(meta.permissions().mode() & 0o777, 0o644,
            "mode should be restored correctly");
        assert_eq!(meta.mtime(), 1773484800,
            "mtime should be restored correctly");
    }

    #[test]
    fn test_generate_sigs_creates_xattrs() {
        use tempfile::TempDir;

        let target_dir = TempDir::new().unwrap();
        let vs_root = target_dir.path().join(".foxing_versions");
        let snap_dir = vs_root.join("2026-03-14T090000");
        let tree_dir = snap_dir.join("tree");
        std::fs::create_dir_all(&tree_dir).unwrap();
        std::fs::write(tree_dir.join("sigtest.dat"), vec![0x42u8; 50_000]).unwrap();
        let summary = serde_json::json!({
            "timestamp": "2026-03-14T09:00:00Z", "status": "success", "type": "full",
            "source": "/test", "trigger": "test", "files": 1, "size_bytes": 50000,
            "disk_usage_bytes": 50000, "savings_pct": 0.0, "elapsed_ms": 1
        });
        std::fs::write(snap_dir.join("summary.json"),
            serde_json::to_string_pretty(&summary).unwrap()).unwrap();

        // Export
        let store = crate::version_store::VersionStore::open(target_dir.path());
        let archive_path = target_dir.path().join("test.fxar");
        let file = std::fs::File::create(&archive_path).unwrap();
        write_archive_seekable(&store, file, "none", None).unwrap();

        // Import with generate_sigs=true
        let restore_dir = TempDir::new().unwrap();
        let f = std::fs::File::open(&archive_path).unwrap();
        let mut reader = FxarReader::open(f).unwrap();
        reader.restore_all(restore_dir.path(), true).unwrap();

        // Verify SyncSignature xattr was written
        let restored = restore_dir.path().join("2026-03-14T090000/tree/sigtest.dat");
        let sig = crate::sidecar::get_sync_signature(&restored);
        assert!(sig.is_some(), "SyncSignature should be written with --generate-sigs");
        let sig = sig.unwrap();
        assert_eq!(sig.size, 50_000);

        // Verify dir_hash on the tree directory
        let tree_restored = restore_dir.path().join("2026-03-14T090000/tree");
        let dir_hash = crate::sidecar::get_metadata(&tree_restored, "dir_hash");
        assert!(dir_hash.is_some(), "dir_hash should be computed with --generate-sigs");
    }

    #[test]
    fn test_dublin_core_xattr_roundtrip() {
        use tempfile::TempDir;

        // Create version store with a snapshot containing Dublin Core xattrs
        let target_dir = TempDir::new().unwrap();
        let vs_root = target_dir.path().join(".foxing_versions");
        let snap_dir = vs_root.join("2026-04-29T120000");
        let tree_dir = snap_dir.join("tree");
        std::fs::create_dir_all(&tree_dir).unwrap();

        let test_file = tree_dir.join("document.txt");
        std::fs::write(&test_file, b"Dublin Core metadata test content").unwrap();

        // Set Dublin Core and xdg.tags xattrs on the test file
        xattr::set(&test_file, "user.dublincore.title", b"Test Document").unwrap();
        xattr::set(&test_file, "user.dublincore.subject", b"Unit Testing").unwrap();
        xattr::set(&test_file, "user.xdg.tags", b"test,fxar,roundtrip").unwrap();

        // Write summary.json
        let summary = serde_json::json!({
            "timestamp": "2026-04-29T12:00:00Z",
            "status": "success",
            "type": "full",
            "source": "/test",
            "trigger": "test",
            "files": 1,
            "size_bytes": 33,
            "disk_usage_bytes": 33,
            "savings_pct": 0.0,
            "elapsed_ms": 1
        });
        std::fs::write(
            snap_dir.join("summary.json"),
            serde_json::to_string_pretty(&summary).unwrap(),
        )
        .unwrap();

        // Export to FXAR v2
        let store = crate::version_store::VersionStore::open(target_dir.path());
        let archive_path = target_dir.path().join("dc_test.fxar");
        let file = std::fs::File::create(&archive_path).unwrap();
        let stats = write_archive_seekable(&store, file, "none", None).unwrap();
        assert_eq!(stats.total_files, 1);

        // Inspect manifest and verify xattrs are captured
        let f = std::fs::File::open(&archive_path).unwrap();
        let inspect = inspect_archive(f).unwrap();
        assert_eq!(inspect.total_files, 1);
        let entry = &inspect.files[0];
        assert!(
            entry.xattr.contains_key("user.dublincore.title"),
            "manifest should contain user.dublincore.title, got keys: {:?}",
            entry.xattr.keys().collect::<Vec<_>>()
        );
        assert!(
            entry.xattr.contains_key("user.dublincore.subject"),
            "manifest should contain user.dublincore.subject"
        );
        assert!(
            entry.xattr.contains_key("user.xdg.tags"),
            "manifest should contain user.xdg.tags"
        );

        // Verify hex-encoded values decode correctly
        let title_hex = &entry.xattr["user.dublincore.title"];
        assert_eq!(hex::decode(title_hex).unwrap(), b"Test Document");
        let tags_hex = &entry.xattr["user.xdg.tags"];
        assert_eq!(hex::decode(tags_hex).unwrap(), b"test,fxar,roundtrip");

        // Restore and verify xattrs survive the roundtrip
        let restore_dir = TempDir::new().unwrap();
        let f = std::fs::File::open(&archive_path).unwrap();
        let mut reader = FxarReader::open(f).unwrap();
        let import_stats = reader.restore_all(restore_dir.path(), false).unwrap();
        assert_eq!(import_stats.files_restored, 1);

        let restored_file =
            restore_dir.path().join("2026-04-29T120000/tree/document.txt");
        assert!(restored_file.exists(), "restored file should exist");

        // Verify restored xattr values match originals
        let restored_title = xattr::get(&restored_file, "user.dublincore.title")
            .unwrap()
            .expect("user.dublincore.title should be restored");
        assert_eq!(restored_title, b"Test Document");

        let restored_subject = xattr::get(&restored_file, "user.dublincore.subject")
            .unwrap()
            .expect("user.dublincore.subject should be restored");
        assert_eq!(restored_subject, b"Unit Testing");

        let restored_tags = xattr::get(&restored_file, "user.xdg.tags")
            .unwrap()
            .expect("user.xdg.tags should be restored");
        assert_eq!(restored_tags, b"test,fxar,roundtrip");
    }

    #[test]
    fn test_restore_file_head_capped() {
        use tempfile::TempDir;

        let target_dir = TempDir::new().unwrap();
        let vs_root = target_dir.path().join(".foxing_versions");
        let snap_dir = vs_root.join("2026-04-29T100000");
        let tree_dir = snap_dir.join("tree");
        std::fs::create_dir_all(&tree_dir).unwrap();

        let big_data = vec![0xCDu8; 32_000];
        std::fs::write(tree_dir.join("big.bin"), &big_data).unwrap();

        let summary = serde_json::json!({
            "timestamp": "2026-04-29T10:00:00Z", "status": "success", "type": "full",
            "source": "/test", "trigger": "test", "files": 1, "size_bytes": 32000,
            "disk_usage_bytes": 32000, "savings_pct": 0.0, "elapsed_ms": 1
        });
        std::fs::write(snap_dir.join("summary.json"),
            serde_json::to_string_pretty(&summary).unwrap()).unwrap();

        let store = crate::version_store::VersionStore::open(target_dir.path());
        let archive_path = target_dir.path().join("head_cap.fxar");
        let file = std::fs::File::create(&archive_path).unwrap();
        write_archive_seekable(&store, file, "none", None).unwrap();

        let f = std::fs::File::open(&archive_path).unwrap();
        let mut reader = FxarReader::open(f).unwrap();
        let manifest = reader.read_manifest().unwrap();
        let entry_path = manifest.files.iter()
            .find(|e| e.path.contains("big.bin"))
            .unwrap()
            .path
            .clone();

        let head = reader.restore_file_head(&entry_path, 8192).unwrap();
        assert_eq!(head.len(), 8192);
        assert_eq!(&head[..], &big_data[..8192]);
    }

    #[test]
    fn test_restore_file_head_small() {
        use tempfile::TempDir;

        let target_dir = TempDir::new().unwrap();
        let vs_root = target_dir.path().join(".foxing_versions");
        let snap_dir = vs_root.join("2026-04-29T110000");
        let tree_dir = snap_dir.join("tree");
        std::fs::create_dir_all(&tree_dir).unwrap();

        let small_data = b"short content for head test";
        std::fs::write(tree_dir.join("small.txt"), small_data).unwrap();

        let summary = serde_json::json!({
            "timestamp": "2026-04-29T11:00:00Z", "status": "success", "type": "full",
            "source": "/test", "trigger": "test", "files": 1,
            "size_bytes": small_data.len(), "disk_usage_bytes": small_data.len(),
            "savings_pct": 0.0, "elapsed_ms": 1
        });
        std::fs::write(snap_dir.join("summary.json"),
            serde_json::to_string_pretty(&summary).unwrap()).unwrap();

        let store = crate::version_store::VersionStore::open(target_dir.path());
        let archive_path = target_dir.path().join("head_small.fxar");
        let file = std::fs::File::create(&archive_path).unwrap();
        write_archive_seekable(&store, file, "none", None).unwrap();

        let f = std::fs::File::open(&archive_path).unwrap();
        let mut reader = FxarReader::open(f).unwrap();
        let manifest = reader.read_manifest().unwrap();
        let entry_path = manifest.files.iter()
            .find(|e| e.path.contains("small.txt"))
            .unwrap()
            .path
            .clone();

        let head = reader.restore_file_head(&entry_path, 8192).unwrap();
        assert_eq!(head, small_data);
    }

    #[test]
    fn test_extension_directory_roundtrip() {
        use tempfile::TempDir;

        let target_dir = TempDir::new().unwrap();
        let vs_root = target_dir.path().join(".foxing_versions");
        let snap_dir = vs_root.join("2026-04-29T130000");
        let tree_dir = snap_dir.join("tree");
        std::fs::create_dir_all(&tree_dir).unwrap();
        std::fs::write(tree_dir.join("ai_test.txt"), b"extension dir test data").unwrap();

        let summary = serde_json::json!({
            "timestamp": "2026-04-29T13:00:00Z", "status": "success", "type": "full",
            "source": "/test", "trigger": "test", "files": 1, "size_bytes": 23,
            "disk_usage_bytes": 23, "savings_pct": 0.0, "elapsed_ms": 1
        });
        std::fs::write(snap_dir.join("summary.json"),
            serde_json::to_string_pretty(&summary).unwrap()).unwrap();

        let store = crate::version_store::VersionStore::open(target_dir.path());
        let archive_path = target_dir.path().join("ext_test.fxar");
        let file = std::fs::File::create(&archive_path).unwrap();

        let chunk_map_data = b"chunk_map_data".to_vec();
        let file_map_data = b"file_map_data".to_vec();

        write_archive_seekable_with_ai(
            &store, file, "none", None,
            vec![(FXAR_EXT_CHUNK_MAP, chunk_map_data.clone()), (FXAR_EXT_FILE_MAP, file_map_data.clone())],
        ).unwrap();

        let f = std::fs::File::open(&archive_path).unwrap();
        let mut reader = FxarReader::open(f).unwrap();

        assert_ne!(reader.header.flags & FLAG_HAS_EXTENSION_DIR, 0);

        let ext_dir = reader.read_extension_directory().unwrap();
        assert!(ext_dir.is_some());
        let ext_dir = ext_dir.unwrap();
        assert_eq!(ext_dir.entries.len(), 2);
        assert_eq!(ext_dir.entries[0].section_type, FXAR_EXT_CHUNK_MAP);
        assert_eq!(ext_dir.entries[1].section_type, FXAR_EXT_FILE_MAP);

        let chunk_data = reader.read_extension_section(FXAR_EXT_CHUNK_MAP).unwrap();
        assert_eq!(chunk_data.unwrap(), chunk_map_data);

        let file_data = reader.read_extension_section(FXAR_EXT_FILE_MAP).unwrap();
        assert_eq!(file_data.unwrap(), file_map_data);

        let missing = reader.read_extension_section(FXAR_EXT_HNSW).unwrap();
        assert!(missing.is_none());

        let manifest = reader.read_manifest().unwrap();
        assert!(!manifest.files.is_empty());
    }

    #[test]
    fn test_v2_archive_compat() {
        use tempfile::TempDir;

        let target_dir = TempDir::new().unwrap();
        let vs_root = target_dir.path().join(".foxing_versions");
        let snap_dir = vs_root.join("2026-04-29T140000");
        let tree_dir = snap_dir.join("tree");
        std::fs::create_dir_all(&tree_dir).unwrap();
        std::fs::write(tree_dir.join("compat.txt"), b"v2 compat test data").unwrap();

        let summary = serde_json::json!({
            "timestamp": "2026-04-29T14:00:00Z", "status": "success", "type": "full",
            "source": "/test", "trigger": "test", "files": 1, "size_bytes": 19,
            "disk_usage_bytes": 19, "savings_pct": 0.0, "elapsed_ms": 1
        });
        std::fs::write(snap_dir.join("summary.json"),
            serde_json::to_string_pretty(&summary).unwrap()).unwrap();

        let store = crate::version_store::VersionStore::open(target_dir.path());
        let archive_path = target_dir.path().join("compat_test.fxar");
        let file = std::fs::File::create(&archive_path).unwrap();
        write_archive_seekable(&store, file, "none", None).unwrap();

        let f = std::fs::File::open(&archive_path).unwrap();
        let mut reader = FxarReader::open(f).unwrap();

        assert_eq!(reader.header.flags & FLAG_HAS_EXTENSION_DIR, 0);

        let ext_dir = reader.read_extension_directory().unwrap();
        assert!(ext_dir.is_none());

        let section = reader.read_extension_section(FXAR_EXT_HNSW).unwrap();
        assert!(section.is_none());

        let manifest = reader.read_manifest().unwrap();
        assert!(!manifest.files.is_empty());
    }

    #[test]
    fn test_ai_prepare_sections_with_vectors() {
        use tempfile::TempDir;

        let dir = TempDir::new().unwrap();
        let index_dir = dir.path().join(".foxing_index");
        std::fs::create_dir_all(&index_dir).unwrap();
        std::fs::write(index_dir.join("chunk_map.bin"), b"chunk_map_payload").unwrap();
        std::fs::write(index_dir.join("file_map.bin"), b"file_map_payload").unwrap();
        std::fs::write(index_dir.join("vectors.usearch"), b"hnsw_payload").unwrap();

        let sections = prepare_ai_extension_sections(&index_dir, true).unwrap();
        assert_eq!(sections.len(), 3);
        assert_eq!(sections[0].0, FXAR_EXT_CHUNK_MAP);
        assert_eq!(sections[0].1, b"chunk_map_payload");
        assert_eq!(sections[1].0, FXAR_EXT_FILE_MAP);
        assert_eq!(sections[1].1, b"file_map_payload");
        assert_eq!(sections[2].0, FXAR_EXT_HNSW);
        assert_eq!(sections[2].1, b"hnsw_payload");
    }

    #[test]
    fn test_ai_prepare_sections_metadata_only() {
        use tempfile::TempDir;

        let dir = TempDir::new().unwrap();
        let index_dir = dir.path().join(".foxing_index");
        std::fs::create_dir_all(&index_dir).unwrap();
        std::fs::write(index_dir.join("chunk_map.bin"), b"cm").unwrap();
        std::fs::write(index_dir.join("file_map.bin"), b"fm").unwrap();
        std::fs::write(index_dir.join("vectors.usearch"), b"hnsw").unwrap();

        let sections = prepare_ai_extension_sections(&index_dir, false).unwrap();
        assert_eq!(sections.len(), 2);
        assert_eq!(sections[0].0, FXAR_EXT_CHUNK_MAP);
        assert_eq!(sections[1].0, FXAR_EXT_FILE_MAP);
    }

    #[test]
    fn test_ai_prepare_sections_missing_dir() {
        let missing = std::path::PathBuf::from("/tmp/nonexistent_foxing_index_dir_xyz");
        let sections = prepare_ai_extension_sections(&missing, true).unwrap();
        assert!(sections.is_empty());
    }

    #[test]
    fn test_ai_section_raw_roundtrip() {
        use tempfile::TempDir;

        let target_dir = TempDir::new().unwrap();
        let vs_root = target_dir.path().join(".foxing_versions");
        let snap_dir = vs_root.join("2026-04-29T150000");
        let tree_dir = snap_dir.join("tree");
        std::fs::create_dir_all(&tree_dir).unwrap();
        std::fs::write(tree_dir.join("roundtrip.txt"), b"raw roundtrip data").unwrap();

        let summary = serde_json::json!({
            "timestamp": "2026-04-29T15:00:00Z", "status": "success", "type": "full",
            "source": "/test", "trigger": "test", "files": 1, "size_bytes": 18,
            "disk_usage_bytes": 18, "savings_pct": 0.0, "elapsed_ms": 1
        });
        std::fs::write(snap_dir.join("summary.json"),
            serde_json::to_string_pretty(&summary).unwrap()).unwrap();

        let index_dir = target_dir.path().join(".foxing_index");
        std::fs::create_dir_all(&index_dir).unwrap();
        let chunk_map_content = b"test_chunk_map_binary_content_12345";
        let file_map_content = b"test_file_map_binary_content_67890";
        std::fs::write(index_dir.join("chunk_map.bin"), chunk_map_content).unwrap();
        std::fs::write(index_dir.join("file_map.bin"), file_map_content).unwrap();

        let store = crate::version_store::VersionStore::open(target_dir.path());
        let archive_path = target_dir.path().join("ai_roundtrip.fxar");
        let file = std::fs::File::create(&archive_path).unwrap();
        write_archive_with_ai_index(
            &store, file, "none", None, Some(&index_dir), false,
        ).unwrap();

        let f = std::fs::File::open(&archive_path).unwrap();
        let mut reader = FxarReader::open(f).unwrap();

        let chunk_data = reader.read_extension_section(FXAR_EXT_CHUNK_MAP).unwrap();
        assert_eq!(chunk_data.unwrap(), chunk_map_content);

        let file_data = reader.read_extension_section(FXAR_EXT_FILE_MAP).unwrap();
        assert_eq!(file_data.unwrap(), file_map_content);

        let hnsw = reader.read_extension_section(FXAR_EXT_HNSW).unwrap();
        assert!(hnsw.is_none());
    }

    // -------------------------------------------------------------------
    // Integration tests: end-to-end FXAR archive pipeline
    // -------------------------------------------------------------------

    /// Helper: create a version store with files and Dublin Core xattrs.
    fn make_test_store_with_xattrs(dir: &Path) -> crate::version_store::VersionStore {
        let vs_root = dir.join(".foxing_versions");
        let snap_dir = vs_root.join("2026-04-29T120000");
        let tree_dir = snap_dir.join("tree");
        std::fs::create_dir_all(&tree_dir).unwrap();

        // Create a test file with Dublin Core xattr
        let test_file = tree_dir.join("report.txt");
        std::fs::write(&test_file, b"Quarterly report content").unwrap();
        xattr::set(&test_file, "user.dublincore.title", b"Quarterly Report 2026").ok();
        xattr::set(&test_file, "user.xdg.tags", b"finance,2026").ok();

        // Create summary.json (required by VersionStore)
        let summary = serde_json::json!({
            "timestamp": "2026-04-29T12:00:00Z", "status": "success", "type": "full",
            "source": "/test", "trigger": "test", "files": 1, "size_bytes": 24,
            "disk_usage_bytes": 24, "savings_pct": 0.0, "elapsed_ms": 1
        });
        std::fs::write(
            snap_dir.join("summary.json"),
            serde_json::to_string(&summary).unwrap(),
        )
        .unwrap();

        crate::version_store::VersionStore::open(dir)
    }

    /// End-to-end: export with AI sections -> inspect extension dir -> load AI
    /// sections -> browse with ArchiveNavigator -> extract file -> verify xattrs.
    #[test]
    fn test_fxar_ai_full_roundtrip() {
        use tempfile::TempDir;

        let dir = TempDir::new().unwrap();
        let store = make_test_store_with_xattrs(dir.path());

        // Create minimal .foxing_index/ for AI sections
        let index_dir = dir.path().join(".foxing_index");
        std::fs::create_dir_all(&index_dir).unwrap();
        std::fs::write(index_dir.join("chunk_map.bin"), b"test_chunk_map_data").unwrap();
        std::fs::write(index_dir.join("file_map.bin"), b"test_file_map_data").unwrap();

        // Export with AI sections (no HNSW)
        let archive_path = dir.path().join("test.fxar");
        let file = std::fs::File::create(&archive_path).unwrap();
        write_archive_with_ai_index(&store, file, "none", None, Some(&index_dir), false)
            .unwrap();

        // --- Phase 1: Open and verify extension directory ---
        let f = std::fs::File::open(&archive_path).unwrap();
        let mut reader = FxarReader::open(f).unwrap();

        let ext_dir = reader.read_extension_directory().unwrap();
        assert!(ext_dir.is_some(), "Extension directory should be present");
        let ext_dir = ext_dir.unwrap();
        assert!(
            ext_dir.entries.iter().any(|e| e.section_type == FXAR_EXT_CHUNK_MAP),
            "CHUNK_MAP section should exist"
        );
        assert!(
            ext_dir.entries.iter().any(|e| e.section_type == FXAR_EXT_FILE_MAP),
            "FILE_MAP section should exist"
        );
        assert!(
            !ext_dir.entries.iter().any(|e| e.section_type == FXAR_EXT_HNSW),
            "HNSW section should NOT exist (include_vectors=false)"
        );

        // --- Phase 2: Read raw sections and verify content ---
        let f = std::fs::File::open(&archive_path).unwrap();
        let mut reader = FxarReader::open(f).unwrap();
        let chunk_data = reader.read_extension_section(FXAR_EXT_CHUNK_MAP).unwrap();
        assert!(chunk_data.is_some());
        assert_eq!(chunk_data.unwrap(), b"test_chunk_map_data");

        let file_data = reader.read_extension_section(FXAR_EXT_FILE_MAP).unwrap();
        assert!(file_data.is_some());
        assert_eq!(file_data.unwrap(), b"test_file_map_data");

        // --- Phase 3: Verify manifest + xattr capture ---
        let f = std::fs::File::open(&archive_path).unwrap();
        let mut reader = FxarReader::open(f).unwrap();
        let manifest = reader.read_manifest().unwrap();
        assert!(!manifest.files.is_empty(), "Manifest should have files");

        // Find report.txt entry
        let entry = manifest
            .files
            .iter()
            .find(|f| f.path.contains("report.txt"));
        assert!(entry.is_some(), "report.txt should be in manifest");
        let entry = entry.unwrap();

        // xattr support on tmpfs may vary  --  guard the check
        if !entry.xattr.is_empty() {
            assert!(
                entry.xattr.contains_key("user.dublincore.title")
                    || entry.xattr.contains_key("user.xdg.tags"),
                "Dublin Core or xdg.tags xattr should be in manifest, got keys: {:?}",
                entry.xattr.keys().collect::<Vec<_>>()
            );
        }

        // --- Phase 4: Extract file via restore_file_head ---
        let f = std::fs::File::open(&archive_path).unwrap();
        let mut reader = FxarReader::open(f).unwrap();
        let path_in_archive = entry.path.clone();
        let content = reader.restore_file_head(&path_in_archive, 8192).unwrap();
        assert!(!content.is_empty(), "Extracted content should not be empty");
        assert!(
            content.starts_with(b"Quarterly report"),
            "Extracted content should match original"
        );

        // --- Phase 5: Build ArchiveNavigator and verify tree ---
        #[cfg(feature = "tui")]
        {
            let f = std::fs::File::open(&archive_path).unwrap();
            let mut reader = FxarReader::open(f).unwrap();
            let manifest2 = reader.read_manifest().unwrap();
            let nav = crate::browser::ArchiveNavigator::from_manifest(manifest2).unwrap();
            assert!(
                nav.entries.len() > 1,
                "Navigator should have entries beyond root"
            );
            // Root entry at 0 has empty full_path
            assert_eq!(nav.entry_at(0).unwrap().full_path, "");
        }
    }

    /// Verify metadata-only export has no HNSW section.
    #[test]
    fn test_fxar_metadata_only_no_hnsw() {
        use tempfile::TempDir;

        let dir = TempDir::new().unwrap();
        let store = make_test_store_with_xattrs(dir.path());

        // Create index dir with chunk_map, file_map, AND vectors.usearch
        let index_dir = dir.path().join(".foxing_index");
        std::fs::create_dir_all(&index_dir).unwrap();
        std::fs::write(index_dir.join("chunk_map.bin"), b"cm_payload").unwrap();
        std::fs::write(index_dir.join("file_map.bin"), b"fm_payload").unwrap();
        std::fs::write(index_dir.join("vectors.usearch"), b"hnsw_payload").unwrap();

        // Export without vectors
        let archive_path = dir.path().join("metadata_only.fxar");
        let file = std::fs::File::create(&archive_path).unwrap();
        write_archive_with_ai_index(&store, file, "none", None, Some(&index_dir), false)
            .unwrap();

        let f = std::fs::File::open(&archive_path).unwrap();
        let mut reader = FxarReader::open(f).unwrap();

        // Extension dir should exist with chunk_map + file_map but NOT HNSW
        let ext_dir = reader.read_extension_directory().unwrap();
        assert!(ext_dir.is_some(), "Extension directory should be present");
        let ext_dir = ext_dir.unwrap();
        assert_eq!(
            ext_dir.entries.len(),
            2,
            "Should have exactly 2 sections (chunk_map + file_map)"
        );
        assert!(
            !ext_dir.entries.iter().any(|e| e.section_type == FXAR_EXT_HNSW),
            "HNSW section should NOT exist when include_vectors=false"
        );

        // Verify HNSW read returns None
        let hnsw = reader.read_extension_section(FXAR_EXT_HNSW).unwrap();
        assert!(hnsw.is_none(), "read_extension_section(HNSW) should return Ok(None)");
    }

    #[test]
    fn test_update_fxar_ai_sections_update_existing() {
        use tempfile::TempDir;

        let dir = TempDir::new().unwrap();
        let store = make_test_store_with_xattrs(dir.path());

        let index_dir = dir.path().join(".foxing_index");
        std::fs::create_dir_all(&index_dir).unwrap();
        std::fs::write(index_dir.join("chunk_map.bin"), b"original_chunk_map").unwrap();
        std::fs::write(index_dir.join("file_map.bin"), b"original_file_map").unwrap();

        let archive_path = dir.path().join("update_test.fxar");
        let file = std::fs::File::create(&archive_path).unwrap();
        write_archive_with_ai_index(&store, file, "none", None, Some(&index_dir), false)
            .unwrap();

        // Confirm original sections exist
        {
            let f = std::fs::File::open(&archive_path).unwrap();
            let mut reader = FxarReader::open(f).unwrap();
            let ext_dir = reader.read_extension_directory().unwrap();
            assert!(ext_dir.is_some());
            let chunk = reader.read_extension_section(FXAR_EXT_CHUNK_MAP).unwrap();
            assert_eq!(chunk.unwrap(), b"original_chunk_map");
        }

        // Update with new data
        let new_sections = vec![
            (FXAR_EXT_CHUNK_MAP, b"UPDATED_chunk_map_data_v2".to_vec()),
            (FXAR_EXT_FILE_MAP, b"UPDATED_file_map_data_v2".to_vec()),
        ];
        update_fxar_ai_sections(&archive_path, &new_sections).unwrap();

        // Verify updated sections
        let f = std::fs::File::open(&archive_path).unwrap();
        let mut reader = FxarReader::open(f).unwrap();
        let ext_dir = reader.read_extension_directory().unwrap();
        assert!(ext_dir.is_some());
        let ext_dir = ext_dir.unwrap();
        assert_eq!(ext_dir.entries.len(), 2);

        let chunk = reader.read_extension_section(FXAR_EXT_CHUNK_MAP).unwrap();
        assert_eq!(chunk.unwrap(), b"UPDATED_chunk_map_data_v2");

        let fmap = reader.read_extension_section(FXAR_EXT_FILE_MAP).unwrap();
        assert_eq!(fmap.unwrap(), b"UPDATED_file_map_data_v2");

        // Manifest preserved
        let f = std::fs::File::open(&archive_path).unwrap();
        let mut reader = FxarReader::open(f).unwrap();
        let manifest = reader.read_manifest().unwrap();
        assert!(!manifest.files.is_empty());
        let data = reader.restore_file(&manifest.files[0].path).unwrap();
        assert_eq!(data, b"Quarterly report content");
    }

    #[test]
    fn test_update_fxar_ai_sections_append_to_plain() {
        use tempfile::TempDir;

        let dir = TempDir::new().unwrap();
        let store = make_test_store_with_xattrs(dir.path());

        let archive_path = dir.path().join("plain.fxar");
        let file = std::fs::File::create(&archive_path).unwrap();
        write_archive_seekable(&store, file, "none", None).unwrap();

        // Confirm no extension directory
        {
            let f = std::fs::File::open(&archive_path).unwrap();
            let mut reader = FxarReader::open(f).unwrap();
            assert!(reader.read_extension_directory().unwrap().is_none());
        }

        // Append AI sections
        let sections = vec![
            (FXAR_EXT_CHUNK_MAP, b"appended_chunk_data".to_vec()),
        ];
        update_fxar_ai_sections(&archive_path, &sections).unwrap();

        // FLAG_HAS_EXTENSION_DIR should now be set
        let f = std::fs::File::open(&archive_path).unwrap();
        let mut reader = FxarReader::open(f).unwrap();
        assert!(
            reader.header.flags & FLAG_HAS_EXTENSION_DIR != 0,
            "FLAG_HAS_EXTENSION_DIR should be set after append"
        );

        let ext_dir = reader.read_extension_directory().unwrap();
        assert!(ext_dir.is_some());
        let ext_dir = ext_dir.unwrap();
        assert_eq!(ext_dir.entries.len(), 1);
        assert_eq!(ext_dir.entries[0].section_type, FXAR_EXT_CHUNK_MAP);

        let chunk = reader.read_extension_section(FXAR_EXT_CHUNK_MAP).unwrap();
        assert_eq!(chunk.unwrap(), b"appended_chunk_data");

        // Manifest preserved
        let f = std::fs::File::open(&archive_path).unwrap();
        let mut reader = FxarReader::open(f).unwrap();
        let manifest = reader.read_manifest().unwrap();
        assert!(!manifest.files.is_empty());
        let data = reader.restore_file(&manifest.files[0].path).unwrap();
        assert_eq!(data, b"Quarterly report content");
    }

    #[test]
    fn test_update_fxar_then_read_back() {
        use tempfile::TempDir;

        let dir = TempDir::new().unwrap();
        let store = make_test_store_with_xattrs(dir.path());

        let archive_path = dir.path().join("readback.fxar");
        let file = std::fs::File::create(&archive_path).unwrap();
        write_archive_seekable(&store, file, "none", None).unwrap();

        let sections = vec![
            (FXAR_EXT_CHUNK_MAP, b"chunk_map_v1".to_vec()),
            (FXAR_EXT_FILE_MAP, b"file_map_v1".to_vec()),
        ];
        update_fxar_ai_sections(&archive_path, &sections).unwrap();

        let f = std::fs::File::open(&archive_path).unwrap();
        let mut reader = FxarReader::open(f).unwrap();

        let cm = reader.read_extension_section(FXAR_EXT_CHUNK_MAP).unwrap();
        assert_eq!(cm.unwrap(), b"chunk_map_v1");

        let fm = reader.read_extension_section(FXAR_EXT_FILE_MAP).unwrap();
        assert_eq!(fm.unwrap(), b"file_map_v1");

        let f = std::fs::File::open(&archive_path).unwrap();
        let mut reader = FxarReader::open(f).unwrap();
        let manifest = reader.read_manifest().unwrap();
        let data = reader.restore_file(&manifest.files[0].path).unwrap();
        assert_eq!(data, b"Quarterly report content");
    }

    #[test]
    fn test_update_fxar_twice_replaces_sections() {
        use tempfile::TempDir;

        let dir = TempDir::new().unwrap();
        let store = make_test_store_with_xattrs(dir.path());

        let archive_path = dir.path().join("twice.fxar");
        let file = std::fs::File::create(&archive_path).unwrap();
        write_archive_seekable(&store, file, "none", None).unwrap();

        update_fxar_ai_sections(
            &archive_path,
            &[(FXAR_EXT_CHUNK_MAP, b"version_1".to_vec())],
        )
        .unwrap();

        update_fxar_ai_sections(
            &archive_path,
            &[(FXAR_EXT_CHUNK_MAP, b"version_2".to_vec())],
        )
        .unwrap();

        let f = std::fs::File::open(&archive_path).unwrap();
        let mut reader = FxarReader::open(f).unwrap();

        let cm = reader.read_extension_section(FXAR_EXT_CHUNK_MAP).unwrap();
        assert_eq!(cm.unwrap(), b"version_2");

        let f = std::fs::File::open(&archive_path).unwrap();
        let mut reader = FxarReader::open(f).unwrap();
        let manifest = reader.read_manifest().unwrap();
        assert!(!manifest.files.is_empty());
        let data = reader.restore_file(&manifest.files[0].path).unwrap();
        assert_eq!(data, b"Quarterly report content");
    }

    /// Verify old-style archive (no AI sections) is backward compatible.
    #[test]
    fn test_fxar_v2_compat_no_extension() {
        use tempfile::TempDir;

        let dir = TempDir::new().unwrap();
        let store = make_test_store_with_xattrs(dir.path());

        let archive_path = dir.path().join("compat.fxar");
        let file = std::fs::File::create(&archive_path).unwrap();

        // Use old-style write (no AI sections)
        write_archive_seekable(&store, file, "none", None).unwrap();

        // Verify extension directory is absent
        let f = std::fs::File::open(&archive_path).unwrap();
        let mut reader = FxarReader::open(f).unwrap();
        let ext_dir = reader.read_extension_directory().unwrap();
        assert!(
            ext_dir.is_none(),
            "Old-style archive should have no extension directory"
        );

        // Verify manifest still readable (backward compat)
        let f = std::fs::File::open(&archive_path).unwrap();
        let mut reader = FxarReader::open(f).unwrap();
        let manifest = reader.read_manifest().unwrap();
        assert!(
            !manifest.files.is_empty(),
            "Old-style archive manifest should be readable"
        );

        // Verify files extractable
        let entry_path = manifest.files[0].path.clone();
        let data = reader.restore_file(&entry_path).unwrap();
        assert_eq!(data, b"Quarterly report content");
    }

    /// Test ArchiveNavigator with multiple snapshots: tree structure,
    /// snapshot listing, filtered visibility, and entry counts.
    #[test]
    #[cfg(feature = "tui")]
    fn test_fxar_archive_navigator_browse() {
        use crate::browser::ArchiveNavigator;

        let manifest = FxarManifest {
            version: 2,
            created: "2026-04-29T00:00:00Z".into(),
            files: vec![
                FxarManifestEntry {
                    path: "snap1/tree/main.rs".into(),
                    size: 100,
                    mtime: 0,
                    mode: 0o644,
                    uid: 0,
                    gid: 0,
                    blake3: crate::cid::blake3_hex_to_cid_string(&"aa".repeat(32)).unwrap(),
                    chunks: vec![0],
                    xattr: Default::default(),
                },
                FxarManifestEntry {
                    path: "snap1/tree/lib.rs".into(),
                    size: 200,
                    mtime: 0,
                    mode: 0o644,
                    uid: 0,
                    gid: 0,
                    blake3: crate::cid::blake3_hex_to_cid_string(&"bb".repeat(32)).unwrap(),
                    chunks: vec![0],
                    xattr: Default::default(),
                },
                FxarManifestEntry {
                    path: "snap2/tree/readme.md".into(),
                    size: 50,
                    mtime: 0,
                    mode: 0o644,
                    uid: 0,
                    gid: 0,
                    blake3: crate::cid::blake3_hex_to_cid_string(&"cc".repeat(32)).unwrap(),
                    chunks: vec![0],
                    xattr: Default::default(),
                },
            ],
            snapshots: vec!["snap1".into(), "snap2".into()],
        };

        let nav = ArchiveNavigator::from_manifest(manifest).unwrap();

        // Root is always index 0 with empty full_path
        assert_eq!(nav.entry_at(0).unwrap().full_path, "");

        // list_snapshots returns correct names
        assert_eq!(nav.list_snapshots(), &["snap1", "snap2"]);

        // visible_entry_indices(None) returns all entries
        let all = nav.visible_entry_indices(None);
        assert_eq!(all.len(), nav.entries.len());

        // visible_entry_indices(Some("snap1")) excludes snap2 entries
        let snap1_only = nav.visible_entry_indices(Some("snap1"));
        for &idx in &snap1_only {
            let e = nav.entry_at(idx).unwrap();
            if !e.full_path.is_empty() {
                assert!(
                    e.full_path == "snap1" || e.full_path.starts_with("snap1/"),
                    "snap1 filter returned unexpected entry: {}",
                    e.full_path
                );
            }
        }

        // snapshot_entry_count("snap1") == 2 (main.rs + lib.rs)
        assert_eq!(nav.snapshot_entry_count("snap1"), 2);
        assert_eq!(nav.snapshot_entry_count("snap2"), 1);
    }

    #[test]
    fn test_decode_blake3_field_hex_and_cid() {
        let hex_hash = "a1b2c3d4".repeat(8);
        let decoded_hex = decode_blake3_field(&hex_hash);
        assert!(decoded_hex.is_some(), "hex-encoded blake3 must decode");
        assert_eq!(hex::encode(decoded_hex.unwrap()), hex_hash);

        let cid_hash = crate::cid::blake3_hex_to_cid_string(&hex_hash).unwrap();
        let decoded_cid = decode_blake3_field(&cid_hash);
        assert!(decoded_cid.is_some(), "CID-encoded blake3 must decode");
        assert_eq!(decoded_hex, decoded_cid, "both formats must produce same bytes");
    }

    #[test]
    fn test_fxar_manifest_blake3_is_cid_encoded() {
        let entry = FxarManifestEntry {
            path: "test.txt".to_string(),
            size: 100,
            mode: 0o644,
            mtime: 0,
            uid: 1000,
            gid: 1000,
            blake3: crate::cid::blake3_to_cid_string(&[0xABu8; 32]),
            chunks: vec![0],
            xattr: std::collections::HashMap::new(),
        };
        assert!(entry.blake3.starts_with("b"), "manifest blake3 should be CID-encoded");
        let decoded = decode_blake3_field(&entry.blake3).unwrap();
        assert_eq!(decoded, [0xABu8; 32]);
    }

    #[test]
    fn test_fxar_manifest_hex_backward_compat() {
        let hex = "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262";
        let entry = FxarManifestEntry {
            path: "old.txt".to_string(),
            size: 50,
            mode: 0o644,
            mtime: 0,
            uid: 0,
            gid: 0,
            blake3: hex.to_string(),
            chunks: vec![],
            xattr: std::collections::HashMap::new(),
        };
        let decoded = decode_blake3_field(&entry.blake3).unwrap();
        assert_eq!(hex::encode(decoded), hex);
    }

    #[test]
    fn test_decode_blake3_field_invalid_returns_none() {
        assert!(decode_blake3_field("").is_none());
        assert!(decode_blake3_field("not-a-hash").is_none());
        assert!(decode_blake3_field("baf").is_none());
    }
}