fxcp_core/sync/
tree.rs

1// SPDX-License-Identifier: GPL-2.0-or-later
2// Copyright (C) 2025 Joel Wirāmu Pauling <aenertia@aenertia.net>
3//
4//! Tree walking, directory operations, copy helpers.
5
6use std::path::Path;
7use std::sync::Arc;
8use tracing::{info, debug};
9
10use crate::constants;
11use crate::operations::{
12    SmartCopier, CopyStats, probe_capabilities, FsyncLatencyTracker,
13};
14use crate::buffer::BufferPool;
15use crate::governor::Governor;
16use crate::hashing::{self, MerkleTree};
17use crate::sidecar;
18
19use super::FICLONE;
20
21/// Store foxingd-compatible sync signatures on the destination file.
22pub fn store_foxing_signatures(dst: &Path) -> crate::Result<()> {
23    let sig = sidecar::SyncSignature::compute(dst)?;
24    sidecar::set_sync_signature(dst, &sig)?;
25    let file_size = std::fs::metadata(dst)?.len();
26    if file_size > (hashing::CHUNK_SIZE * 4) as u64 {
27        let tree = MerkleTree::from_file(dst, hashing::calculate_adaptive_chunk_size(dst.metadata()?.len()), hashing::ComputeMode::Blake3Only)?;
28        let merkle_sig = tree.to_signature();
29        sidecar::set_merkle_signature(dst, &merkle_sig)?;
30    }
31    Ok(())
32}
33
34pub(super) fn try_reflink_copy(src: &Path, dst: &Path) -> bool {
35    use std::os::unix::io::AsRawFd;
36    let src_file = match std::fs::File::open(src) { Ok(f) => f, Err(_) => return false };
37    let dst_file = match std::fs::File::create(dst) { Ok(f) => f, Err(_) => return false };
38    // SAFETY: both fds are valid open file descriptors from File::open/create.
39    unsafe { libc::ioctl(dst_file.as_raw_fd(), FICLONE, src_file.as_raw_fd()) == 0 }
40}
41
42pub(super) fn try_copy_file_range(src: &Path, dst: &Path, size: u64) -> std::io::Result<u64> {
43    use std::os::unix::io::AsRawFd;
44    let src_file = std::fs::File::open(src)?;
45    let dst_file = std::fs::File::create(dst)?;
46    let sfd = src_file.as_raw_fd();
47    let dfd = dst_file.as_raw_fd();
48    let mut total = 0u64;
49    let mut off_in = 0i64;
50    let mut off_out = 0i64;
51    while total < size {
52        let chunk = ((size - total) as usize).min(crate::constants::COPY_FILE_RANGE_CHUNK_SIZE as usize);
53        // SAFETY: sfd/dfd are valid fds. off_in/off_out are valid mutable
54        // references to offsets updated atomically by the kernel.
55        let ret = unsafe { libc::copy_file_range(sfd, &mut off_in, dfd, &mut off_out, chunk, 0) };
56        if ret < 0 {
57            let err = std::io::Error::last_os_error();
58            if total == 0 { drop(dst_file); let _ = std::fs::remove_file(dst); }
59            return Err(err);
60        }
61        if ret == 0 { break; }
62        total += ret as u64;
63    }
64    Ok(total)
65}
66
67pub(super) fn copy_small_file(src: &Path, dst: &Path) -> std::io::Result<u64> {
68    crate::operations::reflink_or_copy(src, dst)
69}
70
71pub(super) async fn try_delta_copy(
72    copier: &mut SmartCopier,
73    src: &Path, dst: &Path, file_size: u64,
74) -> crate::Result<Option<CopyStats>> {
75    let chunk_size = hashing::calculate_adaptive_chunk_size(file_size);
76    let src_tree = MerkleTree::from_file(src, chunk_size, hashing::ComputeMode::Blake3Only)?;
77    let dst_tree = MerkleTree::from_file(dst, chunk_size, hashing::ComputeMode::Blake3Only)?;
78    if src_tree.root == dst_tree.root {
79        return Ok(Some(CopyStats::default()));
80    }
81    let dirty = MerkleTree::diff(&src_tree, &dst_tree);
82    if dirty.is_empty() {
83        return Ok(Some(CopyStats::default()));
84    }
85    let stats = copier.copy_delta(src, dst, &dirty, file_size, "delta").await?;
86    Ok(Some(stats))
87}
88
89pub(crate) async fn create_copier(src: &Path, dst: &Path) -> crate::Result<SmartCopier> {
90    use tokio::io::unix::AsyncFd;
91
92    let src_caps = probe_capabilities(src);
93    let dst_caps = probe_capabilities(dst);
94    let ring = io_uring::IoUring::new(crate::constants::FXCP_IO_URING_SQ_DEPTH)?;
95    // SAFETY: eventfd() creates a new file descriptor. No preconditions.
96    let eventfd = unsafe { libc::eventfd(0, libc::EFD_NONBLOCK) };
97    if eventfd < 0 {
98        return Err(crate::FxcpError::Io(std::io::Error::last_os_error()));
99    }
100    ring.submitter().register_eventfd(eventfd)?;
101    let async_fd = Arc::new(AsyncFd::new(eventfd)?);
102    let mut buffer_pool = BufferPool::new(crate::constants::FXCP_BUFFER_POOL_COUNT, crate::constants::BUFFER_POOL_MIN_CHUNK_SIZE, 512)?;
103    let iovs = buffer_pool.as_io_vecs();
104    // Try to register buffers for zero-copy I/O  --  falls back gracefully on
105    // tmpfs/ramfs/hugetlbfs where page pinning fails with EINVAL.
106    // SAFETY: iovs points to valid AlignedBuffer allocations that outlive the ring.
107    if unsafe { ring.submitter().register_buffers(&iovs) }.is_err() {
108        debug!("io_uring buffer registration failed (tmpfs/ramfs?)  --  using unregistered I/O");
109    }
110    // NAPI busy-poll: register with ring when target is NFS for reduced completion latency.
111    // Best-effort  --  ENOSYS (kernel <6.9) or EINVAL are silently logged and ignored.
112    if dst_caps.is_nfs.load(std::sync::atomic::Ordering::Relaxed)
113        && let Err(e) = crate::operations::napi::try_register_napi(
114            &ring,
115            crate::operations::napi::DEFAULT_NAPI_BUSY_POLL_TO_US,
116            true,
117        ) {
118            debug!("io_uring NAPI registration failed for NFS target (best-effort): {}", e);
119        }
120    use std::os::unix::io::AsRawFd;
121    let postcopy_ring = io_uring::IoUring::builder()
122        .setup_attach_wq(ring.as_raw_fd())
123        .build(8)
124        .map_err(crate::FxcpError::Io)?;
125    let postcopy_eventfd = unsafe { libc::eventfd(0, libc::EFD_NONBLOCK) };
126    if postcopy_eventfd < 0 {
127        return Err(crate::FxcpError::Io(std::io::Error::last_os_error()));
128    }
129    postcopy_ring.submitter().register_eventfd(postcopy_eventfd)?;
130    let postcopy_async_fd = Arc::new(AsyncFd::new(postcopy_eventfd)?);
131
132    let governor = Governor::new(4.0, 0, 10.0, 10.0);
133    Ok(SmartCopier {
134        ring, buffer_pool,
135        atomic_buffer_pool: None, async_fd,
136        vdo_opt: false, direct_io_ok: false,
137        source_caps: src_caps, target_caps: dst_caps,
138        vdo_stall_threshold: 0, barrier_callback: None,
139        source_uncached: false, target_uncached: false,
140        governor: Some(Arc::new(governor)),
141        fsync_tracker: FsyncLatencyTracker::default(),
142        skip_fsync: true,
143        #[allow(deprecated)]
144        segment_stall_timeout_secs: constants::PROCESS_SEGMENT_STALL_SECS,
145        #[allow(deprecated)]
146        segment_overall_timeout_secs: constants::PROCESS_SEGMENT_TIMEOUT_SECS,
147        postcopy_ring,
148        postcopy_async_fd,
149    })
150}
151
152pub fn preserve_metadata(src: &Path, dst: &Path, opts: &super::SyncOptions) -> std::io::Result<()> {
153    use std::os::unix::fs::MetadataExt;
154    let src_meta = std::fs::metadata(src)?;
155    if opts.perms {
156        std::fs::set_permissions(dst, src_meta.permissions())?;
157    }
158    // SAFETY: geteuid() is a pure read of the effective UID, no preconditions.
159    let is_root = unsafe { libc::geteuid() } == 0;
160    if opts.owner && is_root {
161        if let Err(e) = crate::security::set_ownership(dst, src_meta.uid(), src_meta.gid()) {
162            tracing::warn!("chown {:?}: {}", dst, e);
163        }
164    } else if opts.group && is_root {
165        let dst_meta = std::fs::metadata(dst)?;
166        if let Err(e) = crate::security::set_ownership(dst, dst_meta.uid(), src_meta.gid()) {
167            tracing::warn!("chgrp {:?}: {}", dst, e);
168        }
169    }
170    if opts.perms {
171        let times = [
172            libc::timespec { tv_sec: src_meta.atime(), tv_nsec: src_meta.atime_nsec() },
173            libc::timespec { tv_sec: src_meta.mtime(), tv_nsec: src_meta.mtime_nsec() },
174        ];
175        let dst_cstr = std::ffi::CString::new(dst.as_os_str().as_encoded_bytes())
176            .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?;
177        // SAFETY: dst_cstr is a valid null-terminated C string. times is a valid
178        // pointer to a [timespec; 2] array.
179        let ret = unsafe { libc::utimensat(libc::AT_FDCWD, dst_cstr.as_ptr(), times.as_ptr(), 0) };
180        if ret != 0 { return Err(std::io::Error::last_os_error()); }
181    }
182    if opts.xattrs {
183        crate::security::sync_xattrs(src, dst);
184    }
185    Ok(())
186}
187
188pub(super) fn delete_extra_files(source: &Path, target: &Path, filter: &crate::filter::FilterRules) -> crate::Result<u64> {
189    // Fast path: if a tombstone journal exists on the target, replay it
190    // instead of walking the entire target tree.
191    let journal_path = target.join(".foxing_tombstones.jsonl");
192    if journal_path.exists()
193        && let Ok(journal) = crate::tombstone::TombstoneJournal::open(&journal_path)
194            && let Ok(entries) = journal.read_all()
195                && !entries.is_empty() {
196                    info!("Replaying {} tombstones (skipping full target walk)", entries.len());
197                    let deleted = crate::tombstone::replay_tombstones(target, &entries, &filter.excludes)?;
198                    let _ = journal.clear();
199                    return Ok(deleted);
200                }
201
202    // Slow path: full target walk (no journal available)
203    let mut deleted = 0u64;
204    for entry in walkdir::WalkDir::new(target).contents_first(true) {
205        let entry = match entry { Ok(e) => e, Err(_) => continue };
206        let tgt_path = entry.path();
207        let rel = match tgt_path.strip_prefix(target) { Ok(r) => r, Err(_) => continue };
208        if rel.as_os_str().is_empty() { continue; }
209        if filter.should_skip(rel) { continue; }
210        if rel.to_string_lossy().contains(".foxing_tombstones") { continue; }
211        if rel.to_string_lossy().starts_with(".foxing") { continue; }
212        let src_path = source.join(rel);
213        if !src_path.symlink_metadata().is_ok() {
214            if entry.file_type().is_dir() { let _ = std::fs::remove_dir(tgt_path); }
215            else { let _ = std::fs::remove_file(tgt_path); deleted += 1; }
216        }
217    }
218    Ok(deleted)
219}
220
221pub(super) fn run_cleanup(path: &Path) {
222    let mut orphaned = 0u64;
223    let mut dirty = 0u64;
224    let cutoff = std::time::SystemTime::now() - std::time::Duration::from_secs(constants::HOT_DIR_CUTOFF_SECS);
225    for entry in walkdir::WalkDir::new(path) {
226        let entry = match entry { Ok(e) => e, Err(_) => continue };
227        if !entry.file_type().is_file() { continue; }
228        let name = entry.file_name().to_string_lossy();
229        if name.starts_with(".tmp.")
230            && let Ok(meta) = entry.metadata()
231                && let Ok(mtime) = meta.modified()
232                    && mtime < cutoff {
233                        info!("removing orphaned: {:?}", entry.path());
234                        let _ = std::fs::remove_file(entry.path());
235                        orphaned += 1;
236                    }
237        if sidecar::is_dirty(entry.path()) {
238            info!("dirty flag: {:?}", entry.path());
239            dirty += 1;
240        }
241    }
242    println!("Cleanup: {} orphaned files removed, {} dirty flags found", orphaned, dirty);
243}
244
245/// BLAKE3 verification: hash both files and compare.
246pub(super) fn verify_blake3(src: &Path, dst: &Path) -> std::io::Result<bool> {
247    let src_hash = blake3::hash(&std::fs::read(src)?);
248    let dst_hash = blake3::hash(&std::fs::read(dst)?);
249    Ok(src_hash == dst_hash)
250}
251
252#[cfg(test)]
253mod tests {
254    #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
255    use super::*;
256    use std::fs;
257    use tempfile::TempDir;
258
259    fn empty_filter() -> crate::filter::FilterRules {
260        crate::filter::FilterRules::new(&[], &[])
261    }
262
263    #[test]
264    fn delete_preserves_dangling_symlinks() {
265        let src_dir = TempDir::new().unwrap();
266        let tgt_dir = TempDir::new().unwrap();
267
268        // Create a dangling symlink on source (points to nonexistent target)
269        let dangling_name = "dangling_link";
270        std::os::unix::fs::symlink("/nonexistent/path", src_dir.path().join(dangling_name)).unwrap();
271
272        // Create matching dangling symlink on target
273        std::os::unix::fs::symlink("/nonexistent/path", tgt_dir.path().join(dangling_name)).unwrap();
274
275        // Verify the symlink exists on target before delete
276        assert!(tgt_dir.path().join(dangling_name).symlink_metadata().is_ok());
277
278        let deleted = delete_extra_files(src_dir.path(), tgt_dir.path(), &empty_filter()).unwrap();
279        assert_eq!(deleted, 0);
280
281        // Dangling symlink must survive
282        assert!(tgt_dir.path().join(dangling_name).symlink_metadata().is_ok());
283    }
284
285    #[test]
286    fn delete_preserves_foxing_internal_dirs() {
287        let src_dir = TempDir::new().unwrap();
288        let tgt_dir = TempDir::new().unwrap();
289
290        // Create .foxing_* dirs on target only (not on source)
291        fs::create_dir(tgt_dir.path().join(".foxing_versions")).unwrap();
292        fs::create_dir(tgt_dir.path().join(".foxing_meta")).unwrap();
293        fs::create_dir(tgt_dir.path().join(".foxing_cas")).unwrap();
294        fs::create_dir(tgt_dir.path().join(".foxing_index")).unwrap();
295
296        let deleted = delete_extra_files(src_dir.path(), tgt_dir.path(), &empty_filter()).unwrap();
297        assert_eq!(deleted, 0);
298
299        // All .foxing_* dirs must survive
300        assert!(tgt_dir.path().join(".foxing_versions").exists());
301        assert!(tgt_dir.path().join(".foxing_meta").exists());
302        assert!(tgt_dir.path().join(".foxing_cas").exists());
303        assert!(tgt_dir.path().join(".foxing_index").exists());
304    }
305
306    #[test]
307    fn delete_removes_regular_extras() {
308        let src_dir = TempDir::new().unwrap();
309        let tgt_dir = TempDir::new().unwrap();
310
311        // Create a file on source so dirs aren't empty
312        fs::write(src_dir.path().join("keep.txt"), b"keep").unwrap();
313        fs::write(tgt_dir.path().join("keep.txt"), b"keep").unwrap();
314
315        // Create an extra file on target not present on source
316        fs::write(tgt_dir.path().join("extra.txt"), b"should be deleted").unwrap();
317
318        let deleted = delete_extra_files(src_dir.path(), tgt_dir.path(), &empty_filter()).unwrap();
319        assert_eq!(deleted, 1);
320
321        // extra.txt must be gone, keep.txt must survive
322        assert!(!tgt_dir.path().join("extra.txt").exists());
323        assert!(tgt_dir.path().join("keep.txt").exists());
324    }
325
326    /// `preserve_metadata` correctly copies permission bits from src to dst.
327    #[test]
328    fn test_preserve_metadata_preserves_permissions() {
329        use std::os::unix::fs::PermissionsExt;
330        let dir = TempDir::new().unwrap();
331        let src = dir.path().join("src_perms");
332        let dst = dir.path().join("dst_perms");
333
334        fs::write(&src, b"perms-source").unwrap();
335        fs::write(&dst, b"perms-dest").unwrap();
336
337        fs::set_permissions(&src, fs::Permissions::from_mode(0o755)).unwrap();
338        fs::set_permissions(&dst, fs::Permissions::from_mode(0o600)).unwrap();
339
340        assert_ne!(
341            fs::metadata(&src).unwrap().permissions().mode() & 0o7777,
342            fs::metadata(&dst).unwrap().permissions().mode() & 0o7777,
343            "precondition: src and dst modes must differ before preserve_metadata"
344        );
345
346        let opts = crate::sync::SyncOptions::default();
347        preserve_metadata(&src, &dst, &opts).unwrap();
348
349        let src_mode = fs::metadata(&src).unwrap().permissions().mode() & 0o7777;
350        let dst_mode = fs::metadata(&dst).unwrap().permissions().mode() & 0o7777;
351        assert_eq!(src_mode, dst_mode, "preserve_metadata must copy permission bits");
352    }
353
354    /// `preserve_metadata` correctly copies mtime from src to dst.
355    #[test]
356    fn test_preserve_metadata_preserves_timestamps() {
357        use std::os::unix::fs::MetadataExt;
358        let dir = TempDir::new().unwrap();
359        let src = dir.path().join("src_time");
360        let dst = dir.path().join("dst_time");
361
362        fs::write(&src, b"time-source").unwrap();
363        fs::write(&dst, b"time-dest").unwrap();
364
365        let target_mtime: i64 = 1_577_836_800; // 2020-01-01T00:00:00 UTC
366        let times = [
367            libc::timespec { tv_sec: target_mtime, tv_nsec: 0 },
368            libc::timespec { tv_sec: target_mtime, tv_nsec: 0 },
369        ];
370        let src_cstr = std::ffi::CString::new(src.as_os_str().as_encoded_bytes()).unwrap();
371        let ret = unsafe { libc::utimensat(libc::AT_FDCWD, src_cstr.as_ptr(), times.as_ptr(), 0) };
372        assert_eq!(ret, 0, "precondition: utimensat on src must succeed");
373
374        assert_ne!(
375            fs::metadata(&src).unwrap().mtime(),
376            fs::metadata(&dst).unwrap().mtime(),
377            "precondition: src and dst mtimes must differ"
378        );
379
380        let opts = crate::sync::SyncOptions::default();
381        preserve_metadata(&src, &dst, &opts).unwrap();
382
383        let src_mtime = fs::metadata(&src).unwrap().mtime();
384        let dst_mtime = fs::metadata(&dst).unwrap().mtime();
385        assert_eq!(src_mtime, dst_mtime, "preserve_metadata must copy mtime");
386    }
387
388    // requires root  --  run with: sudo cargo test -- --ignored test_preserve_metadata_sets_ownership_as_root
389    #[test]
390    #[ignore]
391    fn test_preserve_metadata_sets_ownership_as_root() {
392        use std::os::unix::fs::MetadataExt;
393        let dir = TempDir::new().unwrap();
394        let src = dir.path().join("src_chown");
395        let dst = dir.path().join("dst_chown");
396
397        fs::write(&src, b"chown-source").unwrap();
398        fs::write(&dst, b"chown-dest").unwrap();
399
400        // SAFETY: chown requires root; this test is #[ignore]d for non-root.
401        unsafe { libc::chown(
402            std::ffi::CString::new(src.as_os_str().as_encoded_bytes()).unwrap().as_ptr(),
403            65534, 65534,
404        ) };
405
406        let opts = crate::sync::SyncOptions::default();
407        preserve_metadata(&src, &dst, &opts).unwrap();
408
409        let dst_meta = fs::metadata(&dst).unwrap();
410        assert_eq!(dst_meta.uid(), 65534, "preserve_metadata must set uid via chown");
411        assert_eq!(dst_meta.gid(), 65534, "preserve_metadata must set gid via chown");
412    }
413
414    #[test]
415    fn test_preserve_metadata_skips_chown_as_nonroot() {
416        use std::os::unix::fs::PermissionsExt;
417        let tmp = tempfile::tempdir().unwrap();
418        let src = tmp.path().join("src.txt");
419        let dst = tmp.path().join("dst.txt");
420        std::fs::write(&src, b"test").unwrap();
421        std::fs::write(&dst, b"test").unwrap();
422
423        let opts = crate::sync::SyncOptions::default();
424        let result = preserve_metadata(&src, &dst, &opts);
425        assert!(result.is_ok(), "preserve_metadata should not fail as non-root: {:?}", result);
426
427        let src_meta = std::fs::metadata(&src).unwrap();
428        let dst_meta = std::fs::metadata(&dst).unwrap();
429        assert_eq!(src_meta.permissions().mode() & 0o777, dst_meta.permissions().mode() & 0o777);
430    }
431
432    #[test]
433    fn test_no_granular_perms_skips_permissions() {
434        use std::os::unix::fs::PermissionsExt;
435        let dir = TempDir::new().unwrap();
436        let src = dir.path().join("src_noperms");
437        let dst = dir.path().join("dst_noperms");
438
439        fs::write(&src, b"source").unwrap();
440        fs::write(&dst, b"dest").unwrap();
441
442        fs::set_permissions(&src, fs::Permissions::from_mode(0o755)).unwrap();
443        fs::set_permissions(&dst, fs::Permissions::from_mode(0o600)).unwrap();
444
445        let mut opts = crate::sync::SyncOptions::default();
446        opts.perms = false;
447
448        preserve_metadata(&src, &dst, &opts).unwrap();
449
450        let dst_mode = fs::metadata(&dst).unwrap().permissions().mode() & 0o777;
451        assert_eq!(dst_mode, 0o600, "perms=false must leave destination permissions unchanged");
452    }
453
454    #[test]
455    fn test_no_granular_xattrs_skips_xattrs() {
456        let dir = TempDir::new().unwrap();
457        let src = dir.path().join("src_noxattr");
458        let dst = dir.path().join("dst_noxattr");
459
460        fs::write(&src, b"source").unwrap();
461        fs::write(&dst, b"dest").unwrap();
462
463        let mut opts = crate::sync::SyncOptions::default();
464        opts.xattrs = false;
465
466        // Given: set an xattr on source
467        let _ = xattr::set(&src, "user.test_key", b"test_value");
468
469        preserve_metadata(&src, &dst, &opts).unwrap();
470
471        // Then: dst should NOT have the xattr (xattrs=false)
472        let dst_val = xattr::get(&dst, "user.test_key").ok().flatten();
473        assert!(dst_val.is_none(), "xattrs=false must skip xattr copy");
474    }
475}
fxcp_core/sync/tree.rs

fxcp_core/sync/
tree.rs
