Partial support of sparse files backup

Author: sakno

src/DotNext.Tests/Net/Cluster/Consensus/Raft/MemoryBasedStateMachineTests.cs

@@ -839,6 +839,37 @@ public static async Task RestoreBackup()
         }
     }
 
+    [Fact]
+    public static async Task CreateSparseBackup()
+    {
+        var entry1 = new TestLogEntry("SET X = 0") { Term = 42L };
+        var entry2 = new TestLogEntry("SET Y = 1") { Term = 43L };
+        var entry3 = new TestLogEntry("SET Z = 2") { Term = 44L };
+        var entry4 = new TestLogEntry("SET U = 3") { Term = 45L };
+        var entry5 = new TestLogEntry("SET V = 4") { Term = 46L };
+        var dir = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
+        var backupFile = Path.GetTempFileName();
+        IPersistentState state = new PersistentStateWithoutSnapshot(dir, RecordsPerPartition, new() { MaxLogEntrySize = 1024 * 1024, BackupFormat = System.Formats.Tar.TarEntryFormat.Gnu });
+        var member = ClusterMemberId.FromEndPoint(new IPEndPoint(IPAddress.IPv6Loopback, 3232));
+        try
+        {
+            //define node state
+            Equal(1, await state.IncrementTermAsync(member));
+            True(state.IsVotedFor(member));
+            //define log entries
+            Equal(1L, await state.AppendAsync(new LogEntryList(entry1, entry2, entry3, entry4, entry5)));
+            //commit some of them
+            Equal(2L, await state.CommitAsync(2L));
+            //save backup
+            await using var backupStream = new FileStream(backupFile, FileMode.Truncate, FileAccess.Write, FileShare.None, 1024, true);
+            await state.CreateBackupAsync(backupStream);
+        }
+        finally
+        {
+            (state as IDisposable)?.Dispose();
+        }
+    }
+
     [Fact]
     public static async Task Reconstruction()
     {

src/cluster/DotNext.Net.Cluster/Net/Cluster/Consensus/Raft/PersistentState.Backup.cs

@@ -46,6 +46,12 @@ private static void ImportAttributes(SafeFileHandle handle, TarEntry entry)
 
             File.SetUnixFileMode(handle, entry.Mode);
         }
+
+        var attributes = FileAttributes.NotContentIndexed;
+        if (entry.EntryType is TarEntryType.SparseFile)
+            attributes |= FileAttributes.SparseFile;
+
+        File.SetAttributes(handle, attributes);
     }
 
     /// <summary>
@@ -55,7 +61,64 @@ private static void ImportAttributes(SafeFileHandle handle, TarEntry entry)
     /// <param name="token">The token that can be used to cancel the operation.</param>
     /// <returns>A task representing state of asynchronous execution.</returns>
     /// <exception cref="OperationCanceledException">The operation has been canceled.</exception>
-    public async Task CreateBackupAsync(Stream output, CancellationToken token = default)
+    public Task CreateBackupAsync(Stream output, CancellationToken token = default)
+        => maxLogEntrySize.HasValue ? CreateSparseBackupAsync(output, token) : CreateRegularBackupAsync(output, token);
+
+    private async Task CreateSparseBackupAsync(Stream output, CancellationToken token)
+    {
+        var tarProcess = new Process
+        {
+            StartInfo = new()
+            {
+                FileName = "tar",
+                WorkingDirectory = Location.FullName,
+            },
+        };
+
+        var outputArchive = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
+        tarProcess.StartInfo.ArgumentList.Add("cfS");
+        tarProcess.StartInfo.ArgumentList.Add(outputArchive);
+
+        FileStream? archiveStream = null;
+        await syncRoot.AcquireAsync(LockType.StrongReadLock, token).ConfigureAwait(false);
+        try
+        {
+            foreach (var file in Location.EnumerateFiles())
+            {
+                tarProcess.StartInfo.ArgumentList.Add(file.Name);
+            }
+
+            tarProcess.StartInfo.ArgumentList.Add($"--format={GetArchiveFormat(backupFormat)}");
+            tarProcess.Start();
+            await tarProcess.WaitForExitAsync(token).ConfigureAwait(false);
+
+            if (tarProcess.ExitCode is not 0)
+                throw new InvalidOperationException() { HResult = tarProcess.ExitCode };
+
+            archiveStream = new(outputArchive, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.SequentialScan | FileOptions.Asynchronous | FileOptions.DeleteOnClose);
+            await archiveStream.CopyToAsync(output, token).ConfigureAwait(false);
+            await output.FlushAsync(token).ConfigureAwait(false);
+        }
+        finally
+        {
+            syncRoot.Release(LockType.StrongReadLock);
+            tarProcess.Dispose();
+
+            if (archiveStream is not null)
+                await archiveStream.DisposeAsync().ConfigureAwait(false);
+        }
+
+        static string GetArchiveFormat(TarEntryFormat format) => format switch
+        {
+            TarEntryFormat.Gnu => "gnu",
+            TarEntryFormat.Pax => "pax",
+            TarEntryFormat.Ustar => "ustar",
+            TarEntryFormat.V7 => "v7",
+            _ => "gnu",
+        };
+    }
+
+    private async Task CreateRegularBackupAsync(Stream output, CancellationToken token)
     {
         TarWriter? archive = null;
         await syncRoot.AcquireAsync(LockType.StrongReadLock, token).ConfigureAwait(false);

src/cluster/DotNext.Net.Cluster/Net/Cluster/Consensus/Raft/PersistentState.Options.cs

@@ -138,6 +138,9 @@ public int MaxConcurrentReads
         /// </summary>
         /// <remarks>
         /// If enabled, WAL uses sparse files to optimize performance.
+        /// <see cref="CreateBackupAsync(Stream, CancellationToken)"/> method supports backup of sparse
+        /// files on Linux only. <see cref="RestoreFromBackupAsync(Stream, DirectoryInfo, CancellationToken)"/>
+        /// method cannot restore the backup, you need to use <c>tar</c> utility to extract files.
         /// </remarks>
         public long? MaxLogEntrySize
         {