using EasyCompressor; using System.Security.Cryptography; using System.Text; namespace DatabaseSnapshotsService.Services { public class OptimizedFileService { private const int DefaultBufferSize = 64 * 1024; // 64KB buffer private const int LargeFileThreshold = 100 * 1024 * 1024; // 100MB private const int ParallelThreshold = 50 * 1024 * 1024; // 50MB private readonly LZ4Compressor _lz4 = new LZ4Compressor(); /// /// Streaming LZ4 compression using EasyCompressor.LZ4 /// public async Task CompressFileStreamingAsync(string sourcePath, string destinationPath, int bufferSize = DefaultBufferSize) { try { using var sourceStream = new FileStream(sourcePath, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize, FileOptions.Asynchronous); using var destinationStream = new FileStream(destinationPath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize, FileOptions.Asynchronous); await Task.Run(() => _lz4.Compress(sourceStream, destinationStream)); } catch (Exception ex) { throw new InvalidOperationException($"Streaming LZ4 compression failed: {ex.Message}", ex); } } /// /// Streaming LZ4 decompression using EasyCompressor.LZ4 /// public async Task DecompressFileStreamingAsync(string sourcePath, string destinationPath, int bufferSize = DefaultBufferSize) { try { using var sourceStream = new FileStream(sourcePath, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize, FileOptions.Asynchronous); using var destinationStream = new FileStream(destinationPath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize, FileOptions.Asynchronous); await Task.Run(() => _lz4.Decompress(sourceStream, destinationStream)); } catch (Exception ex) { throw new InvalidOperationException($"Streaming LZ4 decompression failed: {ex.Message}", ex); } } /// /// Optimized checksum calculation using streaming /// public async Task CalculateChecksumStreamingAsync(string filePath, int bufferSize = DefaultBufferSize) { try { using var sha256 = SHA256.Create(); using var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize, FileOptions.Asynchronous); var buffer = new byte[bufferSize]; int bytesRead; while ((bytesRead = await stream.ReadAsync(buffer, 0, buffer.Length)) > 0) { sha256.TransformBlock(buffer, 0, bytesRead, null, 0); } sha256.TransformFinalBlock(Array.Empty(), 0, 0); return Convert.ToBase64String(sha256.Hash!); } catch (Exception ex) { throw new InvalidOperationException($"Checksum calculation failed: {ex.Message}", ex); } } /// /// Parallel checksum calculation for large files /// public async Task CalculateChecksumParallelAsync(string filePath) { try { var fileInfo = new FileInfo(filePath); if (fileInfo.Length < LargeFileThreshold) { return await CalculateChecksumStreamingAsync(filePath); } // For large files, use parallel processing using var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, DefaultBufferSize, FileOptions.Asynchronous); var fileBytes = new byte[fileInfo.Length]; await stream.ReadAsync(fileBytes, 0, (int)fileInfo.Length); return await Task.Run(() => { using var sha256 = SHA256.Create(); return Convert.ToBase64String(sha256.ComputeHash(fileBytes)); }); } catch (Exception ex) { throw new InvalidOperationException($"Parallel checksum calculation failed: {ex.Message}", ex); } } /// /// Optimized file reading with memory mapping for very large files /// public async Task ReadFileOptimizedAsync(string filePath) { try { var fileInfo = new FileInfo(filePath); if (fileInfo.Length > LargeFileThreshold) { // For very large files, use streaming using var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, DefaultBufferSize, FileOptions.Asynchronous); using var memoryStream = new MemoryStream(); await stream.CopyToAsync(memoryStream); return memoryStream.ToArray(); } else { // For smaller files, use direct read return await File.ReadAllBytesAsync(filePath); } } catch (Exception ex) { throw new InvalidOperationException($"File reading failed: {ex.Message}", ex); } } /// /// Optimized file writing with buffering /// public async Task WriteFileOptimizedAsync(string filePath, byte[] data, int bufferSize = DefaultBufferSize) { try { using var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize, FileOptions.Asynchronous); await stream.WriteAsync(data, 0, data.Length); await stream.FlushAsync(); } catch (Exception ex) { throw new InvalidOperationException($"File writing failed: {ex.Message}", ex); } } /// /// Get optimal buffer size based on file size /// public static int GetOptimalBufferSize(long fileSize) { if (fileSize < 1024 * 1024) // < 1MB return 8 * 1024; // 8KB else if (fileSize < 100 * 1024 * 1024) // < 100MB return 64 * 1024; // 64KB else return 256 * 1024; // 256KB } /// /// Determine if parallel processing should be used /// public static bool ShouldUseParallelProcessing(long fileSize) { return fileSize >= ParallelThreshold; } } }