add project

Services/OptimizedFileService.cs

@@ -0,0 +1,172 @@
+using EasyCompressor;
+using System.Security.Cryptography;
+using System.Text;
+
+namespace DatabaseSnapshotsService.Services
+{
+    public class OptimizedFileService
+    {
+        private const int DefaultBufferSize = 64 * 1024; // 64KB buffer
+        private const int LargeFileThreshold = 100 * 1024 * 1024; // 100MB
+        private const int ParallelThreshold = 50 * 1024 * 1024; // 50MB
+        private readonly LZ4Compressor _lz4 = new LZ4Compressor();
+
+        /// <summary>
+        /// Streaming LZ4 compression using EasyCompressor.LZ4
+        /// </summary>
+        public async Task CompressFileStreamingAsync(string sourcePath, string destinationPath, int bufferSize = DefaultBufferSize)
+        {
+            try
+            {
+                using var sourceStream = new FileStream(sourcePath, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize, FileOptions.Asynchronous);
+                using var destinationStream = new FileStream(destinationPath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize, FileOptions.Asynchronous);
+                await Task.Run(() => _lz4.Compress(sourceStream, destinationStream));
+            }
+            catch (Exception ex)
+            {
+                throw new InvalidOperationException($"Streaming LZ4 compression failed: {ex.Message}", ex);
+            }
+        }
+
+        /// <summary>
+        /// Streaming LZ4 decompression using EasyCompressor.LZ4
+        /// </summary>
+        public async Task DecompressFileStreamingAsync(string sourcePath, string destinationPath, int bufferSize = DefaultBufferSize)
+        {
+            try
+            {
+                using var sourceStream = new FileStream(sourcePath, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize, FileOptions.Asynchronous);
+                using var destinationStream = new FileStream(destinationPath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize, FileOptions.Asynchronous);
+                await Task.Run(() => _lz4.Decompress(sourceStream, destinationStream));
+            }
+            catch (Exception ex)
+            {
+                throw new InvalidOperationException($"Streaming LZ4 decompression failed: {ex.Message}", ex);
+            }
+        }
+
+        /// <summary>
+        /// Optimized checksum calculation using streaming
+        /// </summary>
+        public async Task<string> CalculateChecksumStreamingAsync(string filePath, int bufferSize = DefaultBufferSize)
+        {
+            try
+            {
+                using var sha256 = SHA256.Create();
+                using var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize, FileOptions.Asynchronous);
+
+                var buffer = new byte[bufferSize];
+                int bytesRead;
+
+                while ((bytesRead = await stream.ReadAsync(buffer, 0, buffer.Length)) > 0)
+                {
+                    sha256.TransformBlock(buffer, 0, bytesRead, null, 0);
+                }
+
+                sha256.TransformFinalBlock(Array.Empty<byte>(), 0, 0);
+                return Convert.ToBase64String(sha256.Hash!);
+            }
+            catch (Exception ex)
+            {
+                throw new InvalidOperationException($"Checksum calculation failed: {ex.Message}", ex);
+            }
+        }
+
+        /// <summary>
+        /// Parallel checksum calculation for large files
+        /// </summary>
+        public async Task<string> CalculateChecksumParallelAsync(string filePath)
+        {
+            try
+            {
+                var fileInfo = new FileInfo(filePath);
+                if (fileInfo.Length < LargeFileThreshold)
+                {
+                    return await CalculateChecksumStreamingAsync(filePath);
+                }
+
+                // For large files, use parallel processing
+                using var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, DefaultBufferSize, FileOptions.Asynchronous);
+                var fileBytes = new byte[fileInfo.Length];
+                await stream.ReadAsync(fileBytes, 0, (int)fileInfo.Length);
+
+                return await Task.Run(() =>
+                {
+                    using var sha256 = SHA256.Create();
+                    return Convert.ToBase64String(sha256.ComputeHash(fileBytes));
+                });
+            }
+            catch (Exception ex)
+            {
+                throw new InvalidOperationException($"Parallel checksum calculation failed: {ex.Message}", ex);
+            }
+        }
+
+        /// <summary>
+        /// Optimized file reading with memory mapping for very large files
+        /// </summary>
+        public async Task<byte[]> ReadFileOptimizedAsync(string filePath)
+        {
+            try
+            {
+                var fileInfo = new FileInfo(filePath);
+                
+                if (fileInfo.Length > LargeFileThreshold)
+                {
+                    // For very large files, use streaming
+                    using var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, DefaultBufferSize, FileOptions.Asynchronous);
+                    using var memoryStream = new MemoryStream();
+                    await stream.CopyToAsync(memoryStream);
+                    return memoryStream.ToArray();
+                }
+                else
+                {
+                    // For smaller files, use direct read
+                    return await File.ReadAllBytesAsync(filePath);
+                }
+            }
+            catch (Exception ex)
+            {
+                throw new InvalidOperationException($"File reading failed: {ex.Message}", ex);
+            }
+        }
+
+        /// <summary>
+        /// Optimized file writing with buffering
+        /// </summary>
+        public async Task WriteFileOptimizedAsync(string filePath, byte[] data, int bufferSize = DefaultBufferSize)
+        {
+            try
+            {
+                using var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize, FileOptions.Asynchronous);
+                await stream.WriteAsync(data, 0, data.Length);
+                await stream.FlushAsync();
+            }
+            catch (Exception ex)
+            {
+                throw new InvalidOperationException($"File writing failed: {ex.Message}", ex);
+            }
+        }
+
+        /// <summary>
+        /// Get optimal buffer size based on file size
+        /// </summary>
+        public static int GetOptimalBufferSize(long fileSize)
+        {
+            if (fileSize < 1024 * 1024) // < 1MB
+                return 8 * 1024; // 8KB
+            else if (fileSize < 100 * 1024 * 1024) // < 100MB
+                return 64 * 1024; // 64KB
+            else
+                return 256 * 1024; // 256KB
+        }
+
+        /// <summary>
+        /// Determine if parallel processing should be used
+        /// </summary>
+        public static bool ShouldUseParallelProcessing(long fileSize)
+        {
+            return fileSize >= ParallelThreshold;
+        }
+    }
+}