1087 lines
47 KiB
C#
1087 lines
47 KiB
C#
using System;
|
|
using System.IO;
|
|
using System.Text.Json;
|
|
using CommandLine;
|
|
using Crayon;
|
|
using DatabaseSnapshotsService.Models;
|
|
using DatabaseSnapshotsService.Services;
|
|
using Microsoft.Extensions.Configuration;
|
|
using System.Threading.Tasks;
|
|
using System.Text;
|
|
using System.Linq;
|
|
using System.Collections.Generic;
|
|
using System.ComponentModel.DataAnnotations;
|
|
using MySqlConnector;
|
|
|
|
namespace DatabaseSnapshotsService
|
|
{
|
|
class Program
|
|
{
|
|
static async Task<int> Main(string[] args)
|
|
{
|
|
var parser = new Parser(settings =>
|
|
{
|
|
settings.CaseSensitive = false;
|
|
settings.HelpWriter = Console.Out;
|
|
});
|
|
|
|
var result = parser.ParseArguments<SnapshotOptions, RecoveryOptions, ConfigOptions, BinlogOptions, EventsOptions, RestoreOptions>(args);
|
|
|
|
return await result.MapResult(
|
|
(SnapshotOptions opts) => HandleSnapshotCommand(opts),
|
|
(RecoveryOptions opts) => HandleRecoveryCommand(opts),
|
|
(ConfigOptions opts) => HandleConfigCommand(opts),
|
|
(BinlogOptions opts) => HandleBinlogCommand(opts),
|
|
(EventsOptions opts) => HandleEventsCommand(opts),
|
|
(RestoreOptions opts) => HandleRestoreCommand(opts),
|
|
errors => Task.FromResult(1)
|
|
);
|
|
}
|
|
|
|
static async Task<int> HandleSnapshotCommand(SnapshotOptions options)
|
|
{
|
|
try
|
|
{
|
|
// Validate and sanitize command input
|
|
var sanitizedCommand = InputValidation.SanitizeString(options.Command);
|
|
if (string.IsNullOrWhiteSpace(sanitizedCommand))
|
|
{
|
|
Output.Red("Error: Command cannot be empty");
|
|
return 1;
|
|
}
|
|
|
|
var config = LoadConfiguration(options.ConfigFile);
|
|
var snapshotService = new SnapshotService(config);
|
|
|
|
switch (sanitizedCommand.ToLower())
|
|
{
|
|
case "create":
|
|
// Validate snapshot name
|
|
var nameValidation = InputValidation.SnapshotValidation.ValidateSnapshotName(options.Name);
|
|
if (nameValidation != ValidationResult.Success)
|
|
{
|
|
Output.Red($"Error: {nameValidation.ErrorMessage}");
|
|
return 1;
|
|
}
|
|
|
|
// Validate snapshot type
|
|
var typeValidation = InputValidation.SnapshotValidation.ValidateSnapshotType(options.Type);
|
|
if (typeValidation != ValidationResult.Success)
|
|
{
|
|
Output.Red($"Error: {typeValidation.ErrorMessage}");
|
|
return 1;
|
|
}
|
|
|
|
if (options.Type.ToLower() == "incremental")
|
|
{
|
|
Output.Green($"Creating incremental snapshot: {options.Name}");
|
|
var snapshot = await snapshotService.CreateIncrementalSnapshotAsync(options.Name);
|
|
Output.Green($"Incremental snapshot created successfully: {snapshot.Id}");
|
|
Output.Yellow($"File: {snapshot.FilePath}");
|
|
Output.Yellow($"Size: {snapshot.DataSize} bytes");
|
|
}
|
|
else
|
|
{
|
|
Output.Green($"Creating snapshot: {options.Name}");
|
|
var snapshot = await snapshotService.CreateSnapshotAsync(options.Name);
|
|
Output.Green($"Snapshot created successfully: {snapshot.Id}");
|
|
Output.Yellow($"File: {snapshot.FilePath}");
|
|
Output.Yellow($"Size: {snapshot.DataSize} bytes");
|
|
}
|
|
break;
|
|
|
|
case "list":
|
|
var snapshots = await snapshotService.ListSnapshotsAsync();
|
|
Console.WriteLine("Available snapshots:");
|
|
|
|
// Group snapshots by type and sort by timestamp
|
|
var fullSnapshots = snapshots.Where(s => s.Type.Equals("Full", StringComparison.OrdinalIgnoreCase))
|
|
.OrderByDescending(s => s.CreatedAt)
|
|
.ToList();
|
|
|
|
var incrementalSnapshots = snapshots.Where(s => s.Type.Equals("Incremental", StringComparison.OrdinalIgnoreCase))
|
|
.OrderByDescending(s => s.CreatedAt)
|
|
.ToList();
|
|
|
|
// Display full snapshots first
|
|
foreach (var snapshot in fullSnapshots)
|
|
{
|
|
var timestamp = snapshot.CreatedAt.ToString("yyyy-MM-dd HH:mm:ss");
|
|
var size = snapshot.DataSize switch
|
|
{
|
|
< 1024 => $"{snapshot.DataSize} bytes",
|
|
< 1024 * 1024 => $"{snapshot.DataSize / 1024:N0} KB",
|
|
_ => $"{snapshot.DataSize / (1024 * 1024):N1} MB"
|
|
};
|
|
|
|
Console.WriteLine($" {snapshot.Id}: {snapshot.Description} ({timestamp})");
|
|
Console.WriteLine($" File: {Path.GetFileName(snapshot.FilePath)}");
|
|
Console.WriteLine($" Size: {size}, Status: {snapshot.Status}");
|
|
|
|
// Find all incremental snapshots that belong to this full snapshot (directly or indirectly)
|
|
var allRelatedIncrementals = GetAllRelatedIncrementals(snapshot.Id, incrementalSnapshots)
|
|
.OrderByDescending(inc => inc.CreatedAt)
|
|
.ToList();
|
|
|
|
foreach (var inc in allRelatedIncrementals)
|
|
{
|
|
var incTimestamp = inc.CreatedAt.ToString("yyyy-MM-dd HH:mm:ss");
|
|
var incSize = inc.DataSize switch
|
|
{
|
|
< 1024 => $"{inc.DataSize} bytes",
|
|
< 1024 * 1024 => $"{inc.DataSize / 1024:N0} KB",
|
|
_ => $"{inc.DataSize / (1024 * 1024):N1} MB"
|
|
};
|
|
|
|
Console.WriteLine($" └─ {inc.Id}: {inc.Description} ({incTimestamp})");
|
|
Console.WriteLine($" File: {Path.GetFileName(inc.FilePath)}");
|
|
Console.WriteLine($" Size: {incSize}, Status: {inc.Status}");
|
|
}
|
|
|
|
Console.WriteLine();
|
|
}
|
|
break;
|
|
|
|
case "show":
|
|
// Validate snapshot ID
|
|
if (!int.TryParse(options.Name, out var showId))
|
|
{
|
|
Output.Red("Error: Invalid snapshot ID. Must be a positive integer.");
|
|
return 1;
|
|
}
|
|
|
|
var showIdValidation = InputValidation.SnapshotValidation.ValidateSnapshotId(showId);
|
|
if (showIdValidation != ValidationResult.Success)
|
|
{
|
|
Output.Red($"Error: {showIdValidation.ErrorMessage}");
|
|
return 1;
|
|
}
|
|
|
|
Output.Green($"Snapshot details for: {options.Name}");
|
|
var snapshotDetails = await snapshotService.GetSnapshotAsync(showId);
|
|
if (snapshotDetails != null)
|
|
{
|
|
Output.White($" ID: {snapshotDetails.Id}");
|
|
Output.White($" Created: {snapshotDetails.CreatedAt:yyyy-MM-dd HH:mm:ss}");
|
|
Output.White($" File: {snapshotDetails.FilePath}");
|
|
Output.White($" Size: {snapshotDetails.DataSize:N0} bytes");
|
|
Output.White($" Status: {snapshotDetails.Status}");
|
|
}
|
|
else
|
|
{
|
|
Output.Red($"Snapshot '{options.Name}' not found");
|
|
return 1;
|
|
}
|
|
break;
|
|
|
|
case "delete":
|
|
// Validate snapshot ID
|
|
if (!int.TryParse(options.Name, out var deleteId))
|
|
{
|
|
Output.Red("Error: Invalid snapshot ID. Must be a positive integer.");
|
|
return 1;
|
|
}
|
|
|
|
var deleteIdValidation = InputValidation.SnapshotValidation.ValidateSnapshotId(deleteId);
|
|
if (deleteIdValidation != ValidationResult.Success)
|
|
{
|
|
Output.Red($"Error: {deleteIdValidation.ErrorMessage}");
|
|
return 1;
|
|
}
|
|
|
|
Output.Yellow($"Deleting snapshot: {options.Name}");
|
|
await snapshotService.DeleteSnapshotAsync(deleteId);
|
|
Output.Green("Snapshot deleted successfully");
|
|
break;
|
|
|
|
default:
|
|
Output.Red($"Unknown snapshot command: {sanitizedCommand}");
|
|
return 1;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Output.Red($"Error: {ex.Message}");
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
static async Task<int> HandleRecoveryCommand(RecoveryOptions options)
|
|
{
|
|
try
|
|
{
|
|
// Validate and sanitize command input
|
|
var sanitizedCommand = InputValidation.SanitizeString(options.Command);
|
|
if (string.IsNullOrWhiteSpace(sanitizedCommand))
|
|
{
|
|
Output.Red("Error: Command cannot be empty");
|
|
return 1;
|
|
}
|
|
|
|
var config = LoadConfiguration(options.ConfigFile);
|
|
var recoveryService = new RecoveryService(config);
|
|
|
|
switch (sanitizedCommand)
|
|
{
|
|
case "create-point":
|
|
// Validate recovery point name
|
|
var nameValidation = InputValidation.RecoveryValidation.ValidateRecoveryPointName(options.Name);
|
|
if (nameValidation != ValidationResult.Success)
|
|
{
|
|
Output.Red($"Error: {nameValidation.ErrorMessage}");
|
|
return 1;
|
|
}
|
|
|
|
Output.Green($"Creating recovery point: {options.Name}");
|
|
var point = await recoveryService.CreateRecoveryPointAsync(options.Name);
|
|
Output.Green($"Recovery point created: {point.Id}");
|
|
Output.Yellow($"Name: {point.Name}");
|
|
Output.Yellow($"Timestamp: {point.CreatedAt:yyyy-MM-dd HH:mm:ss}");
|
|
break;
|
|
|
|
case "list-points":
|
|
Output.Green("Available recovery points:");
|
|
var points = await recoveryService.ListRecoveryPointsAsync();
|
|
foreach (var p in points)
|
|
{
|
|
Output.White($" {p.Id}: {p.Name} ({p.CreatedAt:yyyy-MM-dd HH:mm:ss})");
|
|
Output.Gray($" Event Count: {p.EventCount}");
|
|
}
|
|
break;
|
|
|
|
case "restore":
|
|
// Validate recovery point name
|
|
var restoreNameValidation = InputValidation.RecoveryValidation.ValidateRecoveryPointName(options.Name);
|
|
if (restoreNameValidation != ValidationResult.Success)
|
|
{
|
|
Output.Red($"Error: {restoreNameValidation.ErrorMessage}");
|
|
return 1;
|
|
}
|
|
|
|
Output.Yellow($"Restoring to recovery point: {options.Name}");
|
|
if (options.DryRun)
|
|
{
|
|
Output.Green("DRY RUN - No actual restore performed");
|
|
var plan = await recoveryService.PreviewRestoreAsync(DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
|
Output.White($" Target Timestamp: {plan.TargetTimestamp}");
|
|
Output.White($" Event Count: {plan.EventCount}");
|
|
Output.White($" Affected Tables: {string.Join(", ", plan.AffectedTables)}");
|
|
}
|
|
else
|
|
{
|
|
await recoveryService.RestoreAsync(DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
|
Output.Green("Restore completed successfully");
|
|
}
|
|
break;
|
|
|
|
default:
|
|
Output.Red($"Unknown recovery command: {sanitizedCommand}");
|
|
return 1;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Output.Red($"Error: {ex.Message}");
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
static async Task<int> HandleConfigCommand(ConfigOptions options)
|
|
{
|
|
try
|
|
{
|
|
var config = LoadConfiguration(options.ConfigFile);
|
|
Output.Green("Current configuration:");
|
|
Output.White($" Binlog Host: {config.BinlogReader.Host}");
|
|
Output.White($" Binlog Port: {config.BinlogReader.Port}");
|
|
Output.White($" Binlog Username: {config.BinlogReader.Username}");
|
|
Output.White($" Snapshots Path: {config.SnapshotStorage.Path}");
|
|
Output.White($" Compression: {config.SnapshotStorage.Compression}");
|
|
Output.White($" Retention Days: {config.SnapshotStorage.RetentionDays}");
|
|
return 0;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Output.Red($"Error: {ex.Message}");
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
static async Task<int> HandleBinlogCommand(BinlogOptions options)
|
|
{
|
|
try
|
|
{
|
|
Console.WriteLine("Loading configuration...");
|
|
var config = LoadConfiguration(options.ConfigFile);
|
|
Console.WriteLine($"Configuration loaded. Binlog host: {config.BinlogReader.Host}:{config.BinlogReader.Port}");
|
|
|
|
var eventStore = new EventStore(config.EventStore);
|
|
var binlogReader = new BinlogReader(config.BinlogReader, eventStore);
|
|
|
|
// Set up event handlers
|
|
binlogReader.LogMessage += (sender, message) =>
|
|
{
|
|
if (options.Verbose)
|
|
Output.Gray($"[LOG] {message}");
|
|
else
|
|
Console.WriteLine($"[LOG] {message}");
|
|
};
|
|
|
|
binlogReader.EventReceived += (sender, evt) =>
|
|
{
|
|
// Only show individual events in verbose mode
|
|
if (options.Verbose)
|
|
{
|
|
var eventType = evt.EventType.ToString();
|
|
var timestamp = evt.Timestamp.ToString("HH:mm:ss.fff");
|
|
var position = evt.LogPosition;
|
|
|
|
Console.WriteLine($"[{timestamp}] {eventType} @ {position}");
|
|
Output.White($"[{timestamp}] {eventType} @ {position}");
|
|
|
|
if (evt.EventData != null)
|
|
{
|
|
var hexData = BitConverter.ToString(evt.EventData, 0, Math.Min(50, evt.EventData.Length));
|
|
var eventInfo = Encoding.UTF8.GetString(evt.EventData);
|
|
Console.WriteLine($" Info: {eventInfo}");
|
|
Output.Gray($" Info: {eventInfo}");
|
|
}
|
|
}
|
|
};
|
|
|
|
// Connect and start reading
|
|
Output.Green("Connecting to MySQL binlog...");
|
|
if (!await binlogReader.ConnectAsync())
|
|
{
|
|
Output.Red("Failed to connect to MySQL");
|
|
return 1;
|
|
}
|
|
|
|
Output.Green($"Starting binlog read from position {options.Position}");
|
|
if (!string.IsNullOrEmpty(options.BinlogFile))
|
|
{
|
|
Output.Yellow($"Using binlog file: {options.BinlogFile}");
|
|
}
|
|
|
|
// Start reading in background
|
|
var readTask = binlogReader.StartReadingAsync(options.BinlogFile, options.Position);
|
|
|
|
Output.Green("Binlog reading started. Press Ctrl+C to stop.");
|
|
Output.Yellow("Make some database changes to see events...");
|
|
|
|
// Wait for user to stop
|
|
Console.CancelKeyPress += (sender, e) =>
|
|
{
|
|
e.Cancel = true;
|
|
Output.Yellow("\nStopping binlog reader...");
|
|
binlogReader.StopReading();
|
|
};
|
|
|
|
await readTask;
|
|
binlogReader.Disconnect();
|
|
|
|
Output.Green("Binlog reading stopped");
|
|
return 0;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Output.Red($"Error: {ex.Message}");
|
|
Console.WriteLine($"Exception details: {ex}");
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
static async Task<int> HandleEventsCommand(EventsOptions options)
|
|
{
|
|
try
|
|
{
|
|
var config = LoadConfiguration(options.ConfigFile);
|
|
|
|
// Validate limit
|
|
var limitValidation = InputValidation.EventValidation.ValidateLimit(options.Limit);
|
|
if (limitValidation != ValidationResult.Success)
|
|
{
|
|
Output.Red($"Error: {limitValidation.ErrorMessage}");
|
|
return 1;
|
|
}
|
|
|
|
// Validate table name if provided
|
|
if (!string.IsNullOrEmpty(options.Table))
|
|
{
|
|
var tableValidation = InputValidation.EventValidation.ValidateTableName(options.Table);
|
|
if (tableValidation != ValidationResult.Success)
|
|
{
|
|
Output.Red($"Error: {tableValidation.ErrorMessage}");
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
// Validate operation if provided
|
|
if (!string.IsNullOrEmpty(options.Operation))
|
|
{
|
|
var operationValidation = InputValidation.EventValidation.ValidateOperation(options.Operation);
|
|
if (operationValidation != ValidationResult.Success)
|
|
{
|
|
Output.Red($"Error: {operationValidation.ErrorMessage}");
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
// Validate event type if provided
|
|
if (!string.IsNullOrEmpty(options.EventType))
|
|
{
|
|
var eventTypeValidation = InputValidation.EventValidation.ValidateEventType(options.EventType);
|
|
if (eventTypeValidation != ValidationResult.Success)
|
|
{
|
|
Output.Red($"Error: {eventTypeValidation.ErrorMessage}");
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
// Validate data contains filter if provided
|
|
if (!string.IsNullOrEmpty(options.DataContains))
|
|
{
|
|
var dataContainsValidation = InputValidation.DataValidation.ValidateDataContains(options.DataContains);
|
|
if (dataContainsValidation != ValidationResult.Success)
|
|
{
|
|
Output.Red($"Error: {dataContainsValidation.ErrorMessage}");
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
// Validate output file if provided
|
|
if (!string.IsNullOrEmpty(options.OutputFile))
|
|
{
|
|
var outputFileValidation = InputValidation.DataValidation.ValidateOutputFile(options.OutputFile);
|
|
if (outputFileValidation != ValidationResult.Success)
|
|
{
|
|
Output.Red($"Error: {outputFileValidation.ErrorMessage}");
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
// Validate timestamps if provided
|
|
if (options.FromTimestamp > 0)
|
|
{
|
|
var fromTimestampValidation = InputValidation.EventValidation.ValidateTimestamp(options.FromTimestamp);
|
|
if (fromTimestampValidation != ValidationResult.Success)
|
|
{
|
|
Output.Red($"Error: {fromTimestampValidation.ErrorMessage}");
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
if (options.ToTimestamp > 0)
|
|
{
|
|
var toTimestampValidation = InputValidation.EventValidation.ValidateTimestamp(options.ToTimestamp);
|
|
if (toTimestampValidation != ValidationResult.Success)
|
|
{
|
|
Output.Red($"Error: {toTimestampValidation.ErrorMessage}");
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
// Validate snapshot ID
|
|
var snapshotIdValidation = InputValidation.SnapshotValidation.ValidateSnapshotId(options.SnapshotId);
|
|
if (snapshotIdValidation != ValidationResult.Success)
|
|
{
|
|
Output.Red($"Error: {snapshotIdValidation.ErrorMessage}");
|
|
return 1;
|
|
}
|
|
|
|
var snapshotService = new SnapshotService(config);
|
|
var snapshot = await snapshotService.GetSnapshotAsync(options.SnapshotId);
|
|
|
|
if (snapshot == null)
|
|
{
|
|
Output.Red($"Error: Snapshot {options.SnapshotId} not found");
|
|
return 1;
|
|
}
|
|
|
|
if (!snapshot.Type.Equals("Incremental", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
Output.Red($"Error: Snapshot {options.SnapshotId} is not an incremental snapshot");
|
|
return 1;
|
|
}
|
|
|
|
// Validate that output option is only used with sql-only
|
|
if (!string.IsNullOrEmpty(options.OutputFile) && (options.NoSqlOnly || !options.SqlOnly))
|
|
{
|
|
Output.Red("Error: --output option can only be used with --sql-only");
|
|
return 1;
|
|
}
|
|
|
|
// Determine if we should use SQL-only mode
|
|
var useSqlOnly = options.SqlOnly && !options.NoSqlOnly;
|
|
|
|
Output.Green($"Reading events from incremental snapshot {options.SnapshotId}: {snapshot.Description}");
|
|
Output.White($"File: {snapshot.FilePath}");
|
|
Output.White($"Created: {snapshot.CreatedAt:yyyy-MM-dd HH:mm:ss}");
|
|
Console.WriteLine();
|
|
|
|
// Create a temporary options object with the correct sql-only setting
|
|
var tempOptions = new EventsOptions
|
|
{
|
|
ConfigFile = options.ConfigFile,
|
|
SnapshotId = options.SnapshotId,
|
|
Table = options.Table,
|
|
Limit = options.Limit,
|
|
FromTimestamp = options.FromTimestamp,
|
|
ToTimestamp = options.ToTimestamp,
|
|
CountOnly = options.CountOnly,
|
|
Operation = options.Operation,
|
|
EventType = options.EventType,
|
|
DataContains = options.DataContains,
|
|
SqlOnly = useSqlOnly,
|
|
OutputFile = options.OutputFile
|
|
};
|
|
|
|
await ReadEventsFromIncrementalSnapshot(snapshot, tempOptions);
|
|
return 0;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Output.Red($"Error: {ex.Message}");
|
|
if (ex.InnerException != null)
|
|
{
|
|
Output.Red($"Inner error: {ex.InnerException.Message}");
|
|
}
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
static async Task<int> HandleRestoreCommand(RestoreOptions options)
|
|
{
|
|
try
|
|
{
|
|
var config = LoadConfiguration(options.ConfigFile);
|
|
var snapshotService = new SnapshotService(config);
|
|
var recoveryService = new RecoveryService(config);
|
|
|
|
// Validate snapshot exists
|
|
var snapshot = await snapshotService.GetSnapshotAsync(options.FromSnapshot);
|
|
if (snapshot == null)
|
|
{
|
|
Console.WriteLine($"Error: Snapshot {options.FromSnapshot} not found");
|
|
return 1;
|
|
}
|
|
|
|
Console.WriteLine($"Starting restore from snapshot {options.FromSnapshot}");
|
|
Console.WriteLine($"Snapshot: {snapshot.Description} created at {snapshot.CreatedAt:yyyy-MM-dd HH:mm:ss}");
|
|
|
|
if (options.DryRun)
|
|
{
|
|
Console.WriteLine("=== DRY RUN MODE ===");
|
|
Console.WriteLine($"Would restore database from snapshot {options.FromSnapshot}");
|
|
Console.WriteLine("No changes would be made to the database");
|
|
return 0;
|
|
}
|
|
|
|
// Perform the actual restore
|
|
await recoveryService.RestoreAsync(snapshot.Timestamp);
|
|
|
|
Console.WriteLine("Recovery completed successfully!");
|
|
return 0;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"Error: {ex.Message}");
|
|
Console.WriteLine($"Exception details: {ex}");
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
static SnapshotConfiguration LoadConfiguration(string configFile)
|
|
{
|
|
// Validate and sanitize the config file path
|
|
var sanitizedConfigFile = InputValidation.SanitizePath(configFile ?? string.Empty);
|
|
var configPath = string.IsNullOrEmpty(sanitizedConfigFile) ? "appsettings.json" : sanitizedConfigFile;
|
|
|
|
// Validate file path
|
|
var pathValidation = InputValidation.FileValidation.ValidateFilePath(configPath);
|
|
if (pathValidation != ValidationResult.Success)
|
|
{
|
|
throw new ArgumentException($"Invalid configuration file path: {pathValidation.ErrorMessage}");
|
|
}
|
|
|
|
if (!File.Exists(configPath))
|
|
{
|
|
throw new FileNotFoundException($"Configuration file not found: {configPath}");
|
|
}
|
|
|
|
try
|
|
{
|
|
var configuration = new ConfigurationBuilder()
|
|
.SetBasePath(Directory.GetCurrentDirectory())
|
|
.AddJsonFile(configPath, optional: false)
|
|
.Build();
|
|
|
|
var config = new SnapshotConfiguration();
|
|
configuration.Bind(config);
|
|
|
|
// Validate the loaded configuration
|
|
var validationResult = ConfigurationValidation.ValidateConfiguration(config);
|
|
if (validationResult.Count > 0)
|
|
{
|
|
var errorMessages = string.Join("\n", validationResult.Select(e => e.ErrorMessage));
|
|
throw new InvalidOperationException($"Configuration validation failed:\n{errorMessages}");
|
|
}
|
|
|
|
return config;
|
|
}
|
|
catch (Exception ex) when (ex is not InvalidOperationException)
|
|
{
|
|
throw new InvalidOperationException($"Failed to load configuration from {configPath}: {ex.Message}", ex);
|
|
}
|
|
}
|
|
|
|
static List<SnapshotInfo> GetAllRelatedIncrementals(int snapshotId, List<SnapshotInfo> incrementalSnapshots)
|
|
{
|
|
var relatedIncrementals = new List<SnapshotInfo>();
|
|
var queue = new Queue<int>(new[] { snapshotId });
|
|
var visited = new HashSet<int>();
|
|
|
|
while (queue.Count > 0)
|
|
{
|
|
var currentId = queue.Dequeue();
|
|
if (visited.Contains(currentId)) continue;
|
|
visited.Add(currentId);
|
|
|
|
// Find all incremental snapshots that have this snapshot as their parent
|
|
var children = incrementalSnapshots.Where(s => s.ParentSnapshotId == currentId).ToList();
|
|
foreach (var child in children)
|
|
{
|
|
if (!visited.Contains(child.Id))
|
|
{
|
|
relatedIncrementals.Add(child);
|
|
queue.Enqueue(child.Id);
|
|
}
|
|
}
|
|
}
|
|
|
|
return relatedIncrementals;
|
|
}
|
|
|
|
static async Task ReadEventsFromIncrementalSnapshot(SnapshotInfo snapshot, EventsOptions options)
|
|
{
|
|
try
|
|
{
|
|
if (!File.Exists(snapshot.FilePath))
|
|
{
|
|
Console.WriteLine($"Error: Snapshot file not found: {snapshot.FilePath}");
|
|
return;
|
|
}
|
|
|
|
// Read the file with decryption/decompression support
|
|
var content = await ReadSnapshotFileWithDecryptionAsync(snapshot.FilePath, options.ConfigFile);
|
|
var lines = content.Split('\n');
|
|
|
|
// Parse and filter the output
|
|
var events = ParseIncrementalSnapshotOutput(lines, options);
|
|
|
|
if (options.CountOnly)
|
|
{
|
|
Console.WriteLine($"Total events in snapshot: {events.Count}");
|
|
return;
|
|
}
|
|
|
|
// Apply filters
|
|
if (!string.IsNullOrEmpty(options.Table))
|
|
{
|
|
events = events.Where(e => e.Contains($"### {options.Table}") || e.Contains($"`{options.Table}`")).ToList();
|
|
}
|
|
|
|
if (!string.IsNullOrEmpty(options.Operation))
|
|
{
|
|
events = events.Where(e => e.Contains($"### {options.Operation}") || e.Contains($"{options.Operation.ToUpper()} ")).ToList();
|
|
}
|
|
|
|
if (!string.IsNullOrEmpty(options.EventType))
|
|
{
|
|
events = events.Where(e => e.Contains($"### {options.EventType}") || e.Contains($"{options.EventType.ToUpper()} ")).ToList();
|
|
}
|
|
|
|
if (!string.IsNullOrEmpty(options.DataContains))
|
|
{
|
|
events = events.Where(e => e.Contains(options.DataContains, StringComparison.OrdinalIgnoreCase)).ToList();
|
|
}
|
|
|
|
// Apply timestamp filters if specified
|
|
if (options.FromTimestamp > 0 || options.ToTimestamp > 0)
|
|
{
|
|
events = events.Where(e =>
|
|
{
|
|
// Extract timestamp from event if possible
|
|
var timestampMatch = System.Text.RegularExpressions.Regex.Match(e, @"#(\d{10,13})");
|
|
if (timestampMatch.Success && long.TryParse(timestampMatch.Groups[1].Value, out var eventTimestamp))
|
|
{
|
|
if (options.FromTimestamp > 0 && eventTimestamp < options.FromTimestamp) return false;
|
|
if (options.ToTimestamp > 0 && eventTimestamp > options.ToTimestamp) return false;
|
|
}
|
|
return true;
|
|
}).ToList();
|
|
}
|
|
|
|
// Apply limit
|
|
events = events.Take(options.Limit).ToList();
|
|
|
|
// Save to file if output option is specified
|
|
if (!string.IsNullOrEmpty(options.OutputFile))
|
|
{
|
|
try
|
|
{
|
|
var sqlContent = string.Join("\n\n", events);
|
|
await File.WriteAllTextAsync(options.OutputFile, sqlContent);
|
|
Console.WriteLine($"SQL queries saved to: {options.OutputFile}");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"Error saving to file: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
Console.WriteLine($"Found {events.Count} events in snapshot:");
|
|
Console.WriteLine();
|
|
|
|
foreach (var evt in events)
|
|
{
|
|
Console.WriteLine(evt);
|
|
Console.WriteLine();
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"Error reading snapshot events: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
static async Task<string> ReadSnapshotFileWithDecryptionAsync(string filePath, string configFile)
|
|
{
|
|
try
|
|
{
|
|
var config = LoadConfiguration(configFile);
|
|
var fileService = new OptimizedFileService();
|
|
|
|
// Initialize encryption service - match RecoveryService pattern
|
|
var encryptionService = new EncryptionService(
|
|
config.Security.EncryptionKey,
|
|
config.Security.Encryption
|
|
);
|
|
|
|
// Check if file is encrypted and compressed
|
|
if (filePath.EndsWith(".lz4.enc"))
|
|
{
|
|
// First decrypt, then decompress
|
|
var decryptedPath = filePath.Replace(".lz4.enc", ".lz4.tmp");
|
|
var decompressedPath = filePath.Replace(".lz4.enc", ".sql.tmp");
|
|
|
|
try
|
|
{
|
|
// Decrypt the file
|
|
await encryptionService.DecryptFileAsync(filePath, decryptedPath);
|
|
|
|
// Decompress the decrypted file
|
|
await fileService.DecompressFileStreamingAsync(decryptedPath, decompressedPath);
|
|
|
|
// Read the final SQL content
|
|
var content = await fileService.ReadFileOptimizedAsync(decompressedPath);
|
|
return Encoding.UTF8.GetString(content);
|
|
}
|
|
finally
|
|
{
|
|
// Clean up temporary files
|
|
if (File.Exists(decryptedPath)) File.Delete(decryptedPath);
|
|
if (File.Exists(decompressedPath)) File.Delete(decompressedPath);
|
|
}
|
|
}
|
|
else if (filePath.EndsWith(".lz4"))
|
|
{
|
|
// Only compressed, not encrypted
|
|
var tempPath = filePath.Replace(".lz4", ".tmp");
|
|
await fileService.DecompressFileStreamingAsync(filePath, tempPath);
|
|
|
|
var content = await fileService.ReadFileOptimizedAsync(tempPath);
|
|
File.Delete(tempPath); // Clean up temp file
|
|
return Encoding.UTF8.GetString(content);
|
|
}
|
|
else if (filePath.EndsWith(".enc"))
|
|
{
|
|
// Only encrypted, not compressed
|
|
var tempPath = filePath.Replace(".enc", ".tmp");
|
|
await encryptionService.DecryptFileAsync(filePath, tempPath);
|
|
|
|
var content = await fileService.ReadFileOptimizedAsync(tempPath);
|
|
File.Delete(tempPath); // Clean up temp file
|
|
return Encoding.UTF8.GetString(content);
|
|
}
|
|
else
|
|
{
|
|
// Plain text file
|
|
var content = await fileService.ReadFileOptimizedAsync(filePath);
|
|
return Encoding.UTF8.GetString(content);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
throw new InvalidOperationException($"Failed to read snapshot file {filePath}: {ex.Message}", ex);
|
|
}
|
|
}
|
|
|
|
static List<string> ParseIncrementalSnapshotOutput(string[] lines, EventsOptions options)
|
|
{
|
|
if (options.SqlOnly)
|
|
{
|
|
return ExtractSqlQueries(lines);
|
|
}
|
|
|
|
var events = new List<string>();
|
|
var currentEvent = new List<string>();
|
|
var inEvent = false;
|
|
|
|
foreach (var line in lines)
|
|
{
|
|
// Start of a new event
|
|
if (line.StartsWith("# at ") || line.StartsWith("#") && line.Contains("server id"))
|
|
{
|
|
if (inEvent && currentEvent.Count > 0)
|
|
{
|
|
events.Add(string.Join("\n", currentEvent));
|
|
currentEvent.Clear();
|
|
}
|
|
inEvent = true;
|
|
currentEvent.Add(line);
|
|
}
|
|
// SQL statements and data
|
|
else if (inEvent && (line.StartsWith("### ") || line.StartsWith("SET ") || line.StartsWith("INSERT ") ||
|
|
line.StartsWith("UPDATE ") || line.StartsWith("DELETE ") || line.StartsWith("CREATE ") ||
|
|
line.StartsWith("ALTER ") || line.StartsWith("DROP ")))
|
|
{
|
|
currentEvent.Add(line);
|
|
}
|
|
// End of event (empty line or new section)
|
|
else if (inEvent && string.IsNullOrWhiteSpace(line))
|
|
{
|
|
if (currentEvent.Count > 0)
|
|
{
|
|
events.Add(string.Join("\n", currentEvent));
|
|
currentEvent.Clear();
|
|
}
|
|
inEvent = false;
|
|
}
|
|
// Continue adding lines to current event
|
|
else if (inEvent)
|
|
{
|
|
currentEvent.Add(line);
|
|
}
|
|
}
|
|
|
|
// Add the last event if any
|
|
if (inEvent && currentEvent.Count > 0)
|
|
{
|
|
events.Add(string.Join("\n", currentEvent));
|
|
}
|
|
|
|
return events;
|
|
}
|
|
|
|
static List<string> ExtractSqlQueries(string[] lines)
|
|
{
|
|
var sqlQueries = new List<string>();
|
|
var currentQuery = new List<string>();
|
|
var inQuery = false;
|
|
|
|
foreach (var line in lines)
|
|
{
|
|
// Look for SQL query annotations
|
|
if (line.StartsWith("#Q> "))
|
|
{
|
|
if (inQuery && currentQuery.Count > 0)
|
|
{
|
|
sqlQueries.Add(string.Join("\n", currentQuery));
|
|
currentQuery.Clear();
|
|
}
|
|
inQuery = true;
|
|
currentQuery.Add(line.Substring(4)); // Remove "#Q> " prefix
|
|
}
|
|
// Look for direct SQL statements
|
|
else if (line.StartsWith("INSERT ") || line.StartsWith("UPDATE ") ||
|
|
line.StartsWith("DELETE ") || line.StartsWith("CREATE ") ||
|
|
line.StartsWith("ALTER ") || line.StartsWith("DROP ") ||
|
|
line.StartsWith("SET ") || line.StartsWith("START ") ||
|
|
line.StartsWith("COMMIT") || line.StartsWith("ROLLBACK"))
|
|
{
|
|
if (inQuery && currentQuery.Count > 0)
|
|
{
|
|
sqlQueries.Add(string.Join("\n", currentQuery));
|
|
currentQuery.Clear();
|
|
}
|
|
inQuery = true;
|
|
currentQuery.Add(line);
|
|
}
|
|
// Look for SET statements that are part of transactions
|
|
else if (line.StartsWith("/*M!") && line.Contains("SET"))
|
|
{
|
|
// Extract the SET statement from MariaDB-specific comments
|
|
var setMatch = System.Text.RegularExpressions.Regex.Match(line, @"SET[^;]+;");
|
|
if (setMatch.Success)
|
|
{
|
|
if (inQuery && currentQuery.Count > 0)
|
|
{
|
|
sqlQueries.Add(string.Join("\n", currentQuery));
|
|
currentQuery.Clear();
|
|
}
|
|
inQuery = true;
|
|
currentQuery.Add(setMatch.Value);
|
|
}
|
|
}
|
|
// End of query (empty line or new section)
|
|
else if (inQuery && (string.IsNullOrWhiteSpace(line) || line.StartsWith("# at ")))
|
|
{
|
|
if (currentQuery.Count > 0)
|
|
{
|
|
sqlQueries.Add(string.Join("\n", currentQuery));
|
|
currentQuery.Clear();
|
|
}
|
|
inQuery = false;
|
|
}
|
|
// Continue adding lines to current query if we're in a multi-line statement
|
|
else if (inQuery && !line.StartsWith("#") && !string.IsNullOrWhiteSpace(line))
|
|
{
|
|
currentQuery.Add(line);
|
|
}
|
|
}
|
|
|
|
// Add the last query if any
|
|
if (inQuery && currentQuery.Count > 0)
|
|
{
|
|
sqlQueries.Add(string.Join("\n", currentQuery));
|
|
}
|
|
|
|
return sqlQueries.Where(q => !string.IsNullOrWhiteSpace(q)).ToList();
|
|
}
|
|
}
|
|
|
|
[Verb("snapshot", HelpText = "Manage database snapshots")]
|
|
class SnapshotOptions
|
|
{
|
|
[Option('c', "command", Required = true, HelpText = "Command: create, list, show, delete")]
|
|
public string Command { get; set; }
|
|
|
|
[Option('n', "name", HelpText = "Snapshot name")]
|
|
public string Name { get; set; }
|
|
|
|
[Option('f', "config", HelpText = "Configuration file path")]
|
|
public string ConfigFile { get; set; }
|
|
|
|
[Option("type", HelpText = "Snapshot type: full or incremental", Default = "full")]
|
|
public string Type { get; set; }
|
|
}
|
|
|
|
[Verb("recovery", HelpText = "Manage recovery points and restore operations")]
|
|
class RecoveryOptions
|
|
{
|
|
[Option('c', "command", Required = true, HelpText = "Command: create-point, list-points, restore")]
|
|
public string Command { get; set; }
|
|
|
|
[Option('n', "name", HelpText = "Recovery point name")]
|
|
public string Name { get; set; }
|
|
|
|
[Option('f', "config", HelpText = "Configuration file path")]
|
|
public string ConfigFile { get; set; }
|
|
|
|
[Option('d', "dry-run", HelpText = "Perform dry run for restore operations")]
|
|
public bool DryRun { get; set; }
|
|
}
|
|
|
|
[Verb("config", HelpText = "Show current configuration")]
|
|
class ConfigOptions
|
|
{
|
|
[Option('f', "config", HelpText = "Configuration file path")]
|
|
public string ConfigFile { get; set; }
|
|
}
|
|
|
|
[Verb("binlog", HelpText = "Read MySQL binlog events in real-time")]
|
|
class BinlogOptions
|
|
{
|
|
[Option('f', "config", HelpText = "Configuration file path")]
|
|
public string ConfigFile { get; set; }
|
|
|
|
[Option('b', "binlog-file", HelpText = "Binlog file name (optional)")]
|
|
public string BinlogFile { get; set; }
|
|
|
|
[Option('p', "position", Default = 4L, HelpText = "Starting position in binlog")]
|
|
public long Position { get; set; }
|
|
|
|
[Option('v', "verbose", HelpText = "Verbose output")]
|
|
public bool Verbose { get; set; }
|
|
}
|
|
|
|
[Verb("events", HelpText = "Query events from incremental snapshots")]
|
|
public class EventsOptions
|
|
{
|
|
[Option('f', "config", Required = true, HelpText = "Configuration file path")]
|
|
public string ConfigFile { get; set; } = string.Empty;
|
|
|
|
[Option('s', "snapshot", Required = true, HelpText = "Incremental snapshot ID to read events from")]
|
|
public int SnapshotId { get; set; }
|
|
|
|
[Option('t', "table", HelpText = "Filter by table name")]
|
|
public string? Table { get; set; }
|
|
|
|
[Option('l', "limit", Default = 100, HelpText = "Maximum number of events to return")]
|
|
public int Limit { get; set; } = 100;
|
|
|
|
[Option("from", HelpText = "Filter events from timestamp")]
|
|
public long FromTimestamp { get; set; }
|
|
|
|
[Option("to", HelpText = "Filter events to timestamp")]
|
|
public long ToTimestamp { get; set; }
|
|
|
|
[Option('c', "count", HelpText = "Show only count of events")]
|
|
public bool CountOnly { get; set; } = false;
|
|
|
|
[Option('o', "operation", HelpText = "Filter by operation type (insert, update, delete, query, etc.)")]
|
|
public string? Operation { get; set; }
|
|
|
|
[Option('e', "event-type", HelpText = "Filter by event type (binlog, status, etc.)")]
|
|
public string? EventType { get; set; }
|
|
|
|
[Option('k', "data-contains", HelpText = "Filter by keyword in event data (case-insensitive)")]
|
|
public string? DataContains { get; set; }
|
|
|
|
[Option('q', "sql-only", HelpText = "Extract and display only SQL queries from binlog events", Default = true)]
|
|
public bool SqlOnly { get; set; } = true;
|
|
|
|
[Option("no-sql-only", HelpText = "Disable SQL-only mode and show full binlog events")]
|
|
public bool NoSqlOnly { get; set; } = false;
|
|
|
|
[Option('o', "output", HelpText = "Save extracted SQL queries to specified file (only available with --sql-only)")]
|
|
public string? OutputFile { get; set; }
|
|
}
|
|
|
|
[Verb("restore", HelpText = "Restore database from snapshot")]
|
|
public class RestoreOptions
|
|
{
|
|
[Option('f', "config", Required = true, HelpText = "Configuration file path")]
|
|
public string ConfigFile { get; set; }
|
|
|
|
[Option("from-snapshot", Required = true, HelpText = "Source snapshot ID")]
|
|
public int FromSnapshot { get; set; }
|
|
|
|
[Option("dry-run", HelpText = "Preview recovery without performing it")]
|
|
public bool DryRun { get; set; }
|
|
}
|
|
|
|
static class Output
|
|
{
|
|
public static void Green(string s) => Console.WriteLine(Crayon.Output.Green(s));
|
|
public static void Red(string s) => Console.WriteLine(Crayon.Output.Red(s));
|
|
public static void Yellow(string s) => Console.WriteLine(Crayon.Output.Yellow(s));
|
|
public static void White(string s) => Console.WriteLine(Crayon.Output.White(s));
|
|
public static void Gray(string s) => Console.WriteLine(Crayon.Output.Dim(s));
|
|
}
|
|
} |