416 lines
18 KiB
C#
416 lines
18 KiB
C#
using Dalamud.Game.ClientState.Objects.SubKinds;
|
|
using K4os.Compression.LZ4.Legacy;
|
|
using LightlessSync.API.Data;
|
|
using LightlessSync.API.Data.Enum;
|
|
using LightlessSync.API.Dto.CharaData;
|
|
using LightlessSync.FileCache;
|
|
using LightlessSync.LightlessConfiguration.Models;
|
|
using LightlessSync.PlayerData.Factories;
|
|
using LightlessSync.PlayerData.Handlers;
|
|
using LightlessSync.Services.CharaData;
|
|
using LightlessSync.Services.CharaData.Models;
|
|
using LightlessSync.Services.Mediator;
|
|
using LightlessSync.UI.Models;
|
|
using LightlessSync.Utils;
|
|
using LightlessSync.WebAPI.Files;
|
|
using Microsoft.Extensions.Logging;
|
|
|
|
namespace LightlessSync.Services;
|
|
|
|
public sealed class CharaDataFileHandler : IDisposable
|
|
{
|
|
private readonly DalamudUtilService _dalamudUtilService;
|
|
private readonly FileCacheManager _fileCacheManager;
|
|
private readonly FileDownloadManager _fileDownloadManager;
|
|
private readonly FileUploadManager _fileUploadManager;
|
|
private readonly GameObjectHandlerFactory _gameObjectHandlerFactory;
|
|
private readonly ILogger<CharaDataFileHandler> _logger;
|
|
private readonly LightlessCharaFileDataFactory _lightlessCharaFileDataFactory;
|
|
private readonly PlayerDataFactory _playerDataFactory;
|
|
private readonly NotificationService _notificationService;
|
|
private int _globalFileCounter = 0;
|
|
|
|
public CharaDataFileHandler(ILogger<CharaDataFileHandler> logger, FileDownloadManagerFactory fileDownloadManagerFactory, FileUploadManager fileUploadManager, FileCacheManager fileCacheManager,
|
|
DalamudUtilService dalamudUtilService, GameObjectHandlerFactory gameObjectHandlerFactory, PlayerDataFactory playerDataFactory, NotificationService notificationService)
|
|
{
|
|
_fileDownloadManager = fileDownloadManagerFactory.Create();
|
|
_logger = logger;
|
|
_fileUploadManager = fileUploadManager;
|
|
_fileCacheManager = fileCacheManager;
|
|
_dalamudUtilService = dalamudUtilService;
|
|
_gameObjectHandlerFactory = gameObjectHandlerFactory;
|
|
_playerDataFactory = playerDataFactory;
|
|
_notificationService = notificationService;
|
|
_lightlessCharaFileDataFactory = new(fileCacheManager);
|
|
}
|
|
|
|
public void ComputeMissingFiles(CharaDataDownloadDto charaDataDownloadDto, out Dictionary<string, string> modPaths, out List<FileReplacementData> missingFiles)
|
|
{
|
|
modPaths = [];
|
|
missingFiles = [];
|
|
foreach (var file in charaDataDownloadDto.FileGamePaths)
|
|
{
|
|
var localCacheFile = _fileCacheManager.GetFileCacheByHash(file.HashOrFileSwap);
|
|
if (localCacheFile == null)
|
|
{
|
|
var existingFile = missingFiles.Find(f => string.Equals(f.Hash, file.HashOrFileSwap, StringComparison.Ordinal));
|
|
if (existingFile == null)
|
|
{
|
|
missingFiles.Add(new FileReplacementData()
|
|
{
|
|
Hash = file.HashOrFileSwap,
|
|
GamePaths = [file.GamePath]
|
|
});
|
|
}
|
|
else
|
|
{
|
|
existingFile.GamePaths = existingFile.GamePaths.Concat([file.GamePath]).ToArray();
|
|
}
|
|
}
|
|
else
|
|
{
|
|
modPaths[file.GamePath] = localCacheFile.ResolvedFilepath;
|
|
}
|
|
}
|
|
|
|
foreach (var swap in charaDataDownloadDto.FileSwaps)
|
|
{
|
|
modPaths[swap.GamePath] = swap.HashOrFileSwap;
|
|
}
|
|
}
|
|
|
|
public async Task<CharacterData?> CreatePlayerData()
|
|
{
|
|
var chara = await _dalamudUtilService.GetPlayerCharacterAsync().ConfigureAwait(false);
|
|
if (_dalamudUtilService.IsInGpose)
|
|
{
|
|
chara = (IPlayerCharacter?)(await _dalamudUtilService.GetGposeCharacterFromObjectTableByNameAsync(chara.Name.TextValue, _dalamudUtilService.IsInGpose).ConfigureAwait(false));
|
|
}
|
|
|
|
if (chara == null)
|
|
return null;
|
|
|
|
using var tempHandler = await _gameObjectHandlerFactory.Create(ObjectKind.Player,
|
|
() => _dalamudUtilService.GetCharacterFromObjectTableByIndex(chara.ObjectIndex)?.Address ?? IntPtr.Zero, isWatched: false).ConfigureAwait(false);
|
|
PlayerData.Data.CharacterData newCdata = new();
|
|
var fragment = await _playerDataFactory.BuildCharacterData(tempHandler, CancellationToken.None).ConfigureAwait(false);
|
|
newCdata.SetFragment(ObjectKind.Player, fragment);
|
|
if (newCdata.FileReplacements.TryGetValue(ObjectKind.Player, out var playerData) && playerData != null)
|
|
{
|
|
foreach (var data in playerData.Select(g => g.GamePaths))
|
|
{
|
|
data.RemoveWhere(g => g.EndsWith(".pap", StringComparison.OrdinalIgnoreCase)
|
|
|| g.EndsWith(".tmb", StringComparison.OrdinalIgnoreCase)
|
|
|| g.EndsWith(".scd", StringComparison.OrdinalIgnoreCase)
|
|
|| (g.EndsWith(".avfx", StringComparison.OrdinalIgnoreCase)
|
|
&& !g.Contains("/weapon/", StringComparison.OrdinalIgnoreCase)
|
|
&& !g.Contains("/equipment/", StringComparison.OrdinalIgnoreCase))
|
|
|| (g.EndsWith(".atex", StringComparison.OrdinalIgnoreCase)
|
|
&& !g.Contains("/weapon/", StringComparison.OrdinalIgnoreCase)
|
|
&& !g.Contains("/equipment/", StringComparison.OrdinalIgnoreCase)));
|
|
}
|
|
|
|
playerData.RemoveWhere(g => g.GamePaths.Count == 0);
|
|
}
|
|
|
|
return newCdata.ToAPI();
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
_fileDownloadManager.Dispose();
|
|
}
|
|
|
|
public async Task DownloadFilesAsync(GameObjectHandler tempHandler, List<FileReplacementData> missingFiles, Dictionary<string, string> modPaths, CancellationToken token)
|
|
{
|
|
await _fileDownloadManager.InitiateDownloadList(tempHandler, missingFiles, token).ConfigureAwait(false);
|
|
await _fileDownloadManager.DownloadFiles(tempHandler, missingFiles, token).ConfigureAwait(false);
|
|
token.ThrowIfCancellationRequested();
|
|
foreach (var file in missingFiles.SelectMany(m => m.GamePaths, (FileEntry, GamePath) => (FileEntry.Hash, GamePath)))
|
|
{
|
|
var localFile = _fileCacheManager.GetFileCacheByHash(file.Hash)?.ResolvedFilepath;
|
|
if (localFile == null)
|
|
{
|
|
throw new FileNotFoundException("File not found locally.");
|
|
}
|
|
modPaths[file.GamePath] = localFile;
|
|
}
|
|
}
|
|
|
|
public Task<(LightlessCharaFileHeader loadedCharaFile, long expectedLength)> LoadCharaFileHeader(string filePath)
|
|
{
|
|
try
|
|
{
|
|
using var unwrapped = File.OpenRead(filePath);
|
|
using var lz4Stream = new LZ4Stream(unwrapped, LZ4StreamMode.Decompress, LZ4StreamFlags.HighCompression);
|
|
using var reader = new BinaryReader(lz4Stream);
|
|
var loadedCharaFile = LightlessCharaFileHeader.FromBinaryReader(filePath, reader);
|
|
|
|
_logger.LogInformation("Read Lightless Chara File");
|
|
_logger.LogInformation("Version: {ver}", (loadedCharaFile?.Version ?? -1));
|
|
long expectedLength = 0;
|
|
if (loadedCharaFile != null)
|
|
{
|
|
_logger.LogTrace("Data");
|
|
foreach (var item in loadedCharaFile.CharaFileData.FileSwaps)
|
|
{
|
|
foreach (var gamePath in item.GamePaths)
|
|
{
|
|
_logger.LogTrace("Swap: {gamePath} => {fileSwapPath}", gamePath, item.FileSwapPath);
|
|
}
|
|
}
|
|
|
|
var itemNr = 0;
|
|
foreach (var item in loadedCharaFile.CharaFileData.Files)
|
|
{
|
|
itemNr++;
|
|
expectedLength += item.Length;
|
|
foreach (var gamePath in item.GamePaths)
|
|
{
|
|
_logger.LogTrace("File {itemNr}: {gamePath} = {len}", itemNr, gamePath, item.Length.ToByteString());
|
|
}
|
|
}
|
|
|
|
_logger.LogInformation("Expected length: {expected}", expectedLength.ToByteString());
|
|
}
|
|
else
|
|
{
|
|
throw new InvalidOperationException("MCDF Header was null");
|
|
}
|
|
return Task.FromResult((loadedCharaFile, expectedLength));
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogWarning(ex, "Could not parse MCDF header of file {file}", filePath);
|
|
throw;
|
|
}
|
|
}
|
|
|
|
public Dictionary<string, string> McdfExtractFiles(LightlessCharaFileHeader? charaFileHeader, long expectedLength, List<string> extractedFiles)
|
|
{
|
|
if (charaFileHeader == null) return [];
|
|
|
|
using var lz4Stream = new LZ4Stream(File.OpenRead(charaFileHeader.FilePath), LZ4StreamMode.Decompress, LZ4StreamFlags.HighCompression);
|
|
using var reader = new BinaryReader(lz4Stream);
|
|
LightlessCharaFileHeader.AdvanceReaderToData(reader);
|
|
|
|
long totalRead = 0;
|
|
Dictionary<string, string> gamePathToFilePath = new(StringComparer.Ordinal);
|
|
foreach (var fileData in charaFileHeader.CharaFileData.Files)
|
|
{
|
|
var fileName = Path.Combine(_fileCacheManager.CacheFolder, "lightless_" + _globalFileCounter++ + ".tmp");
|
|
extractedFiles.Add(fileName);
|
|
var length = fileData.Length;
|
|
var bufferSize = length;
|
|
using var fs = File.OpenWrite(fileName);
|
|
using var wr = new BinaryWriter(fs);
|
|
_logger.LogTrace("Reading {length} of {fileName}", length.ToByteString(), fileName);
|
|
var buffer = reader.ReadBytes(bufferSize);
|
|
wr.Write(buffer);
|
|
wr.Flush();
|
|
wr.Close();
|
|
if (buffer.Length == 0) throw new EndOfStreamException("Unexpected EOF");
|
|
foreach (var path in fileData.GamePaths)
|
|
{
|
|
gamePathToFilePath[path] = fileName;
|
|
_logger.LogTrace("{path} => {fileName} [{hash}]", path, fileName, fileData.Hash);
|
|
}
|
|
totalRead += length;
|
|
_logger.LogTrace("Read {read}/{expected} bytes", totalRead.ToByteString(), expectedLength.ToByteString());
|
|
}
|
|
|
|
return gamePathToFilePath;
|
|
}
|
|
|
|
public async Task UpdateCharaDataAsync(CharaDataExtendedUpdateDto updateDto)
|
|
{
|
|
var data = await CreatePlayerData().ConfigureAwait(false);
|
|
|
|
if (data != null)
|
|
{
|
|
var hasGlamourerData = data.GlamourerData.TryGetValue(ObjectKind.Player, out var playerDataString);
|
|
if (!hasGlamourerData) updateDto.GlamourerData = null;
|
|
else updateDto.GlamourerData = playerDataString;
|
|
|
|
var hasCustomizeData = data.CustomizePlusData.TryGetValue(ObjectKind.Player, out var customizeDataString);
|
|
if (!hasCustomizeData) updateDto.CustomizeData = null;
|
|
else updateDto.CustomizeData = customizeDataString;
|
|
|
|
updateDto.ManipulationData = data.ManipulationData;
|
|
|
|
var hasFiles = data.FileReplacements.TryGetValue(ObjectKind.Player, out var fileReplacements);
|
|
if (!hasFiles)
|
|
{
|
|
updateDto.FileGamePaths = [];
|
|
updateDto.FileSwaps = [];
|
|
}
|
|
else
|
|
{
|
|
updateDto.FileGamePaths = [.. fileReplacements!.Where(u => string.IsNullOrEmpty(u.FileSwapPath)).SelectMany(u => u.GamePaths, (file, path) => new GamePathEntry(file.Hash, path))];
|
|
updateDto.FileSwaps = [.. fileReplacements!.Where(u => !string.IsNullOrEmpty(u.FileSwapPath)).SelectMany(u => u.GamePaths, (file, path) => new GamePathEntry(file.FileSwapPath, path))];
|
|
}
|
|
}
|
|
}
|
|
|
|
internal async Task SaveCharaFileAsync(string description, string filePath)
|
|
{
|
|
var createPlayerDataStopwatch = System.Diagnostics.Stopwatch.StartNew();
|
|
var data = await CreatePlayerData().ConfigureAwait(false);
|
|
createPlayerDataStopwatch.Stop();
|
|
_logger.LogInformation("CreatePlayerData took {elapsed}ms", createPlayerDataStopwatch.ElapsedMilliseconds);
|
|
|
|
if (data == null) return;
|
|
|
|
await Task.Run(async () => await SaveCharaFileAsyncInternal(description, filePath, data).ConfigureAwait(false)).ConfigureAwait(false);
|
|
}
|
|
|
|
private async Task SaveCharaFileAsyncInternal(string description, string filePath, CharacterData data)
|
|
{
|
|
var tempFilePath = filePath + ".tmp";
|
|
var overallStopwatch = System.Diagnostics.Stopwatch.StartNew();
|
|
|
|
try
|
|
{
|
|
var lightlessCharaFileData = _lightlessCharaFileDataFactory.Create(description, data);
|
|
LightlessCharaFileHeader output = new(LightlessCharaFileHeader.CurrentVersion, lightlessCharaFileData);
|
|
|
|
using var fs = new FileStream(tempFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.None, bufferSize: 65536, useAsync: false);
|
|
using var lz4 = new LZ4Stream(fs, LZ4StreamMode.Compress, LZ4StreamFlags.HighCompression);
|
|
using var writer = new BinaryWriter(lz4);
|
|
output.WriteToStream(writer);
|
|
|
|
int fileIndex = 0;
|
|
long totalBytesWritten = 0;
|
|
long totalBytesToWrite = output.CharaFileData.Files.Sum(f => f.Length);
|
|
var fileWriteStopwatch = System.Diagnostics.Stopwatch.StartNew();
|
|
const long updateIntervalMs = 1000;
|
|
|
|
foreach (var item in output.CharaFileData.Files)
|
|
{
|
|
fileIndex++;
|
|
var fileStopwatch = System.Diagnostics.Stopwatch.StartNew();
|
|
|
|
var file = _fileCacheManager.GetFileCacheByHash(item.Hash)!;
|
|
_logger.LogDebug("Saving to MCDF [{fileNum}/{totalFiles}]: {hash}:{file}", fileIndex, output.CharaFileData.Files.Count, item.Hash, file.ResolvedFilepath);
|
|
_logger.LogDebug("\tAssociated GamePaths:");
|
|
foreach (var path in item.GamePaths)
|
|
{
|
|
_logger.LogDebug("\t{path}", path);
|
|
}
|
|
|
|
using var fsRead = File.OpenRead(file.ResolvedFilepath);
|
|
using var br = new BinaryReader(fsRead);
|
|
byte[] buffer = new byte[item.Length];
|
|
int bytesRead = br.Read(buffer, 0, item.Length);
|
|
|
|
if (bytesRead != item.Length)
|
|
{
|
|
_logger.LogWarning("Expected to read {expected} bytes but got {actual} bytes from {file}", item.Length, bytesRead, file.ResolvedFilepath);
|
|
}
|
|
|
|
writer.Write(buffer);
|
|
totalBytesWritten += bytesRead;
|
|
|
|
fileStopwatch.Stop();
|
|
_logger.LogDebug("Wrote file [{fileNum}/{totalFiles}] in {elapsed}ms ({sizeKb}kb)", fileIndex, output.CharaFileData.Files.Count, fileStopwatch.ElapsedMilliseconds, item.Length / 1024);
|
|
|
|
if (fileWriteStopwatch.ElapsedMilliseconds >= updateIntervalMs && totalBytesToWrite > 0)
|
|
{
|
|
float progress = (float)totalBytesWritten / totalBytesToWrite;
|
|
var elapsed = overallStopwatch.Elapsed;
|
|
var eta = CalculateEta(elapsed, progress);
|
|
|
|
var notification = new LightlessNotification
|
|
{
|
|
Id = "chara_file_save_progress",
|
|
Title = "Character Data",
|
|
Message = $"Compressing and saving character file... {(progress * 100):F0}%\nETA: {FormatTimespan(eta)}",
|
|
Type = NotificationType.Info,
|
|
Duration = TimeSpan.FromMinutes(5),
|
|
ShowProgress = true,
|
|
Progress = progress
|
|
};
|
|
|
|
_notificationService.Mediator.Publish(new LightlessNotificationMessage(notification));
|
|
|
|
fileWriteStopwatch.Restart();
|
|
}
|
|
}
|
|
|
|
var flushStopwatch = System.Diagnostics.Stopwatch.StartNew();
|
|
writer.Flush();
|
|
lz4.Flush();
|
|
fs.Flush();
|
|
fs.Close();
|
|
flushStopwatch.Stop();
|
|
_logger.LogInformation("Flush operations took {elapsed}ms", flushStopwatch.ElapsedMilliseconds);
|
|
|
|
var moveStopwatch = System.Diagnostics.Stopwatch.StartNew();
|
|
File.Move(tempFilePath, filePath, true);
|
|
moveStopwatch.Stop();
|
|
_logger.LogInformation("File move took {elapsed}ms", moveStopwatch.ElapsedMilliseconds);
|
|
|
|
overallStopwatch.Stop();
|
|
_logger.LogInformation("SaveCharaFileAsync completed successfully in {elapsed}ms. Total bytes written: {totalBytes}mb", overallStopwatch.ElapsedMilliseconds, totalBytesWritten / (1024 * 1024));
|
|
|
|
_notificationService.ShowNotification(
|
|
"Character Data",
|
|
"Character file saved successfully!",
|
|
NotificationType.Info,
|
|
duration: TimeSpan.FromSeconds(5));
|
|
|
|
_notificationService.Mediator.Publish(new LightlessNotificationDismissMessage("chara_file_save_progress"));
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
overallStopwatch.Stop();
|
|
_logger.LogError(ex, "Failure Saving Lightless Chara File after {elapsed}ms, deleting output", overallStopwatch.ElapsedMilliseconds);
|
|
try
|
|
{
|
|
File.Delete(tempFilePath);
|
|
}
|
|
catch (Exception deleteEx)
|
|
{
|
|
_logger.LogError(deleteEx, "Failed to delete temporary file {file}", tempFilePath);
|
|
}
|
|
|
|
_notificationService.ShowErrorNotification(
|
|
"Character Data Save Failed",
|
|
"Failed to save character file",
|
|
ex);
|
|
|
|
_notificationService.Mediator.Publish(new LightlessNotificationDismissMessage("chara_file_save_progress"));
|
|
}
|
|
}
|
|
|
|
private static TimeSpan CalculateEta(TimeSpan elapsed, float progress)
|
|
{
|
|
if (progress <= 0 || elapsed.TotalSeconds < 0.1)
|
|
return TimeSpan.Zero;
|
|
|
|
double totalSeconds = elapsed.TotalSeconds / progress;
|
|
double remainingSeconds = totalSeconds - elapsed.TotalSeconds;
|
|
|
|
return TimeSpan.FromSeconds(Math.Max(0, remainingSeconds));
|
|
}
|
|
|
|
private static string FormatTimespan(TimeSpan ts)
|
|
{
|
|
if (ts.TotalSeconds < 1)
|
|
return "< 1s";
|
|
|
|
if (ts.TotalSeconds < 60)
|
|
return $"{ts.TotalSeconds:F0}s";
|
|
|
|
if (ts.TotalMinutes < 60)
|
|
return $"{ts.TotalMinutes:F1}m";
|
|
|
|
return $"{ts.TotalHours:F1}h";
|
|
}
|
|
|
|
internal async Task<List<string>> UploadFiles(List<string> fileList, ValueProgress<string> uploadProgress, CancellationToken token)
|
|
{
|
|
return await _fileUploadManager.UploadFiles(fileList, uploadProgress, token).ConfigureAwait(false);
|
|
}
|
|
}
|