From 8aad714918bc21e9820034c3a96aebbc2c9f0fa3 Mon Sep 17 00:00:00 2001 From: choco Date: Thu, 23 Oct 2025 00:40:54 +0200 Subject: [PATCH 01/28] removed wrong ondisconnect notification --- LightlessSync/Services/BroadcastService.cs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/LightlessSync/Services/BroadcastService.cs b/LightlessSync/Services/BroadcastService.cs index 81e54d5..cca9af6 100644 --- a/LightlessSync/Services/BroadcastService.cs +++ b/LightlessSync/Services/BroadcastService.cs @@ -144,11 +144,7 @@ public class BroadcastService : IHostedService, IMediatorSubscriber IsLightFinderAvailable = false; ApplyBroadcastDisabled(forcePublish: true); _logger.LogDebug("Cleared Lightfinder state due to disconnect."); - - _mediator.Publish(new NotificationMessage( - "Disconnected from Server", - "Your Lightfinder broadcast has been disabled due to disconnection.", - NotificationType.Warning)); + } public Task StartAsync(CancellationToken cancellationToken) From 55e78e088a22036b3088965cd168ab83b12b9a4f Mon Sep 17 00:00:00 2001 From: defnotken Date: Fri, 24 Oct 2025 09:45:24 -0500 Subject: [PATCH 02/28] Initialize .4 --- LightlessSync/LightlessSync.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LightlessSync/LightlessSync.csproj b/LightlessSync/LightlessSync.csproj index b4b5288..726f2ef 100644 --- a/LightlessSync/LightlessSync.csproj +++ b/LightlessSync/LightlessSync.csproj @@ -3,7 +3,7 @@ - 1.12.3 + 1.12.4 https://github.com/Light-Public-Syncshells/LightlessClient From 437731749fc817d0abd80d05a8e336d5fb3dd0ea Mon Sep 17 00:00:00 2001 From: choco Date: Sun, 26 Oct 2025 17:34:17 +0100 Subject: [PATCH 03/28] pair button now has additional checks to show if the user isnt directly paired, and only shows if your own lightfinder is on --- LightlessSync/Services/ContextMenuService.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/LightlessSync/Services/ContextMenuService.cs b/LightlessSync/Services/ContextMenuService.cs index 97bfc17..42ab72a 100644 --- a/LightlessSync/Services/ContextMenuService.cs +++ b/LightlessSync/Services/ContextMenuService.cs @@ -98,7 +98,7 @@ internal class ContextMenuService : IHostedService if (targetData == null || targetData.Address == nint.Zero) return; - //Check if user is paired or is own. + //Check if user is directly paired or is own. if (VisibleUserIds.Any(u => u == target.TargetObjectId) || _clientState.LocalPlayer.GameObjectId == target.TargetObjectId) return; @@ -113,10 +113,15 @@ internal class ContextMenuService : IHostedService if (!_configService.Current.EnableRightClickMenus) return; + + //Check if lightfinder is on. + if (!_configService.Current.BroadcastEnabled) + return; + args.AddMenuItem(new MenuItem { - Name = "Send Pair Request", + Name = "Send Direct Pair Request", PrefixChar = 'L', UseDefaultPrefix = false, PrefixColor = 708, @@ -159,7 +164,7 @@ internal class ContextMenuService : IHostedService } } - private HashSet VisibleUserIds => [.. _pairManager.GetOnlineUserPairs() + private HashSet VisibleUserIds => [.. _pairManager.DirectPairs .Where(u => u.IsVisible && u.PlayerCharacterId != uint.MaxValue) .Select(u => (ulong)u.PlayerCharacterId)]; From ce5f8a43a2048d4a83fbf4619eed8801f022a613 Mon Sep 17 00:00:00 2001 From: cake Date: Mon, 27 Oct 2025 16:16:40 +0100 Subject: [PATCH 04/28] Added list of users names in the dtr entry whenever lightfinder is active --- .../Services/BroadcastScanningService.cs | 12 +++- LightlessSync/Services/PairRequestService.cs | 10 +-- LightlessSync/UI/DtrEntry.cs | 68 +++++++++++++------ 3 files changed, 60 insertions(+), 30 deletions(-) diff --git a/LightlessSync/Services/BroadcastScanningService.cs b/LightlessSync/Services/BroadcastScanningService.cs index 79fe984..95abdae 100644 --- a/LightlessSync/Services/BroadcastScanningService.cs +++ b/LightlessSync/Services/BroadcastScanningService.cs @@ -28,7 +28,7 @@ public class BroadcastScannerService : DisposableMediatorSubscriberBase, IDispos private readonly CancellationTokenSource _cleanupCts = new(); private Task? _cleanupTask; - private int _checkEveryFrames = 20; + private readonly int _checkEveryFrames = 20; private int _frameCounter = 0; private int _lookupsThisFrame = 0; private const int MaxLookupsPerFrame = 30; @@ -221,6 +221,16 @@ public class BroadcastScannerService : DisposableMediatorSubscriberBase, IDispos (excludeHashedCid is null || !comparer.Equals(entry.Key, excludeHashedCid))); } + public List> GetActiveBroadcasts(string? excludeHashedCid = null) + { + var now = DateTime.UtcNow; + var comparer = StringComparer.Ordinal; + return [.. _broadcastCache.Where(entry => + entry.Value.IsBroadcasting && + entry.Value.ExpiryTime > now && + (excludeHashedCid is null || !comparer.Equals(entry.Key, excludeHashedCid)))]; + } + protected override void Dispose(bool disposing) { base.Dispose(disposing); diff --git a/LightlessSync/Services/PairRequestService.cs b/LightlessSync/Services/PairRequestService.cs index 7190825..2531a3a 100644 --- a/LightlessSync/Services/PairRequestService.cs +++ b/LightlessSync/Services/PairRequestService.cs @@ -1,7 +1,3 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; using LightlessSync.LightlessConfiguration.Models; using LightlessSync.PlayerData.Pairs; using LightlessSync.Services.Mediator; @@ -14,10 +10,10 @@ public sealed class PairRequestService : DisposableMediatorSubscriberBase private readonly DalamudUtilService _dalamudUtil; private readonly PairManager _pairManager; private readonly Lazy _apiController; - private readonly object _syncRoot = new(); + private readonly Lock _syncRoot = new(); private readonly List _requests = []; - private static readonly TimeSpan Expiration = TimeSpan.FromMinutes(5); + private static readonly TimeSpan _expiration = TimeSpan.FromMinutes(5); public PairRequestService( ILogger logger, @@ -189,7 +185,7 @@ public sealed class PairRequestService : DisposableMediatorSubscriberBase } var now = DateTime.UtcNow; - return _requests.RemoveAll(r => now - r.ReceivedAt > Expiration) > 0; + return _requests.RemoveAll(r => now - r.ReceivedAt > _expiration) > 0; } public void AcceptPairRequest(string hashedCid, string displayName) diff --git a/LightlessSync/UI/DtrEntry.cs b/LightlessSync/UI/DtrEntry.cs index 89c7389..17bc871 100644 --- a/LightlessSync/UI/DtrEntry.cs +++ b/LightlessSync/UI/DtrEntry.cs @@ -2,19 +2,22 @@ using Dalamud.Game.Gui.Dtr; using Dalamud.Game.Text.SeStringHandling; using Dalamud.Game.Text.SeStringHandling.Payloads; using Dalamud.Plugin.Services; +using Dalamud.Utility; using LightlessSync.LightlessConfiguration; using LightlessSync.LightlessConfiguration.Configurations; using LightlessSync.PlayerData.Pairs; using LightlessSync.Services; using LightlessSync.Services.Mediator; using LightlessSync.Services.ServerConfiguration; +using LightlessSync.Utils; using LightlessSync.WebAPI; using LightlessSync.WebAPI.SignalR.Utils; -using LightlessSync.Utils; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Primitives; using System.Runtime.InteropServices; using System.Text; +using static LightlessSync.Services.PairRequestService; namespace LightlessSync.UI; @@ -106,7 +109,7 @@ public sealed class DtrEntry : IDisposable, IHostedService } catch (OperationCanceledException) { - + _logger.LogInformation("Lightfinder operation was canceled."); } finally { @@ -363,29 +366,46 @@ public sealed class DtrEntry : IDisposable, IHostedService } } - private int GetNearbyBroadcastCount() - { - var localHashedCid = GetLocalHashedCid(); - return _broadcastScannerService.CountActiveBroadcasts( - string.IsNullOrEmpty(localHashedCid) ? null : localHashedCid); - } - - private int GetPendingPairRequestCount() + private List GetNearbyBroadcasts() { try { - return _pairRequestService.GetActiveRequests().Count; + var localHashedCid = GetLocalHashedCid(); + return [.. _broadcastScannerService + .GetActiveBroadcasts(string.IsNullOrEmpty(localHashedCid) ? null : localHashedCid) + .Select(b => _dalamudUtilService.FindPlayerByNameHash(b.Key).Name)]; } catch (Exception ex) { var now = DateTime.UtcNow; + + if (now >= _pairRequestNextErrorLog) + { + _logger.LogDebug(ex, "Failed to retrieve nearby broadcasts for Lightfinder DTR entry."); + _pairRequestNextErrorLog = now + _localHashedCidErrorCooldown; + } + + return []; + } + } + + private IReadOnlyList GetPendingPairRequest() + { + try + { + return _pairRequestService.GetActiveRequests(); + } + catch (Exception ex) + { + var now = DateTime.UtcNow; + if (now >= _pairRequestNextErrorLog) { _logger.LogDebug(ex, "Failed to retrieve pair request count for Lightfinder DTR entry."); _pairRequestNextErrorLog = now + _localHashedCidErrorCooldown; } - return 0; + return []; } } @@ -400,23 +420,15 @@ public sealed class DtrEntry : IDisposable, IHostedService if (_broadcastService.IsBroadcasting) { - var tooltipBuilder = new StringBuilder("Lightfinder - Enabled"); - switch (config.LightfinderDtrDisplayMode) { case LightfinderDtrDisplayMode.PendingPairRequests: { - var requestCount = GetPendingPairRequestCount(); - tooltipBuilder.AppendLine(); - tooltipBuilder.Append("Pending pair requests: ").Append(requestCount); - return ($"{icon} Requests {requestCount}", SwapColorChannels(config.DtrColorsLightfinderEnabled), tooltipBuilder.ToString()); + return FormatTooltip("Pending pair requests", GetPendingPairRequest().Select(x => x.DisplayName), icon, SwapColorChannels(config.DtrColorsLightfinderEnabled)); } default: { - var broadcastCount = GetNearbyBroadcastCount(); - tooltipBuilder.AppendLine(); - tooltipBuilder.Append("Nearby Lightfinder users: ").Append(broadcastCount); - return ($"{icon} {broadcastCount}", SwapColorChannels(config.DtrColorsLightfinderEnabled), tooltipBuilder.ToString()); + return FormatTooltip("Nearby Lightfinder users", GetNearbyBroadcasts(), icon, SwapColorChannels(config.DtrColorsLightfinderEnabled)); } } } @@ -433,6 +445,18 @@ public sealed class DtrEntry : IDisposable, IHostedService return ($"{icon} OFF", colors, tooltip.ToString()); } + private (string, Colors, string) FormatTooltip(string title, IEnumerable names, string icon, Colors color) + { + var list = names.Where(x => !string.IsNullOrEmpty(x)).ToList(); + var tooltip = new StringBuilder() + .Append($"Lightfinder - Enabled{Environment.NewLine}") + .Append($"{title}: {list.Count}{Environment.NewLine}") + .AppendJoin(Environment.NewLine, list) + .ToString(); + + return ($"{icon} {list.Count}", color, tooltip); + } + private static string BuildLightfinderTooltip(string baseTooltip) { var builder = new StringBuilder(); From 8bccdc5ef1bdcfb9236d2ab2b1438d60eb6cd56e Mon Sep 17 00:00:00 2001 From: cake Date: Mon, 27 Oct 2025 22:26:03 +0100 Subject: [PATCH 05/28] Added lightless command. --- LightlessSync/Services/CommandManagerService.cs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/LightlessSync/Services/CommandManagerService.cs b/LightlessSync/Services/CommandManagerService.cs index 7aedc7b..88f8780 100644 --- a/LightlessSync/Services/CommandManagerService.cs +++ b/LightlessSync/Services/CommandManagerService.cs @@ -13,7 +13,8 @@ namespace LightlessSync.Services; public sealed class CommandManagerService : IDisposable { - private const string _commandName = "/light"; + private const string _longName = "/lightless"; + private const string _shortName = "/light"; private readonly ApiController _apiController; private readonly ICommandManager _commandManager; @@ -34,7 +35,11 @@ public sealed class CommandManagerService : IDisposable _apiController = apiController; _mediator = mediator; _lightlessConfigService = lightlessConfigService; - _commandManager.AddHandler(_commandName, new CommandInfo(OnCommand) + _commandManager.AddHandler(_longName, new CommandInfo(OnCommand) + { + HelpMessage = $"\u2191;" + }); + _commandManager.AddHandler(_shortName, new CommandInfo(OnCommand) { HelpMessage = "Opens the Lightless Sync UI" + Environment.NewLine + Environment.NewLine + "Additionally possible commands:" + Environment.NewLine + @@ -49,7 +54,8 @@ public sealed class CommandManagerService : IDisposable public void Dispose() { - _commandManager.RemoveHandler(_commandName); + _commandManager.RemoveHandler(_longName); + _commandManager.RemoveHandler(_shortName); } private void OnCommand(string command, string args) From cabc4ec0fe2df6082c8a020421960500d33794c9 Mon Sep 17 00:00:00 2001 From: choco Date: Tue, 28 Oct 2025 00:57:49 +0100 Subject: [PATCH 06/28] removed lightfinder on check --- LightlessSync/Services/ContextMenuService.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/LightlessSync/Services/ContextMenuService.cs b/LightlessSync/Services/ContextMenuService.cs index 42ab72a..464fee1 100644 --- a/LightlessSync/Services/ContextMenuService.cs +++ b/LightlessSync/Services/ContextMenuService.cs @@ -113,11 +113,6 @@ internal class ContextMenuService : IHostedService if (!_configService.Current.EnableRightClickMenus) return; - - //Check if lightfinder is on. - if (!_configService.Current.BroadcastEnabled) - return; - args.AddMenuItem(new MenuItem { From c16891021c456159b586d94eda4d52b46e8fd4ab Mon Sep 17 00:00:00 2001 From: cake Date: Tue, 28 Oct 2025 18:20:57 +0100 Subject: [PATCH 07/28] Added pause button for all syncshells and grouped syncshells. --- LightlessSync/UI/CompactUI.cs | 17 ++--- .../UI/Components/DrawGroupedGroupFolder.cs | 62 ++++++++++++++++--- LightlessSync/UI/Components/IDrawFolder.cs | 3 +- LightlessSync/UI/Models/GroupFolder.cs | 6 ++ 4 files changed, 71 insertions(+), 17 deletions(-) create mode 100644 LightlessSync/UI/Models/GroupFolder.cs diff --git a/LightlessSync/UI/CompactUI.cs b/LightlessSync/UI/CompactUI.cs index 0700de3..cc8d326 100644 --- a/LightlessSync/UI/CompactUI.cs +++ b/LightlessSync/UI/CompactUI.cs @@ -1,4 +1,3 @@ -using System; using Dalamud.Bindings.ImGui; using Dalamud.Interface; using Dalamud.Interface.Utility; @@ -16,12 +15,14 @@ using LightlessSync.Services.Mediator; using LightlessSync.Services.ServerConfiguration; using LightlessSync.UI.Components; using LightlessSync.UI.Handlers; +using LightlessSync.UI.Models; using LightlessSync.Utils; using LightlessSync.WebAPI; using LightlessSync.WebAPI.Files; using LightlessSync.WebAPI.Files.Models; using LightlessSync.WebAPI.SignalR.Utils; using Microsoft.Extensions.Logging; +using System; using System.Collections.Concurrent; using System.Collections.Immutable; using System.Globalization; @@ -708,23 +709,23 @@ public class CompactUi : WindowMediatorSubscriberBase } //Filter of not foldered syncshells - var groupFolders = new List(); + var groupFolders = new List(); foreach (var group in _pairManager.GroupPairs.Select(g => g.Key).OrderBy(g => g.GroupAliasOrGID, StringComparer.OrdinalIgnoreCase)) { GetGroups(allPairs, filteredPairs, group, out ImmutableList allGroupPairs, out Dictionary> filteredGroupPairs); if (FilterNotTaggedSyncshells(group)) { - groupFolders.Add(_drawEntityFactory.CreateDrawGroupFolder(group, filteredGroupPairs, allGroupPairs)); + groupFolders.Add(new GroupFolder(group, _drawEntityFactory.CreateDrawGroupFolder(group, filteredGroupPairs, allGroupPairs))); } } //Filter of grouped up syncshells (All Syncshells Folder) if (_configService.Current.GroupUpSyncshells) - drawFolders.Add(new DrawGroupedGroupFolder(groupFolders, _tagHandler, _uiSharedService, + drawFolders.Add(new DrawGroupedGroupFolder(groupFolders, _tagHandler, _apiController, _uiSharedService, _selectSyncshellForTagUi, _renameSyncshellTagUi, "")); else - drawFolders.AddRange(groupFolders); + drawFolders.AddRange(groupFolders.Select(v => v.GroupDrawFolder)); //Filter of grouped/foldered pairs foreach (var tag in _tagHandler.GetAllPairTagsSorted()) @@ -738,7 +739,7 @@ public class CompactUi : WindowMediatorSubscriberBase //Filter of grouped/foldered syncshells foreach (var syncshellTag in _tagHandler.GetAllSyncshellTagsSorted()) { - var syncshellFolderTags = new List(); + var syncshellFolderTags = new List(); foreach (var group in _pairManager.GroupPairs.Select(g => g.Key).OrderBy(g => g.GroupAliasOrGID, StringComparer.OrdinalIgnoreCase)) { if (_tagHandler.HasSyncshellTag(group.GID, syncshellTag)) @@ -747,11 +748,11 @@ public class CompactUi : WindowMediatorSubscriberBase out ImmutableList allGroupPairs, out Dictionary> filteredGroupPairs); - syncshellFolderTags.Add(_drawEntityFactory.CreateDrawGroupFolder($"tag_{group.GID}", group, filteredGroupPairs, allGroupPairs)); + syncshellFolderTags.Add(new GroupFolder(group, _drawEntityFactory.CreateDrawGroupFolder($"tag_{group.GID}", group, filteredGroupPairs, allGroupPairs))); } } - drawFolders.Add(new DrawGroupedGroupFolder(syncshellFolderTags, _tagHandler, _uiSharedService, _selectSyncshellForTagUi, _renameSyncshellTagUi, syncshellTag)); + drawFolders.Add(new DrawGroupedGroupFolder(syncshellFolderTags, _tagHandler, _apiController, _uiSharedService, _selectSyncshellForTagUi, _renameSyncshellTagUi, syncshellTag)); } //Filter of not grouped/foldered and offline pairs diff --git a/LightlessSync/UI/Components/DrawGroupedGroupFolder.cs b/LightlessSync/UI/Components/DrawGroupedGroupFolder.cs index 2aa3d5c..59da7ef 100644 --- a/LightlessSync/UI/Components/DrawGroupedGroupFolder.cs +++ b/LightlessSync/UI/Components/DrawGroupedGroupFolder.cs @@ -1,7 +1,11 @@ using Dalamud.Bindings.ImGui; using Dalamud.Interface; using Dalamud.Interface.Utility.Raii; +using LightlessSync.API.Data.Extensions; +using LightlessSync.API.Dto.Group; using LightlessSync.UI.Handlers; +using LightlessSync.UI.Models; +using LightlessSync.WebAPI; using System.Collections.Immutable; using System.Numerics; @@ -10,19 +14,20 @@ namespace LightlessSync.UI.Components; public class DrawGroupedGroupFolder : IDrawFolder { private readonly string _tag; - private readonly IEnumerable _groups; + private readonly IEnumerable _groups; private readonly TagHandler _tagHandler; private readonly UiSharedService _uiSharedService; + private readonly ApiController _apiController; private readonly SelectSyncshellForTagUi _selectSyncshellForTagUi; private readonly RenameSyncshellTagUi _renameSyncshellTagUi; private bool _wasHovered = false; private float _menuWidth; - public IImmutableList DrawPairs => throw new NotSupportedException(); - public int OnlinePairs => _groups.SelectMany(g => g.DrawPairs).Where(g => g.Pair.IsOnline).DistinctBy(g => g.Pair.UserData.UID).Count(); - public int TotalPairs => _groups.Sum(g => g.TotalPairs); + public IImmutableList DrawPairs => _groups.SelectMany(g => g.GroupDrawFolder.DrawPairs).ToImmutableList(); + public int OnlinePairs => _groups.SelectMany(g => g.GroupDrawFolder.DrawPairs).Where(g => g.Pair.IsOnline).DistinctBy(g => g.Pair.UserData.UID).Count(); + public int TotalPairs => _groups.Sum(g => g.GroupDrawFolder.TotalPairs); - public DrawGroupedGroupFolder(IEnumerable groups, TagHandler tagHandler, UiSharedService uiSharedService, SelectSyncshellForTagUi selectSyncshellForTagUi, RenameSyncshellTagUi renameSyncshellTagUi, string tag) + public DrawGroupedGroupFolder(IEnumerable groups, TagHandler tagHandler, ApiController apiController, UiSharedService uiSharedService, SelectSyncshellForTagUi selectSyncshellForTagUi, RenameSyncshellTagUi renameSyncshellTagUi, string tag) { _groups = groups; _tagHandler = tagHandler; @@ -30,6 +35,7 @@ public class DrawGroupedGroupFolder : IDrawFolder _selectSyncshellForTagUi = selectSyncshellForTagUi; _renameSyncshellTagUi = renameSyncshellTagUi; _tag = tag; + _apiController = apiController; } public void Draw() @@ -42,7 +48,7 @@ public class DrawGroupedGroupFolder : IDrawFolder using var id = ImRaii.PushId(_id); var color = ImRaii.PushColor(ImGuiCol.ChildBg, ImGui.GetColorU32(ImGuiCol.FrameBgHovered), _wasHovered); - using (ImRaii.Child("folder__" + _id, new System.Numerics.Vector2(UiSharedService.GetWindowContentRegionWidth() - ImGui.GetCursorPosX(), ImGui.GetFrameHeight()))) + using (ImRaii.Child("folder__" + _id, new Vector2(UiSharedService.GetWindowContentRegionWidth() - ImGui.GetCursorPosX(), ImGui.GetFrameHeight()))) { ImGui.Dummy(new Vector2(0f, ImGui.GetFrameHeight())); using (ImRaii.PushStyle(ImGuiStyleVar.ItemSpacing, new Vector2(0f, 0f))) @@ -83,11 +89,16 @@ public class DrawGroupedGroupFolder : IDrawFolder { ImGui.TextUnformatted(_tag); + ImGui.SameLine(); + DrawPauseButton(); ImGui.SameLine(); DrawMenu(); } else { ImGui.TextUnformatted("All Syncshells"); + + ImGui.SameLine(); + DrawPauseButton(); } } color.Dispose(); @@ -100,10 +111,47 @@ public class DrawGroupedGroupFolder : IDrawFolder using var indent = ImRaii.PushIndent(20f); foreach (var entry in _groups) { - entry.Draw(); + entry.GroupDrawFolder.Draw(); } } } + protected void DrawPauseButton() + { + if (DrawPairs.Count > 0) + { + var isPaused = _groups.Select(g => g.GroupFullInfo).All(g => g.GroupUserPermissions.IsPaused()); + FontAwesomeIcon pauseIcon = isPaused ? FontAwesomeIcon.Play : FontAwesomeIcon.Pause; + + var pauseButtonSize = _uiSharedService.GetIconButtonSize(pauseIcon); + var windowEndX = ImGui.GetWindowContentRegionMin().X + UiSharedService.GetWindowContentRegionWidth(); + if (_tag != "") + { + var spacingX = ImGui.GetStyle().ItemSpacing.X; + var menuButtonSize = _uiSharedService.GetIconButtonSize(FontAwesomeIcon.EllipsisV); + ImGui.SameLine(windowEndX - pauseButtonSize.X - menuButtonSize.X - spacingX); + } + else + { + ImGui.SameLine(windowEndX - pauseButtonSize.X); + } + + + if (_uiSharedService.IconButton(pauseIcon)) + { + ChangePauseStateGroups(); + } + } + } + + protected void ChangePauseStateGroups() + { + foreach(var group in _groups) + { + var perm = group.GroupFullInfo.GroupUserPermissions; + perm.SetPaused(!perm.IsPaused()); + _ = _apiController.GroupChangeIndividualPermissionState(new GroupPairUserPermissionDto(group.GroupFullInfo.Group, new(_apiController.UID), perm)); + } + } protected void DrawMenu() { diff --git a/LightlessSync/UI/Components/IDrawFolder.cs b/LightlessSync/UI/Components/IDrawFolder.cs index eda1fce..faf8a69 100644 --- a/LightlessSync/UI/Components/IDrawFolder.cs +++ b/LightlessSync/UI/Components/IDrawFolder.cs @@ -1,5 +1,4 @@ - -using System.Collections.Immutable; +using System.Collections.Immutable; namespace LightlessSync.UI.Components; diff --git a/LightlessSync/UI/Models/GroupFolder.cs b/LightlessSync/UI/Models/GroupFolder.cs new file mode 100644 index 0000000..cedeef0 --- /dev/null +++ b/LightlessSync/UI/Models/GroupFolder.cs @@ -0,0 +1,6 @@ +using LightlessSync.API.Dto.Group; +using LightlessSync.UI.Components; + +namespace LightlessSync.UI.Models; + +public record GroupFolder(GroupFullInfoDto GroupFullInfo, IDrawFolder GroupDrawFolder); From de75b90703b0bb73118cbd5c09c8aaa1202ab330 Mon Sep 17 00:00:00 2001 From: cake Date: Tue, 28 Oct 2025 18:23:01 +0100 Subject: [PATCH 08/28] Added spacing on function --- LightlessSync/UI/Components/DrawGroupedGroupFolder.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/LightlessSync/UI/Components/DrawGroupedGroupFolder.cs b/LightlessSync/UI/Components/DrawGroupedGroupFolder.cs index 59da7ef..1bb3d79 100644 --- a/LightlessSync/UI/Components/DrawGroupedGroupFolder.cs +++ b/LightlessSync/UI/Components/DrawGroupedGroupFolder.cs @@ -115,6 +115,7 @@ public class DrawGroupedGroupFolder : IDrawFolder } } } + protected void DrawPauseButton() { if (DrawPairs.Count > 0) From 177534d78b32269c8c96f65d5c01043cf5cdca1b Mon Sep 17 00:00:00 2001 From: cake Date: Wed, 29 Oct 2025 04:37:24 +0100 Subject: [PATCH 09/28] Implemented compactor to work on BTRFS, redid cache a bit for better function on linux. Removed error for websockets, it will be forced on wine again. --- LightlessSync/FileCache/CacheMonitor.cs | 17 +- LightlessSync/FileCache/FileCacheManager.cs | 80 ++-- LightlessSync/FileCache/FileCompactor.cs | 374 +++++++++++++++--- .../Models/ServerStorage.cs | 1 - LightlessSync/UI/SettingsUi.cs | 36 +- LightlessSync/Utils/Crypto.cs | 20 + LightlessSync/Utils/FileSystemHelper.cs | 143 +++++++ LightlessSync/WebAPI/SignalR/HubFactory.cs | 7 - 8 files changed, 572 insertions(+), 106 deletions(-) create mode 100644 LightlessSync/Utils/FileSystemHelper.cs diff --git a/LightlessSync/FileCache/CacheMonitor.cs b/LightlessSync/FileCache/CacheMonitor.cs index 3b41d85..23f5c19 100644 --- a/LightlessSync/FileCache/CacheMonitor.cs +++ b/LightlessSync/FileCache/CacheMonitor.cs @@ -115,6 +115,8 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase public bool StorageisNTFS { get; private set; } = false; + public bool StorageIsBtrfs { get ; private set; } = false; + public void StartLightlessWatcher(string? lightlessPath) { LightlessWatcher?.Dispose(); @@ -124,10 +126,19 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase Logger.LogWarning("Lightless file path is not set, cannot start the FSW for Lightless."); return; } + var fsType = FileSystemHelper.GetFilesystemType(_configService.Current.CacheFolder); - DriveInfo di = new(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName); - StorageisNTFS = string.Equals("NTFS", di.DriveFormat, StringComparison.OrdinalIgnoreCase); - Logger.LogInformation("Lightless Storage is on NTFS drive: {isNtfs}", StorageisNTFS); + if (fsType == FileSystemHelper.FilesystemType.NTFS) + { + StorageisNTFS = true; + Logger.LogInformation("Lightless Storage is on NTFS drive: {isNtfs}", StorageisNTFS); + } + + if (fsType == FileSystemHelper.FilesystemType.Btrfs) + { + StorageIsBtrfs = true; + Logger.LogInformation("Lightless Storage is on BTRFS drive: {isNtfs}", StorageIsBtrfs); + } Logger.LogDebug("Initializing Lightless FSW on {path}", lightlessPath); LightlessWatcher = new() diff --git a/LightlessSync/FileCache/FileCacheManager.cs b/LightlessSync/FileCache/FileCacheManager.cs index 972c4d9..7ee6c99 100644 --- a/LightlessSync/FileCache/FileCacheManager.cs +++ b/LightlessSync/FileCache/FileCacheManager.cs @@ -203,42 +203,72 @@ public sealed class FileCacheManager : IHostedService return output; } - public Task> ValidateLocalIntegrity(IProgress<(int, int, FileCacheEntity)> progress, CancellationToken cancellationToken) + public async Task> ValidateLocalIntegrity(IProgress<(int completed, int total, FileCacheEntity current)> progress, CancellationToken cancellationToken) { _lightlessMediator.Publish(new HaltScanMessage(nameof(ValidateLocalIntegrity))); _logger.LogInformation("Validating local storage"); - var cacheEntries = _fileCaches.Values.SelectMany(v => v.Values.Where(e => e != null)).Where(v => v.IsCacheEntry).ToList(); - List brokenEntities = []; - int i = 0; - foreach (var fileCache in cacheEntries) + + var cacheEntries = _fileCaches.Values + .SelectMany(v => v.Values) + .Where(v => v.IsCacheEntry) + .ToList(); + + int total = cacheEntries.Count; + int processed = 0; + var brokenEntities = new ConcurrentBag(); + + _logger.LogInformation("Checking {count} cache entries...", total); + + await Parallel.ForEachAsync(cacheEntries, new ParallelOptions + { + MaxDegreeOfParallelism = Environment.ProcessorCount, + CancellationToken = cancellationToken + }, + async (fileCache, token) => { - if (cancellationToken.IsCancellationRequested) break; - - _logger.LogInformation("Validating {file}", fileCache.ResolvedFilepath); - - progress.Report((i, cacheEntries.Count, fileCache)); - i++; - if (!File.Exists(fileCache.ResolvedFilepath)) - { - brokenEntities.Add(fileCache); - continue; - } - try { - var computedHash = Crypto.GetFileHash(fileCache.ResolvedFilepath); + int current = Interlocked.Increment(ref processed); + if (current % 10 == 0) + progress.Report((current, total, fileCache)); + + if (!File.Exists(fileCache.ResolvedFilepath)) + { + brokenEntities.Add(fileCache); + return; + } + + string computedHash; + try + { + computedHash = await Crypto.GetFileHashAsync(fileCache.ResolvedFilepath, token).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error hashing {file}", fileCache.ResolvedFilepath); + brokenEntities.Add(fileCache); + return; + } + if (!string.Equals(computedHash, fileCache.Hash, StringComparison.Ordinal)) { - _logger.LogInformation("Failed to validate {file}, got hash {computedHash}, expected hash {hash}", fileCache.ResolvedFilepath, computedHash, fileCache.Hash); + _logger.LogInformation( + "Hash mismatch: {file} (got {computedHash}, expected {expected})", + fileCache.ResolvedFilepath, computedHash, fileCache.Hash); + brokenEntities.Add(fileCache); } } - catch (Exception e) + catch (OperationCanceledException) { - _logger.LogWarning(e, "Error during validation of {file}", fileCache.ResolvedFilepath); + _logger.LogError("Validation got cancelled for {file}", fileCache.ResolvedFilepath); + } + catch (Exception ex) + { + _logger.LogError(ex, "Unexpected error validating {file}", fileCache.ResolvedFilepath); brokenEntities.Add(fileCache); } - } + }).ConfigureAwait(false); foreach (var brokenEntity in brokenEntities) { @@ -250,12 +280,14 @@ public sealed class FileCacheManager : IHostedService } catch (Exception ex) { - _logger.LogWarning(ex, "Could not delete {file}", brokenEntity.ResolvedFilepath); + _logger.LogWarning(ex, "Failed to delete invalid cache file {file}", brokenEntity.ResolvedFilepath); } } _lightlessMediator.Publish(new ResumeScanMessage(nameof(ValidateLocalIntegrity))); - return Task.FromResult(brokenEntities); + _logger.LogInformation("Validation complete. Found {count} invalid entries.", brokenEntities.Count); + + return [.. brokenEntities]; } public string GetCacheFilePath(string hash, string extension) diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index 1a35ad6..e5d219e 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -1,11 +1,15 @@ using LightlessSync.LightlessConfiguration; using LightlessSync.Services; +using LightlessSync.Utils; using Microsoft.Extensions.Logging; -using System.Runtime.InteropServices; using System.Collections.Concurrent; +using System.Diagnostics; +using System.IO; +using System.Runtime.InteropServices; using System.Threading; using System.Threading.Channels; using System.Threading.Tasks; +using static LightlessSync.Utils.FileSystemHelper; namespace LightlessSync.FileCache; @@ -87,25 +91,51 @@ public sealed class FileCompactor : IDisposable public long GetFileSizeOnDisk(FileInfo fileInfo, bool? isNTFS = null) { - bool ntfs = isNTFS ?? string.Equals(new DriveInfo(fileInfo.Directory!.Root.FullName).DriveFormat, "NTFS", StringComparison.OrdinalIgnoreCase); + var fsType = FileSystemHelper.GetFilesystemType(fileInfo.FullName); - if (_dalamudUtilService.IsWine || !ntfs) return fileInfo.Length; + bool ntfs = isNTFS ?? fsType == FileSystemHelper.FilesystemType.NTFS; - var clusterSize = GetClusterSize(fileInfo); - if (clusterSize == -1) return fileInfo.Length; - var losize = GetCompressedFileSizeW(fileInfo.FullName, out uint hosize); - var size = (long)hosize << 32 | losize; - return ((size + clusterSize - 1) / clusterSize) * clusterSize; + if (fsType != FileSystemHelper.FilesystemType.Btrfs && !ntfs) + { + return fileInfo.Length; + } + + if (ntfs && !_dalamudUtilService.IsWine) + { + var clusterSize = GetClusterSize(fileInfo); + if (clusterSize == -1) return fileInfo.Length; + var losize = GetCompressedFileSizeW(fileInfo.FullName, out uint hosize); + var size = (long)hosize << 32 | losize; + return ((size + clusterSize - 1) / clusterSize) * clusterSize; + } + + if (fsType == FileSystemHelper.FilesystemType.Btrfs) + { + try + { + long blocks = RunStatGetBlocks(fileInfo.FullName); + return blocks * 512L; + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Failed to get on-disk size via stat for {file}, falling back to Length", fileInfo.FullName); + return fileInfo.Length; + } + } + + return fileInfo.Length; } public async Task WriteAllBytesAsync(string filePath, byte[] decompressedFile, CancellationToken token) { + var dir = Path.GetDirectoryName(filePath); + if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir)) + Directory.CreateDirectory(dir); + await File.WriteAllBytesAsync(filePath, decompressedFile, token).ConfigureAwait(false); - if (_dalamudUtilService.IsWine || !_lightlessConfigService.Current.UseCompactor) - { + if (!_lightlessConfigService.Current.UseCompactor) return; - } EnqueueCompaction(filePath); } @@ -153,56 +183,178 @@ public sealed class FileCompactor : IDisposable private void CompactFile(string filePath) { - var fs = new DriveInfo(new FileInfo(filePath).Directory!.Root.FullName); - bool isNTFS = string.Equals(fs.DriveFormat, "NTFS", StringComparison.OrdinalIgnoreCase); - if (!isNTFS) + var fi = new FileInfo(filePath); + if (!fi.Exists) { - _logger.LogWarning("Drive for file {file} is not NTFS", filePath); + _logger.LogDebug("Skipping compaction for missing file {file}", filePath); return; } - var fi = new FileInfo(filePath); + var fsType = FileSystemHelper.GetFilesystemType(filePath); var oldSize = fi.Length; - var clusterSize = GetClusterSize(fi); + int clusterSize = GetClusterSize(fi); if (oldSize < Math.Max(clusterSize, 8 * 1024)) { _logger.LogDebug("File {file} is smaller than cluster size ({size}), ignoring", filePath, clusterSize); return; } - if (!IsCompactedFile(filePath)) + // NTFS Compression. + if (fsType == FileSystemHelper.FilesystemType.NTFS && !_dalamudUtilService.IsWine) { - _logger.LogDebug("Compacting file to XPRESS8K: {file}", filePath); + if (!IsWOFCompactedFile(filePath)) + { + _logger.LogDebug("Compacting file to XPRESS8K: {file}", filePath); + var success = WOFCompressFile(filePath); - WOFCompressFile(filePath); + if (success) + { + var newSize = GetFileSizeOnDisk(fi); + _logger.LogDebug("Compressed {file} from {orig}b to {comp}b", filePath, oldSize, newSize); + } + else + { + _logger.LogWarning("NTFS compression failed or not available for {file}", filePath); + } - var newSize = GetFileSizeOnDisk(fi); - - _logger.LogDebug("Compressed {file} from {orig}b to {comp}b", filePath, oldSize, newSize); + } + else + { + _logger.LogDebug("File {file} already compressed (NTFS)", filePath); + } } - else + + // BTRFS Compression + if (fsType == FileSystemHelper.FilesystemType.Btrfs) { - _logger.LogDebug("File {file} already compressed", filePath); + if (!IsBtrfsCompressedFile(filePath)) + { + _logger.LogDebug("Attempting btrfs compression for {file}", filePath); + var success = BtrfsCompressFile(filePath); + + if (success) + { + var newSize = GetFileSizeOnDisk(fi); + _logger.LogDebug("Btrfs-compressed {file} from {orig}b to {comp}b", filePath, oldSize, newSize); + } + else + { + _logger.LogWarning("Btrfs compression failed or not available for {file}", filePath); + } + } + else + { + _logger.LogDebug("File {file} already compressed (Btrfs)", filePath); + } } } + private static long RunStatGetBlocks(string path) + { + var psi = new ProcessStartInfo("stat", $"-c %b \"{path}\"") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + using var proc = Process.Start(psi) ?? throw new InvalidOperationException("Could not start stat process"); + var outp = proc.StandardOutput.ReadToEnd(); + var err = proc.StandardError.ReadToEnd(); + proc.WaitForExit(); + if (proc.ExitCode != 0) + { + throw new InvalidOperationException($"stat failed: {err}"); + } + + if (!long.TryParse(outp.Trim(), out var blocks)) + { + throw new InvalidOperationException($"invalid stat output: {outp}"); + } + + return blocks; + } + private void DecompressFile(string path) { _logger.LogDebug("Removing compression from {file}", path); - try + var fsType = FileSystemHelper.GetFilesystemType(path); + if (fsType == null) return; + + //NTFS Decompression + if (fsType == FileSystemHelper.FilesystemType.NTFS && !_dalamudUtilService.IsWine) { - using (var fs = new FileStream(path, FileMode.Open)) + try { + using (var fs = new FileStream(path, FileMode.Open)) + { #pragma warning disable S3869 // "SafeHandle.DangerousGetHandle" should not be called - var hDevice = fs.SafeFileHandle.DangerousGetHandle(); + var hDevice = fs.SafeFileHandle.DangerousGetHandle(); #pragma warning restore S3869 // "SafeHandle.DangerousGetHandle" should not be called - _ = DeviceIoControl(hDevice, FSCTL_DELETE_EXTERNAL_BACKING, nint.Zero, 0, nint.Zero, 0, out _, out _); + _ = DeviceIoControl(hDevice, FSCTL_DELETE_EXTERNAL_BACKING, nint.Zero, 0, nint.Zero, 0, out _, out _); + } } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error decompressing file {path}", path); + } + return; } - catch (Exception ex) + + //BTRFS Decompression + if (fsType == FileSystemHelper.FilesystemType.Btrfs) { - _logger.LogWarning(ex, "Error decompressing file {path}", path); + try + { + var mountOptions = GetMountOptionsForPath(path); + if (mountOptions.Contains("compress", StringComparison.OrdinalIgnoreCase)) + { + _logger.LogWarning( + "Cannot safely decompress {file}: filesystem mounted with compression ({opts}). " + + "Remount with 'compress=no' before running decompression.", path, mountOptions); + return; + } + + _logger.LogDebug("Rewriting {file} to remove btrfs compression...", path); + + var psi = new ProcessStartInfo("btrfs", $"filesystem defragment -- \"{path}\"") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + using var proc = Process.Start(psi); + if (proc == null) + { + _logger.LogWarning("Failed to start btrfs defragment for decompression of {file}", path); + return; + } + + var stdout = proc.StandardOutput.ReadToEnd(); + var stderr = proc.StandardError.ReadToEnd(); + proc.WaitForExit(); + + if (proc.ExitCode != 0) + { + _logger.LogWarning("btrfs defragment failed for {file}: {err}", path, stderr); + } + else + { + // Log output only in debug mode to avoid clutter + if (!string.IsNullOrWhiteSpace(stdout)) + _logger.LogDebug("btrfs defragment output for {file}: {out}", path, stdout.Trim()); + + _logger.LogInformation("Decompressed (rewritten uncompressed) btrfs file: {file}", path); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error rewriting {file} for decompression", path); + } } } @@ -220,7 +372,7 @@ public sealed class FileCompactor : IDisposable return _clusterSizes[root]; } - private static bool IsCompactedFile(string filePath) + private static bool IsWOFCompactedFile(string filePath) { uint buf = 8; _ = WofIsExternalFile(filePath, out int isExtFile, out uint _, out var info, ref buf); @@ -228,40 +380,151 @@ public sealed class FileCompactor : IDisposable return info.Algorithm == CompressionAlgorithm.XPRESS8K; } - private void WOFCompressFile(string path) + private bool IsBtrfsCompressedFile(string path) + { + try + { + var psi = new ProcessStartInfo("filefrag", $"-v \"{path}\"") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + using var proc = Process.Start(psi); + if (proc == null) + { + _logger.LogWarning("Failed to start filefrag for {file}", path); + return false; + } + + string output = proc.StandardOutput.ReadToEnd(); + proc.WaitForExit(); + + // look for "flags: compressed" in the output + if (output.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + return false; + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Failed to detect btrfs compression for {file}", path); + return false; + } + } + + private bool WOFCompressFile(string path) { var efInfoPtr = Marshal.AllocHGlobal(Marshal.SizeOf(_efInfo)); Marshal.StructureToPtr(_efInfo, efInfoPtr, fDeleteOld: true); ulong length = (ulong)Marshal.SizeOf(_efInfo); try { - using (var fs = new FileStream(path, FileMode.Open)) + using var fs = new FileStream(path, FileMode.Open); + #pragma warning disable S3869 // "SafeHandle.DangerousGetHandle" should not be called + var hFile = fs.SafeFileHandle.DangerousGetHandle(); + #pragma warning restore S3869 // "SafeHandle.DangerousGetHandle" should not be called + if (fs.SafeFileHandle.IsInvalid) { -#pragma warning disable S3869 // "SafeHandle.DangerousGetHandle" should not be called - var hFile = fs.SafeFileHandle.DangerousGetHandle(); -#pragma warning restore S3869 // "SafeHandle.DangerousGetHandle" should not be called - if (fs.SafeFileHandle.IsInvalid) - { - _logger.LogWarning("Invalid file handle to {file}", path); - } - else - { - var ret = WofSetFileDataLocation(hFile, WOF_PROVIDER_FILE, efInfoPtr, length); - if (!(ret == 0 || ret == unchecked((int)0x80070158))) - { - _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X")); - } - } + _logger.LogWarning("Invalid file handle to {file}", path); + return false; + } + + var ret = WofSetFileDataLocation(hFile, WOF_PROVIDER_FILE, efInfoPtr, length); + if (!(ret == 0 || ret == unchecked((int)0x80070158))) + { + _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X")); + return false; } } catch (Exception ex) { _logger.LogWarning(ex, "Error compacting file {path}", path); + return false; } finally { Marshal.FreeHGlobal(efInfoPtr); } + return true; + } + + private bool BtrfsCompressFile(string path) + { + try + { + var psi = new ProcessStartInfo("btrfs", $"filesystem defragment -czstd -- \"{path}\"") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + using var proc = Process.Start(psi); + if (proc == null) + { + _logger.LogWarning("Failed to start btrfs process for {file}", path); + return false; + } + + var stdout = proc.StandardOutput.ReadToEnd(); + var stderr = proc.StandardError.ReadToEnd(); + proc.WaitForExit(); + + if (proc.ExitCode != 0) + { + _logger.LogWarning("btrfs defrag returned {code} for {file}: {err}", proc.ExitCode, path, stderr); + return false; + } + + _logger.LogDebug("btrfs output: {out}", stdout); + return true; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error running btrfs defragment for {file}", path); + return false; + } + } + + private string GetMountOptionsForPath(string path) + { + try + { + var fullPath = Path.GetFullPath(path); + var mounts = File.ReadAllLines("/proc/mounts"); + string bestMount = string.Empty; + string mountOptions = string.Empty; + + foreach (var line in mounts) + { + var parts = line.Split(' '); + if (parts.Length < 4) continue; + var mountPoint = parts[1].Replace("\\040", " ", StringComparison.Ordinal); // unescape spaces + string normalized; + try { normalized = Path.GetFullPath(mountPoint); } + catch { normalized = mountPoint; } + + if (fullPath.StartsWith(normalized, StringComparison.Ordinal) && + normalized.Length > bestMount.Length) + { + bestMount = normalized; + mountOptions = parts[3]; + } + } + + return mountOptions; + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Failed to get mount options for {path}", path); + return string.Empty; + } } private struct WOF_FILE_COMPRESSION_INFO_V1 @@ -273,7 +536,14 @@ public sealed class FileCompactor : IDisposable private void EnqueueCompaction(string filePath) { if (!_pendingCompactions.TryAdd(filePath, 0)) + return; + + var fsType = GetFilesystemType(filePath); + + if (fsType != FilesystemType.NTFS && fsType != FilesystemType.Btrfs) { + _logger.LogTrace("Skipping compaction enqueue for unsupported filesystem {fs} ({file})", fsType, filePath); + _pendingCompactions.TryRemove(filePath, out _); return; } @@ -282,6 +552,10 @@ public sealed class FileCompactor : IDisposable _pendingCompactions.TryRemove(filePath, out _); _logger.LogDebug("Failed to enqueue compaction job for {file}", filePath); } + else + { + _logger.LogTrace("Queued compaction job for {file} (fs={fs})", filePath, fsType); + } } private async Task ProcessQueueAsync(CancellationToken token) @@ -299,7 +573,7 @@ public sealed class FileCompactor : IDisposable return; } - if (_dalamudUtilService.IsWine || !_lightlessConfigService.Current.UseCompactor) + if (!_lightlessConfigService.Current.UseCompactor) { continue; } diff --git a/LightlessSync/LightlessConfiguration/Models/ServerStorage.cs b/LightlessSync/LightlessConfiguration/Models/ServerStorage.cs index 2b003bb..20302fe 100644 --- a/LightlessSync/LightlessConfiguration/Models/ServerStorage.cs +++ b/LightlessSync/LightlessConfiguration/Models/ServerStorage.cs @@ -13,5 +13,4 @@ public class ServerStorage public bool UseOAuth2 { get; set; } = false; public string? OAuthToken { get; set; } = null; public HttpTransportType HttpTransportType { get; set; } = HttpTransportType.WebSockets; - public bool ForceWebSockets { get; set; } = false; } \ No newline at end of file diff --git a/LightlessSync/UI/SettingsUi.cs b/LightlessSync/UI/SettingsUi.cs index febc142..d686a75 100644 --- a/LightlessSync/UI/SettingsUi.cs +++ b/LightlessSync/UI/SettingsUi.cs @@ -1227,16 +1227,16 @@ public class SettingsUi : WindowMediatorSubscriberBase ImGui.TextUnformatted($"Currently utilized local storage: Calculating..."); ImGui.TextUnformatted( $"Remaining space free on drive: {UiSharedService.ByteToString(_cacheMonitor.FileCacheDriveFree)}"); + bool useFileCompactor = _configService.Current.UseCompactor; - bool isLinux = _dalamudUtilService.IsWine; - if (!useFileCompactor && !isLinux) + if (!useFileCompactor) { UiSharedService.ColorTextWrapped( "Hint: To free up space when using Lightless consider enabling the File Compactor", UIColors.Get("LightlessYellow")); } - if (isLinux || !_cacheMonitor.StorageisNTFS) ImGui.BeginDisabled(); + if (!_cacheMonitor.StorageIsBtrfs && !_cacheMonitor.StorageisNTFS) ImGui.BeginDisabled(); if (ImGui.Checkbox("Use file compactor", ref useFileCompactor)) { _configService.Current.UseCompactor = useFileCompactor; @@ -1281,10 +1281,20 @@ public class SettingsUi : WindowMediatorSubscriberBase UIColors.Get("LightlessYellow")); } - if (isLinux || !_cacheMonitor.StorageisNTFS) + if (!_cacheMonitor.StorageIsBtrfs && !_cacheMonitor.StorageisNTFS) { ImGui.EndDisabled(); - ImGui.TextUnformatted("The file compactor is only available on Windows and NTFS drives."); + ImGui.TextUnformatted("The file compactor is only available on BTRFS and NTFS drives."); + } + + if (_cacheMonitor.StorageisNTFS) + { + ImGui.TextUnformatted("The file compactor is running on NTFS Drive."); + } + + if (_cacheMonitor.StorageIsBtrfs) + { + ImGui.TextUnformatted("The file compactor is running on Btrfs Drive."); } ImGuiHelpers.ScaledDummy(new Vector2(10, 10)); @@ -3113,22 +3123,6 @@ public class SettingsUi : WindowMediatorSubscriberBase UiSharedService.TooltipSeparator + "Note: if the server does not support a specific Transport Type it will fall through to the next automatically: WebSockets > ServerSentEvents > LongPolling"); - if (_dalamudUtilService.IsWine) - { - bool forceWebSockets = selectedServer.ForceWebSockets; - if (ImGui.Checkbox("[wine only] Force WebSockets", ref forceWebSockets)) - { - selectedServer.ForceWebSockets = forceWebSockets; - _serverConfigurationManager.Save(); - } - - _uiShared.DrawHelpText( - "On wine, Lightless will automatically fall back to ServerSentEvents/LongPolling, even if WebSockets is selected. " - + "WebSockets are known to crash XIV entirely on wine 8.5 shipped with Dalamud. " - + "Only enable this if you are not running wine 8.5." + Environment.NewLine - + "Note: If the issue gets resolved at some point this option will be removed."); - } - ImGuiHelpers.ScaledDummy(5); if (ImGui.Checkbox("Use Discord OAuth2 Authentication", ref useOauth)) diff --git a/LightlessSync/Utils/Crypto.cs b/LightlessSync/Utils/Crypto.cs index de04d26..87d6883 100644 --- a/LightlessSync/Utils/Crypto.cs +++ b/LightlessSync/Utils/Crypto.cs @@ -21,6 +21,26 @@ public static class Crypto return BitConverter.ToString(sha1.ComputeHash(stream)).Replace("-", "", StringComparison.Ordinal); } + public static async Task GetFileHashAsync(string filePath, CancellationToken cancellationToken = default) + { + var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, bufferSize: 65536, options: FileOptions.Asynchronous); + await using (stream.ConfigureAwait(false)) + { + using var sha1 = SHA1.Create(); + + var buffer = new byte[8192]; + int bytesRead; + while ((bytesRead = await stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) > 0) + { + sha1.TransformBlock(buffer, 0, bytesRead, outputBuffer: null, 0); + } + + sha1.TransformFinalBlock([], 0, 0); + + return BitConverter.ToString(sha1.Hash!).Replace("-", "", StringComparison.Ordinal); + } + } + public static string GetHash256(this (string, ushort) playerToHash) { if (_hashListPlayersSHA256.TryGetValue(playerToHash, out var hash)) diff --git a/LightlessSync/Utils/FileSystemHelper.cs b/LightlessSync/Utils/FileSystemHelper.cs new file mode 100644 index 0000000..a80e922 --- /dev/null +++ b/LightlessSync/Utils/FileSystemHelper.cs @@ -0,0 +1,143 @@ +using System.Collections.Concurrent; +using System.Runtime.InteropServices; + +namespace LightlessSync.Utils +{ + public static class FileSystemHelper + { + public enum FilesystemType + { + Unknown = 0, + NTFS, + Btrfs, + Ext4, + Xfs, + Apfs, + HfsPlus, + Fat, + Exfat, + Zfs + } + + private static readonly ConcurrentDictionary _filesystemTypeCache = new(StringComparer.OrdinalIgnoreCase); + + public static FilesystemType GetFilesystemType(string filePath) + { + try + { + string rootPath; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + var info = new FileInfo(filePath); + var dir = info.Directory ?? new DirectoryInfo(filePath); + rootPath = dir.Root.FullName; + } + else + { + rootPath = GetMountPoint(filePath); + if (string.IsNullOrEmpty(rootPath)) + rootPath = "/"; + } + + if (_filesystemTypeCache.TryGetValue(rootPath, out var cachedType)) + return cachedType; + + FilesystemType detected; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + var root = new DriveInfo(rootPath); + var format = root.DriveFormat?.ToUpperInvariant() ?? string.Empty; + detected = format switch + { + "NTFS" => FilesystemType.NTFS, + "FAT32" => FilesystemType.Fat, + "EXFAT" => FilesystemType.Exfat, + _ => FilesystemType.Unknown + }; + } + else + { + detected = GetLinuxFilesystemType(filePath); + } + + _filesystemTypeCache[rootPath] = detected; + return detected; + } + catch (Exception ex) + { + return FilesystemType.Unknown; + } + } + + private static string GetMountPoint(string filePath) + { + try + { + var path = Path.GetFullPath(filePath); + if (!File.Exists("/proc/mounts")) return "/"; + var mounts = File.ReadAllLines("/proc/mounts"); + + string bestMount = "/"; + foreach (var line in mounts) + { + var parts = line.Split(' '); + if (parts.Length < 3) continue; + var mountPoint = parts[1].Replace("\\040", " "); // unescape spaces + + string normalizedMount; + try { normalizedMount = Path.GetFullPath(mountPoint); } + catch { normalizedMount = mountPoint; } + + if (path.StartsWith(normalizedMount, StringComparison.Ordinal) && + normalizedMount.Length > bestMount.Length) + { + bestMount = normalizedMount; + } + } + + return bestMount; + } + catch + { + return "/"; + } + } + + private static FilesystemType GetLinuxFilesystemType(string filePath) + { + try + { + var mountPoint = GetMountPoint(filePath); + var mounts = File.ReadAllLines("/proc/mounts"); + + foreach (var line in mounts) + { + var parts = line.Split(' '); + if (parts.Length < 3) continue; + var mount = parts[1].Replace("\\040", " "); + if (string.Equals(mount, mountPoint, StringComparison.Ordinal)) + { + var fstype = parts[2].ToLowerInvariant(); + return fstype switch + { + "btrfs" => FilesystemType.Btrfs, + "ext4" => FilesystemType.Ext4, + "xfs" => FilesystemType.Xfs, + "zfs" => FilesystemType.Zfs, + "apfs" => FilesystemType.Apfs, + "hfsplus" => FilesystemType.HfsPlus, + _ => FilesystemType.Unknown + }; + } + } + + return FilesystemType.Unknown; + } + catch + { + return FilesystemType.Unknown; + } + } + } +} diff --git a/LightlessSync/WebAPI/SignalR/HubFactory.cs b/LightlessSync/WebAPI/SignalR/HubFactory.cs index ef9f919..1d5a0c8 100644 --- a/LightlessSync/WebAPI/SignalR/HubFactory.cs +++ b/LightlessSync/WebAPI/SignalR/HubFactory.cs @@ -70,13 +70,6 @@ public class HubFactory : MediatorSubscriberBase _ => HttpTransportType.WebSockets | HttpTransportType.ServerSentEvents | HttpTransportType.LongPolling }; - if (_isWine && !_serverConfigurationManager.CurrentServer.ForceWebSockets - && transportType.HasFlag(HttpTransportType.WebSockets)) - { - Logger.LogDebug("Wine detected, falling back to ServerSentEvents / LongPolling"); - transportType = HttpTransportType.ServerSentEvents | HttpTransportType.LongPolling; - } - Logger.LogDebug("Building new HubConnection using transport {transport}", transportType); _instance = new HubConnectionBuilder() From 9a846a37d40c1d3115dc7bf21438fa39a489bdac Mon Sep 17 00:00:00 2001 From: cake Date: Wed, 29 Oct 2025 04:43:18 +0100 Subject: [PATCH 10/28] Redone handling of windows compactor handling. --- LightlessSync/FileCache/FileCompactor.cs | 40 +++++++++++++++++------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index e5d219e..dbc2574 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -372,6 +372,22 @@ public sealed class FileCompactor : IDisposable return _clusterSizes[root]; } + public static bool UseSafeHandle(SafeHandle handle, Func action) + { + bool addedRef = false; + try + { + handle.DangerousAddRef(ref addedRef); + IntPtr ptr = handle.DangerousGetHandle(); + return action(ptr); + } + finally + { + if (addedRef) + handle.DangerousRelease(); + } + } + private static bool IsWOFCompactedFile(string filePath) { uint buf = 8; @@ -424,22 +440,25 @@ public sealed class FileCompactor : IDisposable ulong length = (ulong)Marshal.SizeOf(_efInfo); try { - using var fs = new FileStream(path, FileMode.Open); - #pragma warning disable S3869 // "SafeHandle.DangerousGetHandle" should not be called - var hFile = fs.SafeFileHandle.DangerousGetHandle(); - #pragma warning restore S3869 // "SafeHandle.DangerousGetHandle" should not be called - if (fs.SafeFileHandle.IsInvalid) + using var fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.None); + var handle = fs.SafeFileHandle; + + if (handle.IsInvalid) { _logger.LogWarning("Invalid file handle to {file}", path); return false; } - var ret = WofSetFileDataLocation(hFile, WOF_PROVIDER_FILE, efInfoPtr, length); - if (!(ret == 0 || ret == unchecked((int)0x80070158))) + return UseSafeHandle(handle, hFile => { - _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X")); - return false; - } + int ret = WofSetFileDataLocation(hFile, WOF_PROVIDER_FILE, efInfoPtr, length); + if (ret != 0 && ret != unchecked((int)0x80070158)) + { + _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X")); + return false; + } + return true; + }); } catch (Exception ex) { @@ -450,7 +469,6 @@ public sealed class FileCompactor : IDisposable { Marshal.FreeHGlobal(efInfoPtr); } - return true; } private bool BtrfsCompressFile(string path) From 3e626c5e47042c7941a9c52460c72ec98a92ac21 Mon Sep 17 00:00:00 2001 From: cake Date: Wed, 29 Oct 2025 04:49:46 +0100 Subject: [PATCH 11/28] Cleanup some code, removed ntfs usage on cache monitor --- LightlessSync/FileCache/CacheMonitor.cs | 2 +- LightlessSync/FileCache/FileCompactor.cs | 143 ++++++++++++++++++----- 2 files changed, 112 insertions(+), 33 deletions(-) diff --git a/LightlessSync/FileCache/CacheMonitor.cs b/LightlessSync/FileCache/CacheMonitor.cs index 23f5c19..c724225 100644 --- a/LightlessSync/FileCache/CacheMonitor.cs +++ b/LightlessSync/FileCache/CacheMonitor.cs @@ -429,7 +429,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase try { - return _fileCompactor.GetFileSizeOnDisk(f, StorageisNTFS); + return _fileCompactor.GetFileSizeOnDisk(f); } catch { diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index dbc2574..e961fbd 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -4,11 +4,8 @@ using LightlessSync.Utils; using Microsoft.Extensions.Logging; using System.Collections.Concurrent; using System.Diagnostics; -using System.IO; using System.Runtime.InteropServices; -using System.Threading; using System.Threading.Channels; -using System.Threading.Tasks; using static LightlessSync.Utils.FileSystemHelper; namespace LightlessSync.FileCache; @@ -89,27 +86,21 @@ public sealed class FileCompactor : IDisposable MassCompactRunning = false; } - public long GetFileSizeOnDisk(FileInfo fileInfo, bool? isNTFS = null) + public long GetFileSizeOnDisk(FileInfo fileInfo) { - var fsType = FileSystemHelper.GetFilesystemType(fileInfo.FullName); + var fsType = GetFilesystemType(fileInfo.FullName); - bool ntfs = isNTFS ?? fsType == FileSystemHelper.FilesystemType.NTFS; - - if (fsType != FileSystemHelper.FilesystemType.Btrfs && !ntfs) + if (fsType != FilesystemType.Btrfs && fsType != FilesystemType.NTFS) { return fileInfo.Length; } - if (ntfs && !_dalamudUtilService.IsWine) + if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) { - var clusterSize = GetClusterSize(fileInfo); - if (clusterSize == -1) return fileInfo.Length; - var losize = GetCompressedFileSizeW(fileInfo.FullName, out uint hosize); - var size = (long)hosize << 32 | losize; - return ((size + clusterSize - 1) / clusterSize) * clusterSize; + return GetFileSizeOnDisk(fileInfo, GetClusterSize); } - if (fsType == FileSystemHelper.FilesystemType.Btrfs) + if (fsType == FilesystemType.Btrfs) { try { @@ -163,6 +154,41 @@ public sealed class FileCompactor : IDisposable GC.SuppressFinalize(this); } + [DllImport("libc", SetLastError = true)] + private static extern int statvfs(string path, out Statvfs buf); + + [StructLayout(LayoutKind.Sequential)] + private struct Statvfs + { + public ulong f_bsize; + public ulong f_frsize; + public ulong f_blocks; + public ulong f_bfree; + public ulong f_bavail; + public ulong f_files; + public ulong f_ffree; + public ulong f_favail; + public ulong f_fsid; + public ulong f_flag; + public ulong f_namemax; + } + + private static int GetLinuxBlockSize(string path) + { + try + { + int result = statvfs(path, out var buf); + if (result != 0) + return -1; + + return (int)buf.f_frsize; + } + catch + { + return -1; + } + } + [DllImport("kernel32.dll")] private static extern int DeviceIoControl(IntPtr hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out IntPtr lpBytesReturned, out IntPtr lpOverlapped); @@ -190,7 +216,7 @@ public sealed class FileCompactor : IDisposable return; } - var fsType = FileSystemHelper.GetFilesystemType(filePath); + var fsType = GetFilesystemType(filePath); var oldSize = fi.Length; int clusterSize = GetClusterSize(fi); @@ -201,7 +227,7 @@ public sealed class FileCompactor : IDisposable } // NTFS Compression. - if (fsType == FileSystemHelper.FilesystemType.NTFS && !_dalamudUtilService.IsWine) + if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) { if (!IsWOFCompactedFile(filePath)) { @@ -250,6 +276,17 @@ public sealed class FileCompactor : IDisposable } } + private static long GetFileSizeOnDisk(FileInfo fileInfo, Func getClusterSize) + { + int clusterSize = getClusterSize(fileInfo); + if (clusterSize <= 0) + return fileInfo.Length; + + uint low = GetCompressedFileSizeW(fileInfo.FullName, out uint high); + long compressed = ((long)high << 32) | low; + return ((compressed + clusterSize - 1) / clusterSize) * clusterSize; + } + private static long RunStatGetBlocks(string path) { var psi = new ProcessStartInfo("stat", $"-c %b \"{path}\"") @@ -280,11 +317,10 @@ public sealed class FileCompactor : IDisposable private void DecompressFile(string path) { _logger.LogDebug("Removing compression from {file}", path); - var fsType = FileSystemHelper.GetFilesystemType(path); - if (fsType == null) return; + var fsType = GetFilesystemType(path); //NTFS Decompression - if (fsType == FileSystemHelper.FilesystemType.NTFS && !_dalamudUtilService.IsWine) + if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) { try { @@ -304,7 +340,7 @@ public sealed class FileCompactor : IDisposable } //BTRFS Decompression - if (fsType == FileSystemHelper.FilesystemType.Btrfs) + if (fsType == FilesystemType.Btrfs) { try { @@ -360,16 +396,60 @@ public sealed class FileCompactor : IDisposable private int GetClusterSize(FileInfo fi) { - if (!fi.Exists) return -1; - var root = fi.Directory?.Root.FullName.ToLower() ?? string.Empty; - if (string.IsNullOrEmpty(root)) return -1; - if (_clusterSizes.TryGetValue(root, out int value)) return value; - _logger.LogDebug("Getting Cluster Size for {path}, root {root}", fi.FullName, root); - int result = GetDiskFreeSpaceW(root, out uint sectorsPerCluster, out uint bytesPerSector, out _, out _); - if (result == 0) return -1; - _clusterSizes[root] = (int)(sectorsPerCluster * bytesPerSector); - _logger.LogDebug("Determined Cluster Size for root {root}: {cluster}", root, _clusterSizes[root]); - return _clusterSizes[root]; + try + { + if (!fi.Exists) + return -1; + + var root = fi.Directory?.Root.FullName; + if (string.IsNullOrEmpty(root)) + return -1; + + root = root.ToLowerInvariant(); + + if (_clusterSizes.TryGetValue(root, out int cached)) + return cached; + + _logger.LogDebug("Determining cluster/block size for {path} (root: {root})", fi.FullName, root); + + int clusterSize; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + int result = GetDiskFreeSpaceW( + root, + out uint sectorsPerCluster, + out uint bytesPerSector, + out _, + out _); + + if (result == 0) + { + _logger.LogWarning("GetDiskFreeSpaceW failed for {root}", root); + return -1; + } + + clusterSize = (int)(sectorsPerCluster * bytesPerSector); + } + else + { + clusterSize = GetLinuxBlockSize(root); + if (clusterSize <= 0) + { + _logger.LogWarning("Failed to determine block size for {root}", root); + return -1; + } + } + + _clusterSizes[root] = clusterSize; + _logger.LogDebug("Determined cluster/block size for {root}: {cluster} bytes", root, clusterSize); + return clusterSize; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error determining cluster size for {file}", fi.FullName); + return -1; + } } public static bool UseSafeHandle(SafeHandle handle, Func action) @@ -418,7 +498,6 @@ public sealed class FileCompactor : IDisposable string output = proc.StandardOutput.ReadToEnd(); proc.WaitForExit(); - // look for "flags: compressed" in the output if (output.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase)) { return true; From 3f85852618c6c3143c4ebe5fa14bc687f6269d05 Mon Sep 17 00:00:00 2001 From: cake Date: Wed, 29 Oct 2025 04:52:17 +0100 Subject: [PATCH 12/28] Added string comparisons --- LightlessSync/Utils/FileSystemHelper.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LightlessSync/Utils/FileSystemHelper.cs b/LightlessSync/Utils/FileSystemHelper.cs index a80e922..a80c900 100644 --- a/LightlessSync/Utils/FileSystemHelper.cs +++ b/LightlessSync/Utils/FileSystemHelper.cs @@ -83,7 +83,7 @@ namespace LightlessSync.Utils { var parts = line.Split(' '); if (parts.Length < 3) continue; - var mountPoint = parts[1].Replace("\\040", " "); // unescape spaces + var mountPoint = parts[1].Replace("\\040", " ", StringComparison.Ordinal); string normalizedMount; try { normalizedMount = Path.GetFullPath(mountPoint); } @@ -115,7 +115,7 @@ namespace LightlessSync.Utils { var parts = line.Split(' '); if (parts.Length < 3) continue; - var mount = parts[1].Replace("\\040", " "); + var mount = parts[1].Replace("\\040", " ", StringComparison.Ordinal); if (string.Equals(mount, mountPoint, StringComparison.Ordinal)) { var fstype = parts[2].ToLowerInvariant(); From c37e3badf16da5a55c536d4a4621f669ebafcd71 Mon Sep 17 00:00:00 2001 From: cake Date: Wed, 29 Oct 2025 06:09:44 +0100 Subject: [PATCH 13/28] Check if wine is used. --- LightlessSync/FileCache/CacheMonitor.cs | 2 +- LightlessSync/FileCache/FileCompactor.cs | 26 +++++++++++------------- LightlessSync/Utils/FileSystemHelper.cs | 8 +++++--- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/LightlessSync/FileCache/CacheMonitor.cs b/LightlessSync/FileCache/CacheMonitor.cs index c724225..95c3150 100644 --- a/LightlessSync/FileCache/CacheMonitor.cs +++ b/LightlessSync/FileCache/CacheMonitor.cs @@ -126,7 +126,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase Logger.LogWarning("Lightless file path is not set, cannot start the FSW for Lightless."); return; } - var fsType = FileSystemHelper.GetFilesystemType(_configService.Current.CacheFolder); + var fsType = FileSystemHelper.GetFilesystemType(_configService.Current.CacheFolder, _dalamudUtil.IsWine); if (fsType == FileSystemHelper.FilesystemType.NTFS) { diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index e961fbd..d1aa60a 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -4,6 +4,7 @@ using LightlessSync.Utils; using Microsoft.Extensions.Logging; using System.Collections.Concurrent; using System.Diagnostics; +using System.IO; using System.Runtime.InteropServices; using System.Threading.Channels; using static LightlessSync.Utils.FileSystemHelper; @@ -88,7 +89,7 @@ public sealed class FileCompactor : IDisposable public long GetFileSizeOnDisk(FileInfo fileInfo) { - var fsType = GetFilesystemType(fileInfo.FullName); + var fsType = GetFilesystemType(fileInfo.FullName, _dalamudUtilService.IsWine); if (fsType != FilesystemType.Btrfs && fsType != FilesystemType.NTFS) { @@ -216,7 +217,7 @@ public sealed class FileCompactor : IDisposable return; } - var fsType = GetFilesystemType(filePath); + var fsType = GetFilesystemType(filePath, _dalamudUtilService.IsWine); var oldSize = fi.Length; int clusterSize = GetClusterSize(fi); @@ -252,7 +253,7 @@ public sealed class FileCompactor : IDisposable } // BTRFS Compression - if (fsType == FileSystemHelper.FilesystemType.Btrfs) + if (fsType == FilesystemType.Btrfs) { if (!IsBtrfsCompressedFile(filePath)) { @@ -317,20 +318,18 @@ public sealed class FileCompactor : IDisposable private void DecompressFile(string path) { _logger.LogDebug("Removing compression from {file}", path); - var fsType = GetFilesystemType(path); + var fsType = GetFilesystemType(path, _dalamudUtilService.IsWine); //NTFS Decompression if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) { try { - using (var fs = new FileStream(path, FileMode.Open)) - { + using var fs = new FileStream(path, FileMode.Open); #pragma warning disable S3869 // "SafeHandle.DangerousGetHandle" should not be called - var hDevice = fs.SafeFileHandle.DangerousGetHandle(); + var hDevice = fs.SafeFileHandle.DangerousGetHandle(); #pragma warning restore S3869 // "SafeHandle.DangerousGetHandle" should not be called - _ = DeviceIoControl(hDevice, FSCTL_DELETE_EXTERNAL_BACKING, nint.Zero, 0, nint.Zero, 0, out _, out _); - } + _ = DeviceIoControl(hDevice, FSCTL_DELETE_EXTERNAL_BACKING, nint.Zero, 0, nint.Zero, 0, out _, out _); } catch (Exception ex) { @@ -380,7 +379,6 @@ public sealed class FileCompactor : IDisposable } else { - // Log output only in debug mode to avoid clutter if (!string.IsNullOrWhiteSpace(stdout)) _logger.LogDebug("btrfs defragment output for {file}: {out}", path, stdout.Trim()); @@ -414,7 +412,7 @@ public sealed class FileCompactor : IDisposable int clusterSize; - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && !_dalamudUtilService.IsWine) { int result = GetDiskFreeSpaceW( root, @@ -602,7 +600,7 @@ public sealed class FileCompactor : IDisposable { var parts = line.Split(' '); if (parts.Length < 4) continue; - var mountPoint = parts[1].Replace("\\040", " ", StringComparison.Ordinal); // unescape spaces + var mountPoint = parts[1].Replace("\\040", " ", StringComparison.Ordinal); string normalized; try { normalized = Path.GetFullPath(mountPoint); } catch { normalized = mountPoint; } @@ -635,7 +633,7 @@ public sealed class FileCompactor : IDisposable if (!_pendingCompactions.TryAdd(filePath, 0)) return; - var fsType = GetFilesystemType(filePath); + var fsType = GetFilesystemType(filePath, _dalamudUtilService.IsWine); if (fsType != FilesystemType.NTFS && fsType != FilesystemType.Btrfs) { @@ -700,7 +698,7 @@ public sealed class FileCompactor : IDisposable } catch (OperationCanceledException) { - // expected during shutdown + _logger.LogDebug("Queue has been cancelled by token"); } } } diff --git a/LightlessSync/Utils/FileSystemHelper.cs b/LightlessSync/Utils/FileSystemHelper.cs index a80c900..a5bb427 100644 --- a/LightlessSync/Utils/FileSystemHelper.cs +++ b/LightlessSync/Utils/FileSystemHelper.cs @@ -21,12 +21,12 @@ namespace LightlessSync.Utils private static readonly ConcurrentDictionary _filesystemTypeCache = new(StringComparer.OrdinalIgnoreCase); - public static FilesystemType GetFilesystemType(string filePath) + public static FilesystemType GetFilesystemType(string filePath, bool isWine = false) { try { string rootPath; - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && (!IsProbablyWine() || !isWine)) { var info = new FileInfo(filePath); var dir = info.Directory ?? new DirectoryInfo(filePath); @@ -44,7 +44,7 @@ namespace LightlessSync.Utils FilesystemType detected; - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && (!IsProbablyWine() || !isWine)) { var root = new DriveInfo(rootPath); var format = root.DriveFormat?.ToUpperInvariant() ?? string.Empty; @@ -139,5 +139,7 @@ namespace LightlessSync.Utils return FilesystemType.Unknown; } } + + private static bool IsProbablyWine() => Environment.GetEnvironmentVariable("WINELOADERNOEXEC") != null || Environment.GetEnvironmentVariable("WINEDLLPATH") != null || Directory.Exists("/proc/self") && File.Exists("/proc/mounts"); } } From 7c4d0fd5e99a897c0bbaa6ae2e819cf98e17185e Mon Sep 17 00:00:00 2001 From: cake Date: Wed, 29 Oct 2025 22:54:50 +0100 Subject: [PATCH 14/28] Added comments, clean-up --- LightlessSync/FileCache/CacheMonitor.cs | 12 ++++++--- LightlessSync/FileCache/FileCompactor.cs | 31 ++++++++++++------------ LightlessSync/Utils/Crypto.cs | 27 ++++++++++----------- LightlessSync/Utils/FileSystemHelper.cs | 14 ++++++----- 4 files changed, 45 insertions(+), 39 deletions(-) diff --git a/LightlessSync/FileCache/CacheMonitor.cs b/LightlessSync/FileCache/CacheMonitor.cs index 95c3150..64910f3 100644 --- a/LightlessSync/FileCache/CacheMonitor.cs +++ b/LightlessSync/FileCache/CacheMonitor.cs @@ -409,11 +409,17 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase return; } - FileCacheSize = -1; - DriveInfo di = new(new DirectoryInfo(_configService.Current.CacheFolder).Root.FullName); + FileCacheSize = -1; + + var drive = DriveInfo.GetDrives().FirstOrDefault(d => _configService.Current.CacheFolder.StartsWith(d.Name, StringComparison.Ordinal)); + if (drive == null) + { + return; + } + try { - FileCacheDriveFree = di.AvailableFreeSpace; + FileCacheDriveFree = drive.AvailableFreeSpace; } catch (Exception ex) { diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index d1aa60a..9a73e81 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -1,10 +1,8 @@ using LightlessSync.LightlessConfiguration; using LightlessSync.Services; -using LightlessSync.Utils; using Microsoft.Extensions.Logging; using System.Collections.Concurrent; using System.Diagnostics; -using System.IO; using System.Runtime.InteropServices; using System.Threading.Channels; using static LightlessSync.Utils.FileSystemHelper; @@ -106,6 +104,7 @@ public sealed class FileCompactor : IDisposable try { long blocks = RunStatGetBlocks(fileInfo.FullName); + //st_blocks are always calculated in 512-byte units, hence we use 512L return blocks * 512L; } catch (Exception ex) @@ -161,17 +160,17 @@ public sealed class FileCompactor : IDisposable [StructLayout(LayoutKind.Sequential)] private struct Statvfs { - public ulong f_bsize; - public ulong f_frsize; - public ulong f_blocks; - public ulong f_bfree; - public ulong f_bavail; - public ulong f_files; - public ulong f_ffree; - public ulong f_favail; - public ulong f_fsid; - public ulong f_flag; - public ulong f_namemax; + public ulong f_bsize; /* Filesystem block size */ + public ulong f_frsize; /* Fragment size */ + public ulong f_blocks; /* Size of fs in f_frsize units */ + public ulong f_bfree; /* Number of free blocks */ + public ulong f_bavail; /* Number of free blocks for unprivileged users */ + public ulong f_files; /* Number of inodes */ + public ulong f_ffree; /* Number of free inodes */ + public ulong f_favail; /* Number of free inodes for unprivileged users */ + public ulong f_fsid; /* Filesystem ID */ + public ulong f_flag; /* Mount flags */ + public ulong f_namemax; /* Maximum filename length */ } private static int GetLinuxBlockSize(string path) @@ -182,6 +181,7 @@ public sealed class FileCompactor : IDisposable if (result != 0) return -1; + //return fragment size of linux return (int)buf.f_frsize; } catch @@ -346,9 +346,7 @@ public sealed class FileCompactor : IDisposable var mountOptions = GetMountOptionsForPath(path); if (mountOptions.Contains("compress", StringComparison.OrdinalIgnoreCase)) { - _logger.LogWarning( - "Cannot safely decompress {file}: filesystem mounted with compression ({opts}). " + - "Remount with 'compress=no' before running decompression.", path, mountOptions); + _logger.LogWarning("Cannot safely decompress {file}: filesystem mounted with compression ({opts}). Remount with 'compress=no' before running decompression.", path, mountOptions); return; } @@ -369,6 +367,7 @@ public sealed class FileCompactor : IDisposable return; } + //End stream of process to read the files var stdout = proc.StandardOutput.ReadToEnd(); var stderr = proc.StandardError.ReadToEnd(); proc.WaitForExit(); diff --git a/LightlessSync/Utils/Crypto.cs b/LightlessSync/Utils/Crypto.cs index 87d6883..c31f82f 100644 --- a/LightlessSync/Utils/Crypto.cs +++ b/LightlessSync/Utils/Crypto.cs @@ -1,16 +1,15 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Security.Cryptography; +using System.Security.Cryptography; using System.Text; namespace LightlessSync.Utils; public static class Crypto { + //This buffersize seems to be the best sweetpoint for Linux and Windows + private const int _bufferSize = 65536; #pragma warning disable SYSLIB0021 // Type or member is obsolete - private static readonly Dictionary<(string, ushort), string> _hashListPlayersSHA256 = new(); + private static readonly Dictionary<(string, ushort), string> _hashListPlayersSHA256 = []; private static readonly Dictionary _hashListSHA256 = new(StringComparer.Ordinal); private static readonly SHA256CryptoServiceProvider _sha256CryptoProvider = new(); @@ -23,21 +22,21 @@ public static class Crypto public static async Task GetFileHashAsync(string filePath, CancellationToken cancellationToken = default) { - var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, bufferSize: 65536, options: FileOptions.Asynchronous); + var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, bufferSize: _bufferSize, options: FileOptions.Asynchronous); await using (stream.ConfigureAwait(false)) { using var sha1 = SHA1.Create(); - var buffer = new byte[8192]; - int bytesRead; - while ((bytesRead = await stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) > 0) - { - sha1.TransformBlock(buffer, 0, bytesRead, outputBuffer: null, 0); - } + var buffer = new byte[8192]; + int bytesRead; + while ((bytesRead = await stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) > 0) + { + sha1.TransformBlock(buffer, 0, bytesRead, outputBuffer: null, 0); + } - sha1.TransformFinalBlock([], 0, 0); + sha1.TransformFinalBlock([], 0, 0); - return BitConverter.ToString(sha1.Hash!).Replace("-", "", StringComparison.Ordinal); + return Convert.ToHexString(sha1.Hash!); } } diff --git a/LightlessSync/Utils/FileSystemHelper.cs b/LightlessSync/Utils/FileSystemHelper.cs index a5bb427..cda36c3 100644 --- a/LightlessSync/Utils/FileSystemHelper.cs +++ b/LightlessSync/Utils/FileSystemHelper.cs @@ -8,17 +8,18 @@ namespace LightlessSync.Utils public enum FilesystemType { Unknown = 0, - NTFS, - Btrfs, + NTFS, // Compressable + Btrfs, // Compressable Ext4, Xfs, Apfs, HfsPlus, Fat, Exfat, - Zfs + Zfs // Compressable } + private const string _mountPath = "/proc/mounts"; private static readonly ConcurrentDictionary _filesystemTypeCache = new(StringComparer.OrdinalIgnoreCase); public static FilesystemType GetFilesystemType(string filePath, bool isWine = false) @@ -75,8 +76,8 @@ namespace LightlessSync.Utils try { var path = Path.GetFullPath(filePath); - if (!File.Exists("/proc/mounts")) return "/"; - var mounts = File.ReadAllLines("/proc/mounts"); + if (!File.Exists(_mountPath)) return "/"; + var mounts = File.ReadAllLines(_mountPath); string bestMount = "/"; foreach (var line in mounts) @@ -109,7 +110,7 @@ namespace LightlessSync.Utils try { var mountPoint = GetMountPoint(filePath); - var mounts = File.ReadAllLines("/proc/mounts"); + var mounts = File.ReadAllLines(_mountPath); foreach (var line in mounts) { @@ -140,6 +141,7 @@ namespace LightlessSync.Utils } } + //Extra check on private static bool IsProbablyWine() => Environment.GetEnvironmentVariable("WINELOADERNOEXEC") != null || Environment.GetEnvironmentVariable("WINEDLLPATH") != null || Directory.Exists("/proc/self") && File.Exists("/proc/mounts"); } } From b3cc41382f28f8edf6dd49ee023a0174ae297a66 Mon Sep 17 00:00:00 2001 From: cake Date: Thu, 30 Oct 2025 03:05:53 +0100 Subject: [PATCH 15/28] Refactored a bit, added comments on the file systems. --- LightlessSync/FileCache/FileCompactor.cs | 53 +++--------------------- LightlessSync/Utils/FileSystemHelper.cs | 52 +++++++++++++++++++---- 2 files changed, 49 insertions(+), 56 deletions(-) diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index 9a73e81..60cc78e 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -1,5 +1,6 @@ using LightlessSync.LightlessConfiguration; using LightlessSync.Services; +using LightlessSync.Utils; using Microsoft.Extensions.Logging; using System.Collections.Concurrent; using System.Diagnostics; @@ -181,7 +182,7 @@ public sealed class FileCompactor : IDisposable if (result != 0) return -1; - //return fragment size of linux + //return fragment size of Linux file system return (int)buf.f_frsize; } catch @@ -413,12 +414,7 @@ public sealed class FileCompactor : IDisposable if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && !_dalamudUtilService.IsWine) { - int result = GetDiskFreeSpaceW( - root, - out uint sectorsPerCluster, - out uint bytesPerSector, - out _, - out _); + int result = GetDiskFreeSpaceW(root, out uint sectorsPerCluster, out uint bytesPerSector, out _, out _); if (result == 0) { @@ -495,12 +491,10 @@ public sealed class FileCompactor : IDisposable string output = proc.StandardOutput.ReadToEnd(); proc.WaitForExit(); - if (output.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase)) - { - return true; - } + bool compressed = output.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase); - return false; + _logger.LogTrace("Btrfs compression check for {file}: {compressed}", path, compressed); + return compressed; } catch (Exception ex) { @@ -586,41 +580,6 @@ public sealed class FileCompactor : IDisposable } } - private string GetMountOptionsForPath(string path) - { - try - { - var fullPath = Path.GetFullPath(path); - var mounts = File.ReadAllLines("/proc/mounts"); - string bestMount = string.Empty; - string mountOptions = string.Empty; - - foreach (var line in mounts) - { - var parts = line.Split(' '); - if (parts.Length < 4) continue; - var mountPoint = parts[1].Replace("\\040", " ", StringComparison.Ordinal); - string normalized; - try { normalized = Path.GetFullPath(mountPoint); } - catch { normalized = mountPoint; } - - if (fullPath.StartsWith(normalized, StringComparison.Ordinal) && - normalized.Length > bestMount.Length) - { - bestMount = normalized; - mountOptions = parts[3]; - } - } - - return mountOptions; - } - catch (Exception ex) - { - _logger.LogDebug(ex, "Failed to get mount options for {path}", path); - return string.Empty; - } - } - private struct WOF_FILE_COMPRESSION_INFO_V1 { public CompressionAlgorithm Algorithm; diff --git a/LightlessSync/Utils/FileSystemHelper.cs b/LightlessSync/Utils/FileSystemHelper.cs index cda36c3..4bbfc75 100644 --- a/LightlessSync/Utils/FileSystemHelper.cs +++ b/LightlessSync/Utils/FileSystemHelper.cs @@ -8,15 +8,15 @@ namespace LightlessSync.Utils public enum FilesystemType { Unknown = 0, - NTFS, // Compressable - Btrfs, // Compressable - Ext4, - Xfs, - Apfs, - HfsPlus, - Fat, - Exfat, - Zfs // Compressable + NTFS, // Compressable on file level + Btrfs, // Compressable on file level + Ext4, // Uncompressable + Xfs, // Uncompressable + Apfs, // Compressable on OS + HfsPlus, // Compressable on OS + Fat, // Uncompressable + Exfat, // Uncompressable + Zfs // Compressable, not on file level } private const string _mountPath = "/proc/mounts"; @@ -105,6 +105,40 @@ namespace LightlessSync.Utils } } + public static string GetMountOptionsForPath(string path) + { + try + { + var fullPath = Path.GetFullPath(path); + var mounts = File.ReadAllLines("/proc/mounts"); + string bestMount = string.Empty; + string mountOptions = string.Empty; + + foreach (var line in mounts) + { + var parts = line.Split(' '); + if (parts.Length < 4) continue; + var mountPoint = parts[1].Replace("\\040", " ", StringComparison.Ordinal); + string normalized; + try { normalized = Path.GetFullPath(mountPoint); } + catch { normalized = mountPoint; } + + if (fullPath.StartsWith(normalized, StringComparison.Ordinal) && + normalized.Length > bestMount.Length) + { + bestMount = normalized; + mountOptions = parts[3]; + } + } + + return mountOptions; + } + catch (Exception ex) + { + return string.Empty; + } + } + private static FilesystemType GetLinuxFilesystemType(string filePath) { try From bf139c128b75b8d744864cb9b574c74c91e65ad6 Mon Sep 17 00:00:00 2001 From: cake Date: Thu, 30 Oct 2025 03:11:38 +0100 Subject: [PATCH 16/28] Added fail safes in compact of WOF incase --- LightlessSync/FileCache/FileCompactor.cs | 32 ++++++++++++++++++++---- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index 60cc78e..2e32a3e 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -138,7 +138,7 @@ public sealed class FileCompactor : IDisposable _compactionCts.Cancel(); try { - if (!_compactionWorker.Wait(TimeSpan.FromSeconds(5))) + if (!_compactionWorker.Wait(TimeSpan.FromSeconds(5), _compactionCts.Token)) { _logger.LogDebug("Compaction worker did not shut down within timeout"); } @@ -463,10 +463,32 @@ public sealed class FileCompactor : IDisposable private static bool IsWOFCompactedFile(string filePath) { - uint buf = 8; - _ = WofIsExternalFile(filePath, out int isExtFile, out uint _, out var info, ref buf); - if (isExtFile == 0) return false; - return info.Algorithm == CompressionAlgorithm.XPRESS8K; + try + { + uint buf = (uint)Marshal.SizeOf(); + int result = WofIsExternalFile(filePath, out int isExternal, out uint _, out var info, ref buf); + + if (result != 0 || isExternal == 0) + return false; + + return info.Algorithm == CompressionAlgorithm.XPRESS8K || info.Algorithm == CompressionAlgorithm.XPRESS4K + || info.Algorithm == CompressionAlgorithm.XPRESS16K || info.Algorithm == CompressionAlgorithm.LZX; + } + catch (DllNotFoundException) + { + // WofUtil.dll not available + return false; + } + catch (EntryPointNotFoundException) + { + // Running under Wine or non-NTFS systems + return false; + } + catch (Exception) + { + // Exception happened + return false; + } } private bool IsBtrfsCompressedFile(string path) From c1770528f3c79300ac7fc42f65f862cbb72e37d9 Mon Sep 17 00:00:00 2001 From: cake Date: Thu, 30 Oct 2025 03:34:56 +0100 Subject: [PATCH 17/28] Added wine checks, path fixing on wine -> linux --- LightlessSync/FileCache/FileCompactor.cs | 105 ++++++++++++++++++++--- 1 file changed, 91 insertions(+), 14 deletions(-) diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index 2e32a3e..a8170f4 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -1,6 +1,5 @@ using LightlessSync.LightlessConfiguration; using LightlessSync.Services; -using LightlessSync.Utils; using Microsoft.Extensions.Logging; using System.Collections.Concurrent; using System.Diagnostics; @@ -191,6 +190,16 @@ public sealed class FileCompactor : IDisposable } } + private static string ConvertWinePathToLinux(string winePath) + { + if (winePath.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) + return "/" + winePath.Substring(3).Replace('\\', '/'); + if (winePath.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) + return Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Personal), + winePath.Substring(3).Replace('\\', '/')).Replace('\\', '/'); + return winePath.Replace('\\', '/'); + } + [DllImport("kernel32.dll")] private static extern int DeviceIoControl(IntPtr hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out IntPtr lpBytesReturned, out IntPtr lpOverlapped); @@ -347,18 +356,42 @@ public sealed class FileCompactor : IDisposable var mountOptions = GetMountOptionsForPath(path); if (mountOptions.Contains("compress", StringComparison.OrdinalIgnoreCase)) { - _logger.LogWarning("Cannot safely decompress {file}: filesystem mounted with compression ({opts}). Remount with 'compress=no' before running decompression.", path, mountOptions); + _logger.LogWarning( + "Cannot safely decompress {file}: filesystem mounted with compression ({opts}). Remount with 'compress=no' before running decompression.", + path, mountOptions); return; } - _logger.LogDebug("Rewriting {file} to remove btrfs compression...", path); - - var psi = new ProcessStartInfo("btrfs", $"filesystem defragment -- \"{path}\"") + string realPath = path; + bool isWine = _dalamudUtilService?.IsWine ?? false; + if (isWine) { + if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) + { + realPath = "/" + path.Substring(3).Replace('\\', '/'); + } + else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) + { + // fallback for Wine's C:\ mapping + realPath = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.Personal), + path.Substring(3).Replace('\\', '/') + ).Replace('\\', '/'); + } + + _logger.LogTrace("Detected Wine environment. Converted path for decompression: {realPath}", realPath); + } + + string command = $"btrfs filesystem defragment -- \"{realPath}\""; + var psi = new ProcessStartInfo + { + FileName = isWine ? "/bin/bash" : "btrfs", + Arguments = isWine ? $"-c \"{command}\"" : $"filesystem defragment -- \"{realPath}\"", RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, - CreateNoWindow = true + CreateNoWindow = true, + WorkingDirectory = "/" }; using var proc = Process.Start(psi); @@ -368,7 +401,7 @@ public sealed class FileCompactor : IDisposable return; } - //End stream of process to read the files + // 4️⃣ Read process output var stdout = proc.StandardOutput.ReadToEnd(); var stderr = proc.StandardError.ReadToEnd(); proc.WaitForExit(); @@ -495,12 +528,36 @@ public sealed class FileCompactor : IDisposable { try { - var psi = new ProcessStartInfo("filefrag", $"-v \"{path}\"") + bool isWine = _dalamudUtilService?.IsWine ?? false; + string realPath = path; + + if (isWine) { + if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) + { + realPath = "/" + path.Substring(3).Replace('\\', '/'); + } + else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) + { + realPath = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.Personal), + path.Substring(3).Replace('\\', '/') + ).Replace('\\', '/'); + } + + _logger.LogTrace("Detected Wine environment. Converted path for filefrag: {realPath}", realPath); + } + + string command = $"filefrag -v -- \"{realPath}\""; + var psi = new ProcessStartInfo + { + FileName = isWine ? "/bin/bash" : "filefrag", + Arguments = isWine ? $"-c \"{command}\"" : $"-v -- \"{realPath}\"", RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, - CreateNoWindow = true + CreateNoWindow = true, + WorkingDirectory = "/" }; using var proc = Process.Start(psi); @@ -511,10 +568,17 @@ public sealed class FileCompactor : IDisposable } string output = proc.StandardOutput.ReadToEnd(); + string stderr = proc.StandardError.ReadToEnd(); proc.WaitForExit(); - bool compressed = output.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase); + if (proc.ExitCode != 0) + { + _logger.LogDebug("filefrag exited with {code} for {file}. stderr: {stderr}", + proc.ExitCode, path, stderr); + return false; + } + bool compressed = output.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase); _logger.LogTrace("Btrfs compression check for {file}: {compressed}", path, compressed); return compressed; } @@ -567,7 +631,20 @@ public sealed class FileCompactor : IDisposable { try { - var psi = new ProcessStartInfo("btrfs", $"filesystem defragment -czstd -- \"{path}\"") + string realPath = path; + if (_dalamudUtilService.IsWine) + { + realPath = ConvertWinePathToLinux(path); + _logger.LogTrace("Detected Wine environment, remapped path: {realPath}", realPath); + } + + if (!File.Exists("/usr/bin/btrfs") && !File.Exists("/bin/btrfs")) + { + _logger.LogWarning("Skipping Btrfs compression — btrfs binary not found"); + return false; + } + + var psi = new ProcessStartInfo("btrfs", $"filesystem defragment -czstd -- \"{realPath}\"") { RedirectStandardOutput = true, RedirectStandardError = true, @@ -578,7 +655,7 @@ public sealed class FileCompactor : IDisposable using var proc = Process.Start(psi); if (proc == null) { - _logger.LogWarning("Failed to start btrfs process for {file}", path); + _logger.LogWarning("Failed to start btrfs process for {file}", realPath); return false; } @@ -588,7 +665,7 @@ public sealed class FileCompactor : IDisposable if (proc.ExitCode != 0) { - _logger.LogWarning("btrfs defrag returned {code} for {file}: {err}", proc.ExitCode, path, stderr); + _logger.LogWarning("btrfs defrag returned {code} for {file}: {err}", proc.ExitCode, realPath, stderr); return false; } @@ -597,7 +674,7 @@ public sealed class FileCompactor : IDisposable } catch (Exception ex) { - _logger.LogWarning(ex, "Error running btrfs defragment for {file}", path); + _logger.LogWarning(ex, "Error running btrfs defragment for {file}", realPath); return false; } } From 5feb74c1c08594245e5a523339dc685d70c357c6 Mon Sep 17 00:00:00 2001 From: cake Date: Thu, 30 Oct 2025 03:46:55 +0100 Subject: [PATCH 18/28] Added another wine check in parralel with dalamud --- LightlessSync/FileCache/FileCompactor.cs | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index a8170f4..5f05549 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -364,7 +364,7 @@ public sealed class FileCompactor : IDisposable string realPath = path; bool isWine = _dalamudUtilService?.IsWine ?? false; - if (isWine) + if (isWine && IsProbablyWine()) { if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) { @@ -372,7 +372,6 @@ public sealed class FileCompactor : IDisposable } else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) { - // fallback for Wine's C:\ mapping realPath = Path.Combine( Environment.GetFolderPath(Environment.SpecialFolder.Personal), path.Substring(3).Replace('\\', '/') @@ -401,7 +400,6 @@ public sealed class FileCompactor : IDisposable return; } - // 4️⃣ Read process output var stdout = proc.StandardOutput.ReadToEnd(); var stderr = proc.StandardError.ReadToEnd(); proc.WaitForExit(); @@ -531,7 +529,7 @@ public sealed class FileCompactor : IDisposable bool isWine = _dalamudUtilService?.IsWine ?? false; string realPath = path; - if (isWine) + if (isWine && IsProbablyWine()) { if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) { @@ -539,10 +537,7 @@ public sealed class FileCompactor : IDisposable } else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) { - realPath = Path.Combine( - Environment.GetFolderPath(Environment.SpecialFolder.Personal), - path.Substring(3).Replace('\\', '/') - ).Replace('\\', '/'); + realPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Personal), path.Substring(3).Replace('\\', '/')).Replace('\\', '/'); } _logger.LogTrace("Detected Wine environment. Converted path for filefrag: {realPath}", realPath); @@ -632,7 +627,7 @@ public sealed class FileCompactor : IDisposable try { string realPath = path; - if (_dalamudUtilService.IsWine) + if (_dalamudUtilService.IsWine && IsProbablyWine()) { realPath = ConvertWinePathToLinux(path); _logger.LogTrace("Detected Wine environment, remapped path: {realPath}", realPath); @@ -674,7 +669,7 @@ public sealed class FileCompactor : IDisposable } catch (Exception ex) { - _logger.LogWarning(ex, "Error running btrfs defragment for {file}", realPath); + _logger.LogWarning(ex, "Error running btrfs defragment for {file}", path); return false; } } @@ -758,4 +753,6 @@ public sealed class FileCompactor : IDisposable _logger.LogDebug("Queue has been cancelled by token"); } } + + private static bool IsProbablyWine() => Environment.GetEnvironmentVariable("WINELOADERNOEXEC") != null || Environment.GetEnvironmentVariable("WINEDLLPATH") != null || Directory.Exists("/proc/self") && File.Exists("/proc/mounts"); } From 6e3c60f627c5a732dc5a15b02647b1d67989a723 Mon Sep 17 00:00:00 2001 From: cake Date: Fri, 31 Oct 2025 23:47:41 +0100 Subject: [PATCH 19/28] Changes in file compression for windows, redone linux side because wine issues. --- LightlessSync/FileCache/CacheMonitor.cs | 68 +- LightlessSync/FileCache/FileCompactor.cs | 960 ++++++++++++----------- LightlessSync/Utils/FileSystemHelper.cs | 107 ++- 3 files changed, 627 insertions(+), 508 deletions(-) diff --git a/LightlessSync/FileCache/CacheMonitor.cs b/LightlessSync/FileCache/CacheMonitor.cs index 64910f3..85fb30e 100644 --- a/LightlessSync/FileCache/CacheMonitor.cs +++ b/LightlessSync/FileCache/CacheMonitor.cs @@ -137,7 +137,7 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase if (fsType == FileSystemHelper.FilesystemType.Btrfs) { StorageIsBtrfs = true; - Logger.LogInformation("Lightless Storage is on BTRFS drive: {isNtfs}", StorageIsBtrfs); + Logger.LogInformation("Lightless Storage is on BTRFS drive: {isBtrfs}", StorageIsBtrfs); } Logger.LogDebug("Initializing Lightless FSW on {path}", lightlessPath); @@ -661,44 +661,44 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase if (ct.IsCancellationRequested) return; - // scan new files - if (allScannedFiles.Any(c => !c.Value)) + var newFiles = allScannedFiles.Where(c => !c.Value).Select(c => c.Key).ToList(); + foreach (var cachePath in newFiles) { - Parallel.ForEach(allScannedFiles.Where(c => !c.Value).Select(c => c.Key), - new ParallelOptions() - { - MaxDegreeOfParallelism = threadCount, - CancellationToken = ct - }, (cachePath) => - { - if (_fileDbManager == null || _ipcManager?.Penumbra == null || cachePath == null) - { - Logger.LogTrace("Potential null in db: {isDbNull} penumbra: {isPenumbraNull} cachepath: {isPathNull}", _fileDbManager == null, _ipcManager?.Penumbra == null, cachePath == null); - return; - } + if (ct.IsCancellationRequested) break; + ProcessOne(cachePath); + Interlocked.Increment(ref _currentFileProgress); + } - if (ct.IsCancellationRequested) return; + Logger.LogTrace("Scanner added {count} new files to db", newFiles.Count); - if (!_ipcManager.Penumbra.APIAvailable) - { - Logger.LogWarning("Penumbra not available"); - return; - } + void ProcessOne(string? cachePath) + { + if (_fileDbManager == null || _ipcManager?.Penumbra == null || cachePath == null) + { + Logger.LogTrace("Potential null in db: {isDbNull} penumbra: {isPenumbraNull} cachepath: {isPathNull}", + _fileDbManager == null, _ipcManager?.Penumbra == null, cachePath == null); + return; + } - try - { - var entry = _fileDbManager.CreateFileEntry(cachePath); - if (entry == null) _ = _fileDbManager.CreateCacheEntry(cachePath); - } - catch (Exception ex) - { - Logger.LogWarning(ex, "Failed adding {file}", cachePath); - } + if (!_ipcManager.Penumbra.APIAvailable) + { + Logger.LogWarning("Penumbra not available"); + return; + } - Interlocked.Increment(ref _currentFileProgress); - }); - - Logger.LogTrace("Scanner added {notScanned} new files to db", allScannedFiles.Count(c => !c.Value)); + try + { + var entry = _fileDbManager.CreateFileEntry(cachePath); + if (entry == null) _ = _fileDbManager.CreateCacheEntry(cachePath); + } + catch (IOException ioex) + { + Logger.LogDebug(ioex, "File busy or locked: {file}", cachePath); + } + catch (Exception ex) + { + Logger.LogWarning(ex, "Failed adding {file}", cachePath); + } } Logger.LogDebug("Scan complete"); diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index 5f05549..a917b59 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -1,6 +1,7 @@ using LightlessSync.LightlessConfiguration; using LightlessSync.Services; using Microsoft.Extensions.Logging; +using Microsoft.Win32.SafeHandles; using System.Collections.Concurrent; using System.Diagnostics; using System.Runtime.InteropServices; @@ -14,44 +15,21 @@ public sealed class FileCompactor : IDisposable public const uint FSCTL_DELETE_EXTERNAL_BACKING = 0x90314U; public const ulong WOF_PROVIDER_FILE = 2UL; - private readonly Dictionary _clusterSizes; private readonly ConcurrentDictionary _pendingCompactions; - private readonly WOF_FILE_COMPRESSION_INFO_V1 _efInfo; private readonly ILogger _logger; - private readonly LightlessConfigService _lightlessConfigService; private readonly DalamudUtilService _dalamudUtilService; + private readonly Channel _compactionQueue; private readonly CancellationTokenSource _compactionCts = new(); private readonly Task _compactionWorker; - - public FileCompactor(ILogger logger, LightlessConfigService lightlessConfigService, DalamudUtilService dalamudUtilService) + + private readonly WOF_FILE_COMPRESSION_INFO_V1 _efInfo = new() { - _clusterSizes = new(StringComparer.Ordinal); - _pendingCompactions = new(StringComparer.OrdinalIgnoreCase); - _logger = logger; - _lightlessConfigService = lightlessConfigService; - _dalamudUtilService = dalamudUtilService; - _efInfo = new WOF_FILE_COMPRESSION_INFO_V1 - { - Algorithm = CompressionAlgorithm.XPRESS8K, - Flags = 0 - }; - - _compactionQueue = Channel.CreateUnbounded(new UnboundedChannelOptions - { - SingleReader = true, - SingleWriter = false - }); - _compactionWorker = Task.Factory.StartNew( - () => ProcessQueueAsync(_compactionCts.Token), - _compactionCts.Token, - TaskCreationOptions.LongRunning, - TaskScheduler.Default) - .Unwrap(); - } - - private enum CompressionAlgorithm + Algorithm = (int)CompressionAlgorithm.XPRESS8K, + Flags = 0 + }; + private enum CompressionAlgorithm { NO_COMPRESSION = -2, LZNT1 = -1, @@ -61,493 +39,500 @@ public sealed class FileCompactor : IDisposable XPRESS16K = 3 } - public bool MassCompactRunning { get; private set; } = false; + public FileCompactor(ILogger logger, LightlessConfigService lightlessConfigService, DalamudUtilService dalamudUtilService) + { + _pendingCompactions = new(StringComparer.OrdinalIgnoreCase); + _logger = logger; + _lightlessConfigService = lightlessConfigService; + _dalamudUtilService = dalamudUtilService; + _compactionQueue = Channel.CreateUnbounded(new UnboundedChannelOptions + { + SingleReader = true, + SingleWriter = false + }); + + _compactionWorker = Task.Factory.StartNew(() => ProcessQueueAsync(_compactionCts.Token), _compactionCts.Token, TaskCreationOptions.LongRunning,TaskScheduler.Default).Unwrap(); + } + + public bool MassCompactRunning { get; private set; } public string Progress { get; private set; } = string.Empty; public void CompactStorage(bool compress) { MassCompactRunning = true; - - int currentFile = 1; - var allFiles = Directory.EnumerateFiles(_lightlessConfigService.Current.CacheFolder).ToList(); - int allFilesCount = allFiles.Count; - foreach (var file in allFiles) + try { - Progress = $"{currentFile}/{allFilesCount}"; - if (compress) - CompactFile(file); - else - DecompressFile(file); - currentFile++; - } + var allFiles = Directory.EnumerateFiles(_lightlessConfigService.Current.CacheFolder).ToList(); + int total = allFiles.Count; + int current = 0; - MassCompactRunning = false; + foreach (var file in allFiles) + { + current++; + Progress = $"{current}/{total}"; + + try + { + if (compress) + CompactFile(file); + else + DecompressFile(file); + } + catch (IOException ioEx) + { + _logger.LogDebug(ioEx, "File {file} locked or busy, skipping", file); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error compacting/decompressing file {file}", file); + } + } + } + finally + { + MassCompactRunning = false; + Progress = string.Empty; + } } + /// + /// Write all bytes into a directory async + /// + /// Bytes will be writen to this filepath + /// Bytes that have to be written + /// Cancellation Token for interupts + /// Writing Task + public async Task WriteAllBytesAsync(string filePath, byte[] bytes, CancellationToken token) + { + var dir = Path.GetDirectoryName(filePath); + if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir)) + Directory.CreateDirectory(dir); + + await File.WriteAllBytesAsync(filePath, bytes, token).ConfigureAwait(false); + + if (_lightlessConfigService.Current.UseCompactor) + EnqueueCompaction(filePath); + } + + /// + /// Gets the File size for an BTRFS or NTFS file system for the given FileInfo + /// + /// Amount of blocks used in the disk public long GetFileSizeOnDisk(FileInfo fileInfo) { var fsType = GetFilesystemType(fileInfo.FullName, _dalamudUtilService.IsWine); - if (fsType != FilesystemType.Btrfs && fsType != FilesystemType.NTFS) - { - return fileInfo.Length; - } - if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) { - return GetFileSizeOnDisk(fileInfo, GetClusterSize); + var blockSize = GetBlockSizeForPath(fileInfo.FullName, _logger, _dalamudUtilService.IsWine); + var losize = GetCompressedFileSizeW(fileInfo.FullName, out uint hosize); + var size = (long)hosize << 32 | losize; + return ((size + blockSize - 1) / blockSize) * blockSize; } if (fsType == FilesystemType.Btrfs) { try { - long blocks = RunStatGetBlocks(fileInfo.FullName); - //st_blocks are always calculated in 512-byte units, hence we use 512L + var realPath = fileInfo.FullName.Replace("\"", "\\\"", StringComparison.Ordinal); + var psi = new ProcessStartInfo("stat", $"-c %b \"{realPath}\"") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + using var proc = Process.Start(psi) ?? throw new InvalidOperationException("Could not start stat"); + var outp = proc.StandardOutput.ReadToEnd(); + var err = proc.StandardError.ReadToEnd(); + proc.WaitForExit(); + + if (proc.ExitCode != 0) + throw new InvalidOperationException($"stat failed: {err}"); + + if (!long.TryParse(outp.Trim(), out var blocks)) + throw new InvalidOperationException($"invalid stat output: {outp}"); + + // st_blocks are always 512-byte on Linux enviroment. return blocks * 512L; } catch (Exception ex) { - _logger.LogDebug(ex, "Failed to get on-disk size via stat for {file}, falling back to Length", fileInfo.FullName); - return fileInfo.Length; + _logger.LogDebug(ex, "Failed stat size for {file}, fallback to Length", fileInfo.FullName); } } return fileInfo.Length; } - public async Task WriteAllBytesAsync(string filePath, byte[] decompressedFile, CancellationToken token) - { - var dir = Path.GetDirectoryName(filePath); - if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir)) - Directory.CreateDirectory(dir); - - await File.WriteAllBytesAsync(filePath, decompressedFile, token).ConfigureAwait(false); - - if (!_lightlessConfigService.Current.UseCompactor) - return; - - EnqueueCompaction(filePath); - } - - public void Dispose() - { - _compactionQueue.Writer.TryComplete(); - _compactionCts.Cancel(); - try - { - if (!_compactionWorker.Wait(TimeSpan.FromSeconds(5), _compactionCts.Token)) - { - _logger.LogDebug("Compaction worker did not shut down within timeout"); - } - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - _logger.LogDebug(ex, "Error shutting down compaction worker"); - } - finally - { - _compactionCts.Dispose(); - } - - GC.SuppressFinalize(this); - } - - [DllImport("libc", SetLastError = true)] - private static extern int statvfs(string path, out Statvfs buf); - - [StructLayout(LayoutKind.Sequential)] - private struct Statvfs - { - public ulong f_bsize; /* Filesystem block size */ - public ulong f_frsize; /* Fragment size */ - public ulong f_blocks; /* Size of fs in f_frsize units */ - public ulong f_bfree; /* Number of free blocks */ - public ulong f_bavail; /* Number of free blocks for unprivileged users */ - public ulong f_files; /* Number of inodes */ - public ulong f_ffree; /* Number of free inodes */ - public ulong f_favail; /* Number of free inodes for unprivileged users */ - public ulong f_fsid; /* Filesystem ID */ - public ulong f_flag; /* Mount flags */ - public ulong f_namemax; /* Maximum filename length */ - } - - private static int GetLinuxBlockSize(string path) - { - try - { - int result = statvfs(path, out var buf); - if (result != 0) - return -1; - - //return fragment size of Linux file system - return (int)buf.f_frsize; - } - catch - { - return -1; - } - } - - private static string ConvertWinePathToLinux(string winePath) - { - if (winePath.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) - return "/" + winePath.Substring(3).Replace('\\', '/'); - if (winePath.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) - return Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Personal), - winePath.Substring(3).Replace('\\', '/')).Replace('\\', '/'); - return winePath.Replace('\\', '/'); - } - - [DllImport("kernel32.dll")] - private static extern int DeviceIoControl(IntPtr hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out IntPtr lpBytesReturned, out IntPtr lpOverlapped); - - [DllImport("kernel32.dll")] - private static extern uint GetCompressedFileSizeW([In, MarshalAs(UnmanagedType.LPWStr)] string lpFileName, - [Out, MarshalAs(UnmanagedType.U4)] out uint lpFileSizeHigh); - - [DllImport("kernel32.dll", SetLastError = true, PreserveSig = true)] - private static extern int GetDiskFreeSpaceW([In, MarshalAs(UnmanagedType.LPWStr)] string lpRootPathName, - out uint lpSectorsPerCluster, out uint lpBytesPerSector, out uint lpNumberOfFreeClusters, - out uint lpTotalNumberOfClusters); - - [DllImport("WoFUtil.dll")] - private static extern int WofIsExternalFile([MarshalAs(UnmanagedType.LPWStr)] string Filepath, out int IsExternalFile, out uint Provider, out WOF_FILE_COMPRESSION_INFO_V1 Info, ref uint BufferLength); - - [DllImport("WofUtil.dll")] - private static extern int WofSetFileDataLocation(IntPtr FileHandle, ulong Provider, IntPtr ExternalFileInfo, ulong Length); - + /// + /// Compressing the given path with BTRFS or NTFS file system. + /// + /// Path of the decompressed/normal file private void CompactFile(string filePath) { var fi = new FileInfo(filePath); if (!fi.Exists) { - _logger.LogDebug("Skipping compaction for missing file {file}", filePath); + _logger.LogTrace("Skip compact: missing {file}", filePath); return; } var fsType = GetFilesystemType(filePath, _dalamudUtilService.IsWine); + _logger.LogTrace("Detected filesystem {fs} for {file} (isWine={wine})", fsType, filePath, _dalamudUtilService.IsWine); var oldSize = fi.Length; - int clusterSize = GetClusterSize(fi); - if (oldSize < Math.Max(clusterSize, 8 * 1024)) + int blockSize = GetBlockSizeForPath(fi.FullName, _logger, _dalamudUtilService.IsWine); + if (oldSize < Math.Max(blockSize, 8 * 1024)) { - _logger.LogDebug("File {file} is smaller than cluster size ({size}), ignoring", filePath, clusterSize); + _logger.LogTrace("Skip compact: {file} < block {block}", filePath, blockSize); return; } - // NTFS Compression. if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) { if (!IsWOFCompactedFile(filePath)) { - _logger.LogDebug("Compacting file to XPRESS8K: {file}", filePath); - var success = WOFCompressFile(filePath); - - if (success) + _logger.LogDebug("NTFS compact XPRESS8K: {file}", filePath); + if (WOFCompressFile(filePath)) { var newSize = GetFileSizeOnDisk(fi); - _logger.LogDebug("Compressed {file} from {orig}b to {comp}b", filePath, oldSize, newSize); + _logger.LogDebug("NTFS compressed {file} {old} -> {new}", filePath, oldSize, newSize); } else { - _logger.LogWarning("NTFS compression failed or not available for {file}", filePath); + _logger.LogWarning("NTFS compression failed or unavailable for {file}", filePath); } - } else { - _logger.LogDebug("File {file} already compressed (NTFS)", filePath); - } - } - - // BTRFS Compression - if (fsType == FilesystemType.Btrfs) - { - if (!IsBtrfsCompressedFile(filePath)) - { - _logger.LogDebug("Attempting btrfs compression for {file}", filePath); - var success = BtrfsCompressFile(filePath); - - if (success) - { - var newSize = GetFileSizeOnDisk(fi); - _logger.LogDebug("Btrfs-compressed {file} from {orig}b to {comp}b", filePath, oldSize, newSize); - } - else - { - _logger.LogWarning("Btrfs compression failed or not available for {file}", filePath); - } - } - else - { - _logger.LogDebug("File {file} already compressed (Btrfs)", filePath); - } - } - } - - private static long GetFileSizeOnDisk(FileInfo fileInfo, Func getClusterSize) - { - int clusterSize = getClusterSize(fileInfo); - if (clusterSize <= 0) - return fileInfo.Length; - - uint low = GetCompressedFileSizeW(fileInfo.FullName, out uint high); - long compressed = ((long)high << 32) | low; - return ((compressed + clusterSize - 1) / clusterSize) * clusterSize; - } - - private static long RunStatGetBlocks(string path) - { - var psi = new ProcessStartInfo("stat", $"-c %b \"{path}\"") - { - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - }; - - using var proc = Process.Start(psi) ?? throw new InvalidOperationException("Could not start stat process"); - var outp = proc.StandardOutput.ReadToEnd(); - var err = proc.StandardError.ReadToEnd(); - proc.WaitForExit(); - if (proc.ExitCode != 0) - { - throw new InvalidOperationException($"stat failed: {err}"); - } - - if (!long.TryParse(outp.Trim(), out var blocks)) - { - throw new InvalidOperationException($"invalid stat output: {outp}"); - } - - return blocks; - } - - private void DecompressFile(string path) - { - _logger.LogDebug("Removing compression from {file}", path); - var fsType = GetFilesystemType(path, _dalamudUtilService.IsWine); - - //NTFS Decompression - if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) - { - try - { - using var fs = new FileStream(path, FileMode.Open); -#pragma warning disable S3869 // "SafeHandle.DangerousGetHandle" should not be called - var hDevice = fs.SafeFileHandle.DangerousGetHandle(); -#pragma warning restore S3869 // "SafeHandle.DangerousGetHandle" should not be called - _ = DeviceIoControl(hDevice, FSCTL_DELETE_EXTERNAL_BACKING, nint.Zero, 0, nint.Zero, 0, out _, out _); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Error decompressing file {path}", path); + _logger.LogTrace("Already NTFS-compressed: {file}", filePath); } return; } - //BTRFS Decompression if (fsType == FilesystemType.Btrfs) { - try + if (!IsBtrfsCompressedFile(filePath)) { - var mountOptions = GetMountOptionsForPath(path); - if (mountOptions.Contains("compress", StringComparison.OrdinalIgnoreCase)) + _logger.LogDebug("Btrfs compress zstd: {file}", filePath); + if (BtrfsCompressFile(filePath)) { - _logger.LogWarning( - "Cannot safely decompress {file}: filesystem mounted with compression ({opts}). Remount with 'compress=no' before running decompression.", - path, mountOptions); - return; - } - - string realPath = path; - bool isWine = _dalamudUtilService?.IsWine ?? false; - if (isWine && IsProbablyWine()) - { - if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) - { - realPath = "/" + path.Substring(3).Replace('\\', '/'); - } - else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) - { - realPath = Path.Combine( - Environment.GetFolderPath(Environment.SpecialFolder.Personal), - path.Substring(3).Replace('\\', '/') - ).Replace('\\', '/'); - } - - _logger.LogTrace("Detected Wine environment. Converted path for decompression: {realPath}", realPath); - } - - string command = $"btrfs filesystem defragment -- \"{realPath}\""; - var psi = new ProcessStartInfo - { - FileName = isWine ? "/bin/bash" : "btrfs", - Arguments = isWine ? $"-c \"{command}\"" : $"filesystem defragment -- \"{realPath}\"", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true, - WorkingDirectory = "/" - }; - - using var proc = Process.Start(psi); - if (proc == null) - { - _logger.LogWarning("Failed to start btrfs defragment for decompression of {file}", path); - return; - } - - var stdout = proc.StandardOutput.ReadToEnd(); - var stderr = proc.StandardError.ReadToEnd(); - proc.WaitForExit(); - - if (proc.ExitCode != 0) - { - _logger.LogWarning("btrfs defragment failed for {file}: {err}", path, stderr); + var newSize = GetFileSizeOnDisk(fi); + _logger.LogDebug("Btrfs compressed {file} {old} -> {new}", filePath, oldSize, newSize); } else { - if (!string.IsNullOrWhiteSpace(stdout)) - _logger.LogDebug("btrfs defragment output for {file}: {out}", path, stdout.Trim()); + _logger.LogWarning("Btrfs compression failed or unavailable for {file}", filePath); + } + } + else + { + _logger.LogTrace("Already Btrfs-compressed: {file}", filePath); + } + return; + } - _logger.LogInformation("Decompressed (rewritten uncompressed) btrfs file: {file}", path); + _logger.LogTrace("Skip compact: unsupported FS for {file}", filePath); + } + + /// + /// Decompressing the given path with BTRFS file system or NTFS file system. + /// + /// Path of the compressed file + private void DecompressFile(string path) + { + _logger.LogDebug("Decompress request: {file}", path); + var fsType = GetFilesystemType(path, _dalamudUtilService.IsWine); + + if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) + { + try + { + bool flowControl = DecompressWOFFile(path, out FileStream fs); + if (!flowControl) + { + return; } } catch (Exception ex) { - _logger.LogWarning(ex, "Error rewriting {file} for decompression", path); + _logger.LogWarning(ex, "NTFS decompress error {file}", path); + } + } + + if (fsType == FilesystemType.Btrfs) + { + try + { + bool flowControl = DecompressBtrfsFile(path); + if (!flowControl) + { + return; + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Btrfs decompress error {file}", path); } } } - private int GetClusterSize(FileInfo fi) + /// + /// Decompress an BTRFS File + /// + /// Path of the compressed file + /// Decompessing state + private bool DecompressBtrfsFile(string path) { try { - if (!fi.Exists) - return -1; - - var root = fi.Directory?.Root.FullName; - if (string.IsNullOrEmpty(root)) - return -1; - - root = root.ToLowerInvariant(); - - if (_clusterSizes.TryGetValue(root, out int cached)) - return cached; - - _logger.LogDebug("Determining cluster/block size for {path} (root: {root})", fi.FullName, root); - - int clusterSize; - - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && !_dalamudUtilService.IsWine) + var opts = GetMountOptionsForPath(path); + if (opts.Contains("compress", StringComparison.OrdinalIgnoreCase)) { - int result = GetDiskFreeSpaceW(root, out uint sectorsPerCluster, out uint bytesPerSector, out _, out _); - - if (result == 0) - { - _logger.LogWarning("GetDiskFreeSpaceW failed for {root}", root); - return -1; - } - - clusterSize = (int)(sectorsPerCluster * bytesPerSector); - } - else - { - clusterSize = GetLinuxBlockSize(root); - if (clusterSize <= 0) - { - _logger.LogWarning("Failed to determine block size for {root}", root); - return -1; - } + _logger.LogWarning("Cannot safely decompress {file}: mount options include compression ({opts})", path, opts); + return false; } - _clusterSizes[root] = clusterSize; - _logger.LogDebug("Determined cluster/block size for {root}: {cluster} bytes", root, clusterSize); - return clusterSize; + string realPath = ToLinuxPathIfWine(path, _dalamudUtilService.IsWine); + var psi = new ProcessStartInfo + { + FileName = _dalamudUtilService.IsWine ? "/bin/bash" : "btrfs", + Arguments = _dalamudUtilService.IsWine + ? $"-c \"btrfs filesystem defragment -- '{EscapeSingle(realPath)}'\"" + : $"filesystem defragment -- \"{realPath}\"", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + WorkingDirectory = "/" + }; + + using var proc = Process.Start(psi); + if (proc == null) + { + _logger.LogWarning("Failed to start btrfs defragment for {file}", path); + return false; + } + + var stdout = proc.StandardOutput.ReadToEnd(); + var stderr = proc.StandardError.ReadToEnd(); + proc.WaitForExit(); + + if (proc.ExitCode != 0) + { + _logger.LogWarning("btrfs defragment failed for {file}: {err}", path, stderr); + return false; + } + + if (!string.IsNullOrWhiteSpace(stdout)) + _logger.LogTrace("btrfs defragment output {file}: {out}", path, stdout.Trim()); + + _logger.LogInformation("Btrfs rewritten uncompressed: {file}", path); } catch (Exception ex) { - _logger.LogWarning(ex, "Error determining cluster size for {file}", fi.FullName); - return -1; + _logger.LogWarning(ex, "Btrfs decompress error {file}", path); } + + return true; } - public static bool UseSafeHandle(SafeHandle handle, Func action) + /// + /// Decompress an NTFS File + /// + /// Path of the compressed file + /// Decompessing state + private bool DecompressWOFFile(string path, out FileStream fs) { - bool addedRef = false; + fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); + var handle = fs.SafeFileHandle; + + if (handle.IsInvalid) + { + _logger.LogWarning("Invalid handle: {file}", path); + return false; + } + + if (!DeviceIoControl(handle, FSCTL_DELETE_EXTERNAL_BACKING, + IntPtr.Zero, 0, IntPtr.Zero, 0, + out _, IntPtr.Zero)) + { + int err = Marshal.GetLastWin32Error(); + + if (err == 342) + { + _logger.LogTrace("File {file} not externally backed (already decompressed)", path); + } + else + { + _logger.LogWarning("DeviceIoControl failed for {file} with Win32 error {err}", path, err); + } + } + else + { + _logger.LogTrace("Successfully decompressed NTFS file {file}", path); + } + + return true; + } + + private static string EscapeSingle(string p) => p.Replace("'", "'\\'", StringComparison.Ordinal); + + private static string ToLinuxPathIfWine(string path, bool isWine) + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + return path; + + if (!IsProbablyWine() && !isWine) + return path; + + string realPath = path; + if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) + realPath = "/" + path[3..].Replace('\\', '/'); + else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) + realPath = Path.Combine(Environment.GetEnvironmentVariable("HOME") ?? "/home", path[3..].Replace('\\', '/')).Replace('\\', '/'); + + return realPath; + } + + /// + /// Compress an WOF File + /// + /// Path of the decompressed/normal file + /// Compessing state + private bool WOFCompressFile(string path) + { + FileStream? fs = null; + int size = Marshal.SizeOf(); + IntPtr efInfoPtr = Marshal.AllocHGlobal(size); + try { - handle.DangerousAddRef(ref addedRef); - IntPtr ptr = handle.DangerousGetHandle(); - return action(ptr); + Marshal.StructureToPtr(_efInfo, efInfoPtr, fDeleteOld: false); + ulong length = (ulong)size; + + const int maxRetries = 3; + + for (int attempt = 0; attempt < maxRetries; attempt++) + { + try + { + fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); + break; + } + catch (IOException) + { + if (attempt == maxRetries - 1) + { + _logger.LogWarning("File still in use after {attempts} attempts, skipping compression for {file}", maxRetries, path); + return false; + } + + int delay = 150 * (attempt + 1); + _logger.LogTrace("File in use, retrying in {delay}ms for {file}", delay, path); + Thread.Sleep(delay); + } + } + + if (fs == null) + { + _logger.LogWarning("Failed to open {file} for compression; skipping", path); + return false; + } + + var handle = fs.SafeFileHandle; + + if (handle.IsInvalid) + { + _logger.LogWarning("Invalid file handle for {file}", path); + return false; + } + + int ret = WofSetFileDataLocation(handle, WOF_PROVIDER_FILE, efInfoPtr, length); + + // 0x80070158 is WOF error whenever compression fails in an non-fatal way. + if (ret != 0 && ret != unchecked((int)0x80070158)) + { + _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X")); + return false; + } + + return true; + } + catch (DllNotFoundException) + { + _logger.LogTrace("WofUtil.dll not available; skipping NTFS compaction for {file}", path); + return false; + } + catch (EntryPointNotFoundException) + { + _logger.LogTrace("WOF entrypoint missing (Wine/older OS); skipping NTFS compaction for {file}", path); + return false; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error compacting file {path}", path); + return false; } finally { - if (addedRef) - handle.DangerousRelease(); + fs?.Dispose(); + + if (efInfoPtr != IntPtr.Zero) + { + Marshal.FreeHGlobal(efInfoPtr); + } } } + /// + /// Checks if an File is compacted with WOF compression + /// + /// Path of the file + /// State of the file private static bool IsWOFCompactedFile(string filePath) { try { uint buf = (uint)Marshal.SizeOf(); - int result = WofIsExternalFile(filePath, out int isExternal, out uint _, out var info, ref buf); - + int result = WofIsExternalFile(filePath, out int isExternal, out _, out var info, ref buf); if (result != 0 || isExternal == 0) return false; - return info.Algorithm == CompressionAlgorithm.XPRESS8K || info.Algorithm == CompressionAlgorithm.XPRESS4K - || info.Algorithm == CompressionAlgorithm.XPRESS16K || info.Algorithm == CompressionAlgorithm.LZX; + return info.Algorithm == (int)CompressionAlgorithm.XPRESS8K + || info.Algorithm == (int)CompressionAlgorithm.XPRESS4K + || info.Algorithm == (int)CompressionAlgorithm.XPRESS16K + || info.Algorithm == (int)CompressionAlgorithm.LZX + || info.Algorithm == (int)CompressionAlgorithm.LZNT1 + || info.Algorithm == (int)CompressionAlgorithm.NO_COMPRESSION; } - catch (DllNotFoundException) + catch { - // WofUtil.dll not available - return false; - } - catch (EntryPointNotFoundException) - { - // Running under Wine or non-NTFS systems - return false; - } - catch (Exception) - { - // Exception happened return false; } } + /// + /// Checks if an File is compacted with Btrfs compression + /// + /// Path of the file + /// State of the file private bool IsBtrfsCompressedFile(string path) { try { bool isWine = _dalamudUtilService?.IsWine ?? false; - string realPath = path; + string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; - if (isWine && IsProbablyWine()) - { - if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) - { - realPath = "/" + path.Substring(3).Replace('\\', '/'); - } - else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) - { - realPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Personal), path.Substring(3).Replace('\\', '/')).Replace('\\', '/'); - } - - _logger.LogTrace("Detected Wine environment. Converted path for filefrag: {realPath}", realPath); - } - - string command = $"filefrag -v -- \"{realPath}\""; var psi = new ProcessStartInfo { FileName = isWine ? "/bin/bash" : "filefrag", - Arguments = isWine ? $"-c \"{command}\"" : $"-v -- \"{realPath}\"", + Arguments = isWine + ? $"-c \"filefrag -v '{EscapeSingle(realPath)}'\"" + : $"-v \"{realPath}\"", RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, @@ -562,109 +547,116 @@ public sealed class FileCompactor : IDisposable return false; } - string output = proc.StandardOutput.ReadToEnd(); + string stdout = proc.StandardOutput.ReadToEnd(); string stderr = proc.StandardError.ReadToEnd(); proc.WaitForExit(); - if (proc.ExitCode != 0) + if (proc.ExitCode != 0 && !string.IsNullOrWhiteSpace(stderr)) { - _logger.LogDebug("filefrag exited with {code} for {file}. stderr: {stderr}", - proc.ExitCode, path, stderr); - return false; + _logger.LogTrace("filefrag exited with code {code}: {stderr}", proc.ExitCode, stderr); } - bool compressed = output.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase); + bool compressed = stdout.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase); _logger.LogTrace("Btrfs compression check for {file}: {compressed}", path, compressed); return compressed; } catch (Exception ex) { - _logger.LogDebug(ex, "Failed to detect btrfs compression for {file}", path); + _logger.LogDebug(ex, "Failed to detect Btrfs compression for {file}", path); return false; } } - private bool WOFCompressFile(string path) - { - var efInfoPtr = Marshal.AllocHGlobal(Marshal.SizeOf(_efInfo)); - Marshal.StructureToPtr(_efInfo, efInfoPtr, fDeleteOld: true); - ulong length = (ulong)Marshal.SizeOf(_efInfo); - try - { - using var fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.None); - var handle = fs.SafeFileHandle; - - if (handle.IsInvalid) - { - _logger.LogWarning("Invalid file handle to {file}", path); - return false; - } - - return UseSafeHandle(handle, hFile => - { - int ret = WofSetFileDataLocation(hFile, WOF_PROVIDER_FILE, efInfoPtr, length); - if (ret != 0 && ret != unchecked((int)0x80070158)) - { - _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X")); - return false; - } - return true; - }); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Error compacting file {path}", path); - return false; - } - finally - { - Marshal.FreeHGlobal(efInfoPtr); - } - } - + /// + /// Compress an Btrfs File + /// + /// Path of the decompressed/normal file + /// Compessing state private bool BtrfsCompressFile(string path) { try { - string realPath = path; - if (_dalamudUtilService.IsWine && IsProbablyWine()) + bool isWine = _dalamudUtilService?.IsWine ?? false; + string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; + + if (isWine && IsProbablyWine()) { - realPath = ConvertWinePathToLinux(path); - _logger.LogTrace("Detected Wine environment, remapped path: {realPath}", realPath); + if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) + { + realPath = "/" + path[3..].Replace('\\', '/'); + } + else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) + { + string linuxHome = Environment.GetEnvironmentVariable("HOME") ?? "/home"; + realPath = Path.Combine(linuxHome, path[3..].Replace('\\', '/')).Replace('\\', '/'); + } + + _logger.LogTrace("Detected Wine environment. Converted path for compression: {realPath}", realPath); } - if (!File.Exists("/usr/bin/btrfs") && !File.Exists("/bin/btrfs")) + const int maxRetries = 3; + for (int attempt = 0; attempt < maxRetries; attempt++) { - _logger.LogWarning("Skipping Btrfs compression — btrfs binary not found"); - return false; + try + { + using var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + break; + } + catch (IOException) + { + if (attempt == maxRetries - 1) + { + _logger.LogWarning("File still in use after {attempts} attempts; skipping btrfs compression for {file}", maxRetries, path); + return false; + } + + int delay = 150 * (attempt + 1); + _logger.LogTrace("File busy, retrying in {delay}ms for {file}", delay, path); + Thread.Sleep(delay); + } } - var psi = new ProcessStartInfo("btrfs", $"filesystem defragment -czstd -- \"{realPath}\"") + string command = $"btrfs filesystem defragment -czstd -- \"{realPath}\""; + var psi = new ProcessStartInfo { + FileName = isWine ? "/bin/bash" : "btrfs", + Arguments = isWine ? $"-c \"{command}\"" : $"filesystem defragment -czstd -- \"{realPath}\"", RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, - CreateNoWindow = true + CreateNoWindow = true, + WorkingDirectory = "/" }; using var proc = Process.Start(psi); if (proc == null) { - _logger.LogWarning("Failed to start btrfs process for {file}", realPath); + _logger.LogWarning("Failed to start btrfs defragment for compression of {file}", path); return false; } - var stdout = proc.StandardOutput.ReadToEnd(); - var stderr = proc.StandardError.ReadToEnd(); - proc.WaitForExit(); + string stdout = proc.StandardOutput.ReadToEnd(); + string stderr = proc.StandardError.ReadToEnd(); + + try + { + proc.WaitForExit(); + } + catch (Exception ex) + { + _logger.LogTrace(ex, "Process.WaitForExit threw under Wine for {file}", path); + } if (proc.ExitCode != 0) { - _logger.LogWarning("btrfs defrag returned {code} for {file}: {err}", proc.ExitCode, realPath, stderr); + _logger.LogWarning("btrfs defragment failed for {file}: {stderr}", path, stderr); return false; } - _logger.LogDebug("btrfs output: {out}", stdout); + if (!string.IsNullOrWhiteSpace(stdout)) + _logger.LogTrace("btrfs defragment output for {file}: {stdout}", path, stdout.Trim()); + + _logger.LogInformation("Compressed btrfs file successfully: {file}", path); return true; } catch (Exception ex) @@ -674,22 +666,15 @@ public sealed class FileCompactor : IDisposable } } - private struct WOF_FILE_COMPRESSION_INFO_V1 - { - public CompressionAlgorithm Algorithm; - public ulong Flags; - } - private void EnqueueCompaction(string filePath) { if (!_pendingCompactions.TryAdd(filePath, 0)) return; - - var fsType = GetFilesystemType(filePath, _dalamudUtilService.IsWine); + var fsType = GetFilesystemType(filePath, _dalamudUtilService.IsWine); if (fsType != FilesystemType.NTFS && fsType != FilesystemType.Btrfs) { - _logger.LogTrace("Skipping compaction enqueue for unsupported filesystem {fs} ({file})", fsType, filePath); + _logger.LogTrace("Skip enqueue (unsupported fs) {fs} {file}", fsType, filePath); _pendingCompactions.TryRemove(filePath, out _); return; } @@ -697,11 +682,7 @@ public sealed class FileCompactor : IDisposable if (!_compactionQueue.Writer.TryWrite(filePath)) { _pendingCompactions.TryRemove(filePath, out _); - _logger.LogDebug("Failed to enqueue compaction job for {file}", filePath); - } - else - { - _logger.LogTrace("Queued compaction job for {file} (fs={fs})", filePath, fsType); + _logger.LogDebug("Failed to enqueue compaction {file}", filePath); } } @@ -727,13 +708,13 @@ public sealed class FileCompactor : IDisposable if (!File.Exists(filePath)) { - _logger.LogTrace("Skipping compaction for missing file {file}", filePath); + _logger.LogTrace("Skip compact (missing) {file}", filePath); continue; } CompactFile(filePath); } - catch (OperationCanceledException) + catch (OperationCanceledException) { return; } @@ -750,9 +731,46 @@ public sealed class FileCompactor : IDisposable } catch (OperationCanceledException) { - _logger.LogDebug("Queue has been cancelled by token"); + _logger.LogDebug("Compaction queue cancelled"); } } - private static bool IsProbablyWine() => Environment.GetEnvironmentVariable("WINELOADERNOEXEC") != null || Environment.GetEnvironmentVariable("WINEDLLPATH") != null || Directory.Exists("/proc/self") && File.Exists("/proc/mounts"); + [StructLayout(LayoutKind.Sequential, Pack = 1)] + private struct WOF_FILE_COMPRESSION_INFO_V1 + { + public int Algorithm; + public ulong Flags; + } + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool DeviceIoControl(SafeFileHandle hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out uint lpBytesReturned, IntPtr lpOverlapped); + + [DllImport("kernel32.dll")] + private static extern uint GetCompressedFileSizeW([In, MarshalAs(UnmanagedType.LPWStr)] string lpFileName, [Out, MarshalAs(UnmanagedType.U4)] out uint lpFileSizeHigh); + + [DllImport("WofUtil.dll")] + private static extern int WofIsExternalFile([MarshalAs(UnmanagedType.LPWStr)] string Filepath, out int IsExternalFile, out uint Provider, out WOF_FILE_COMPRESSION_INFO_V1 Info, ref uint BufferLength); + + [DllImport("WofUtil.dll", SetLastError = true)] + private static extern int WofSetFileDataLocation(SafeFileHandle FileHandle, ulong Provider, IntPtr ExternalFileInfo, ulong Length); + + public void Dispose() + { + _compactionQueue.Writer.TryComplete(); + _compactionCts.Cancel(); + try + { + _compactionWorker.Wait(TimeSpan.FromSeconds(5)); + } + catch + { + //ignore on catch ^^ + } + finally + { + _compactionCts.Dispose(); + } + + GC.SuppressFinalize(this); + } } diff --git a/LightlessSync/Utils/FileSystemHelper.cs b/LightlessSync/Utils/FileSystemHelper.cs index 4bbfc75..af4c98b 100644 --- a/LightlessSync/Utils/FileSystemHelper.cs +++ b/LightlessSync/Utils/FileSystemHelper.cs @@ -1,4 +1,6 @@ -using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; +using System.Collections.Concurrent; +using System.Diagnostics; using System.Runtime.InteropServices; namespace LightlessSync.Utils @@ -20,6 +22,8 @@ namespace LightlessSync.Utils } private const string _mountPath = "/proc/mounts"; + private const int _defaultBlockSize = 4096; + private static readonly Dictionary _blockSizeCache = new(StringComparer.OrdinalIgnoreCase); private static readonly ConcurrentDictionary _filesystemTypeCache = new(StringComparer.OrdinalIgnoreCase); public static FilesystemType GetFilesystemType(string filePath, bool isWine = false) @@ -27,6 +31,7 @@ namespace LightlessSync.Utils try { string rootPath; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && (!IsProbablyWine() || !isWine)) { var info = new FileInfo(filePath); @@ -49,6 +54,7 @@ namespace LightlessSync.Utils { var root = new DriveInfo(rootPath); var format = root.DriveFormat?.ToUpperInvariant() ?? string.Empty; + detected = format switch { "NTFS" => FilesystemType.NTFS, @@ -62,10 +68,28 @@ namespace LightlessSync.Utils detected = GetLinuxFilesystemType(filePath); } + if (isWine || IsProbablyWine()) + { + switch (detected) + { + case FilesystemType.NTFS: + case FilesystemType.Unknown: + { + var linuxDetected = GetLinuxFilesystemType(filePath); + if (linuxDetected != FilesystemType.Unknown) + { + detected = linuxDetected; + } + + break; + } + } + } + _filesystemTypeCache[rootPath] = detected; return detected; } - catch (Exception ex) + catch { return FilesystemType.Unknown; } @@ -175,7 +199,84 @@ namespace LightlessSync.Utils } } + public static int GetBlockSizeForPath(string path, ILogger? logger = null, bool isWine = false) + { + try + { + if (string.IsNullOrWhiteSpace(path)) + return _defaultBlockSize; + + var fi = new FileInfo(path); + if (!fi.Exists) + return _defaultBlockSize; + + var root = fi.Directory?.Root.FullName.ToLowerInvariant() ?? "/"; + if (_blockSizeCache.TryGetValue(root, out int cached)) + return cached; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && !isWine) + { + int result = GetDiskFreeSpaceW(root, + out uint sectorsPerCluster, + out uint bytesPerSector, + out _, + out _); + + if (result == 0) + { + logger?.LogWarning("Failed to determine block size for {root}", root); + return _defaultBlockSize; + } + + int clusterSize = (int)(sectorsPerCluster * bytesPerSector); + _blockSizeCache[root] = clusterSize; + logger?.LogTrace("NTFS cluster size for {root}: {cluster}", root, clusterSize); + return clusterSize; + } + + string realPath = fi.FullName; + if (isWine && realPath.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) + { + realPath = "/" + realPath.Substring(3).Replace('\\', '/'); + } + + var psi = new ProcessStartInfo + { + FileName = "/bin/bash", + Arguments = $"-c \"stat -f -c %s '{realPath.Replace("'", "'\\''")}'\"", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + WorkingDirectory = "/" + }; + + using var proc = Process.Start(psi); + string stdout = proc?.StandardOutput.ReadToEnd().Trim() ?? ""; + proc?.WaitForExit(); + + if (int.TryParse(stdout, out int blockSize) && blockSize > 0) + { + _blockSizeCache[root] = blockSize; + logger?.LogTrace("Filesystem block size via stat for {root}: {block}", root, blockSize); + return blockSize; + } + + logger?.LogTrace("stat did not return valid block size for {file}, output: {out}", fi.FullName, stdout); + _blockSizeCache[root] = _defaultBlockSize; + return _defaultBlockSize; + } + catch (Exception ex) + { + logger?.LogTrace(ex, "Error determining block size for {path}", path); + return _defaultBlockSize; + } + } + + [DllImport("kernel32.dll", SetLastError = true, PreserveSig = true)] + private static extern int GetDiskFreeSpaceW([In, MarshalAs(UnmanagedType.LPWStr)] string lpRootPathName, out uint lpSectorsPerCluster, out uint lpBytesPerSector, out uint lpNumberOfFreeClusters, out uint lpTotalNumberOfClusters); + //Extra check on - private static bool IsProbablyWine() => Environment.GetEnvironmentVariable("WINELOADERNOEXEC") != null || Environment.GetEnvironmentVariable("WINEDLLPATH") != null || Directory.Exists("/proc/self") && File.Exists("/proc/mounts"); + public static bool IsProbablyWine() => Environment.GetEnvironmentVariable("WINELOADERNOEXEC") != null || Environment.GetEnvironmentVariable("WINEDLLPATH") != null || Directory.Exists("/proc/self") && File.Exists("/proc/mounts"); } } From 0af2a6134be75a175c810449320a76f23c0c1869 Mon Sep 17 00:00:00 2001 From: cake Date: Sat, 1 Nov 2025 00:09:54 +0100 Subject: [PATCH 20/28] Changed warning text for Brio --- LightlessSync/UI/CharaDataHubUi.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LightlessSync/UI/CharaDataHubUi.cs b/LightlessSync/UI/CharaDataHubUi.cs index 9016e6c..51723b9 100644 --- a/LightlessSync/UI/CharaDataHubUi.cs +++ b/LightlessSync/UI/CharaDataHubUi.cs @@ -170,7 +170,7 @@ internal sealed partial class CharaDataHubUi : WindowMediatorSubscriberBase if (!_charaDataManager.BrioAvailable) { ImGuiHelpers.ScaledDummy(3); - UiSharedService.DrawGroupedCenteredColorText("To utilize any features related to posing or spawning characters you require to have Brio installed.", ImGuiColors.DalamudRed); + UiSharedService.DrawGroupedCenteredColorText("To utilize any features related to posing or spawning characters, you are required to have Brio installed.", ImGuiColors.DalamudRed); UiSharedService.DistanceSeparator(); } From d4dca455ba8d8abe9323ef4081f9938e4f7f8e98 Mon Sep 17 00:00:00 2001 From: cake Date: Mon, 3 Nov 2025 18:54:35 +0100 Subject: [PATCH 21/28] Clean-up, added extra checks on linux in cache monitor, documentation added --- LightlessSync/FileCache/CacheMonitor.cs | 91 ++++-- LightlessSync/FileCache/FileCompactor.cs | 372 ++++++++++++----------- 2 files changed, 260 insertions(+), 203 deletions(-) diff --git a/LightlessSync/FileCache/CacheMonitor.cs b/LightlessSync/FileCache/CacheMonitor.cs index 85fb30e..486e11e 100644 --- a/LightlessSync/FileCache/CacheMonitor.cs +++ b/LightlessSync/FileCache/CacheMonitor.cs @@ -403,57 +403,94 @@ public sealed class CacheMonitor : DisposableMediatorSubscriberBase public void RecalculateFileCacheSize(CancellationToken token) { - if (string.IsNullOrEmpty(_configService.Current.CacheFolder) || !Directory.Exists(_configService.Current.CacheFolder)) + if (string.IsNullOrEmpty(_configService.Current.CacheFolder) || + !Directory.Exists(_configService.Current.CacheFolder)) { FileCacheSize = 0; return; } - FileCacheSize = -1; - - var drive = DriveInfo.GetDrives().FirstOrDefault(d => _configService.Current.CacheFolder.StartsWith(d.Name, StringComparison.Ordinal)); - if (drive == null) - { - return; - } + FileCacheSize = -1; + bool isWine = _dalamudUtil?.IsWine ?? false; try { - FileCacheDriveFree = drive.AvailableFreeSpace; + var drive = DriveInfo.GetDrives() + .FirstOrDefault(d => _configService.Current.CacheFolder + .StartsWith(d.Name, StringComparison.OrdinalIgnoreCase)); + + if (drive != null) + FileCacheDriveFree = drive.AvailableFreeSpace; } catch (Exception ex) { - Logger.LogWarning(ex, "Could not determine drive size for Storage Folder {folder}", _configService.Current.CacheFolder); + Logger.LogWarning(ex, "Could not determine drive size for storage folder {folder}", _configService.Current.CacheFolder); } - var files = Directory.EnumerateFiles(_configService.Current.CacheFolder).Select(f => new FileInfo(f)) - .OrderBy(f => f.LastAccessTime).ToList(); - FileCacheSize = files - .Sum(f => - { - token.ThrowIfCancellationRequested(); + var files = Directory.EnumerateFiles(_configService.Current.CacheFolder) + .Select(f => new FileInfo(f)) + .OrderBy(f => f.LastAccessTime) + .ToList(); - try + long totalSize = 0; + + foreach (var f in files) + { + token.ThrowIfCancellationRequested(); + + try + { + long size = 0; + + if (!isWine) { - return _fileCompactor.GetFileSizeOnDisk(f); + try + { + size = _fileCompactor.GetFileSizeOnDisk(f); + } + catch (Exception ex) + { + Logger.LogTrace(ex, "GetFileSizeOnDisk failed for {file}, using fallback length", f.FullName); + size = f.Length; + } } - catch + else { - return 0; + size = f.Length; } - }); + + totalSize += size; + } + catch (Exception ex) + { + Logger.LogTrace(ex, "Error getting size for {file}", f.FullName); + } + } + + FileCacheSize = totalSize; var maxCacheInBytes = (long)(_configService.Current.MaxLocalCacheInGiB * 1024d * 1024d * 1024d); - - if (FileCacheSize < maxCacheInBytes) return; + if (FileCacheSize < maxCacheInBytes) + return; var maxCacheBuffer = maxCacheInBytes * 0.05d; - while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer) + + while (FileCacheSize > maxCacheInBytes - (long)maxCacheBuffer && files.Count > 0) { var oldestFile = files[0]; - FileCacheSize -= _fileCompactor.GetFileSizeOnDisk(oldestFile); - File.Delete(oldestFile.FullName); - files.Remove(oldestFile); + + try + { + long fileSize = oldestFile.Length; + File.Delete(oldestFile.FullName); + FileCacheSize -= fileSize; + } + catch (Exception ex) + { + Logger.LogTrace(ex, "Failed to delete old file {file}", oldestFile.FullName); + } + + files.RemoveAt(0); } } diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index a917b59..b9c2118 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Win32.SafeHandles; using System.Collections.Concurrent; using System.Diagnostics; +using System.IO; using System.Runtime.InteropServices; using System.Threading.Channels; using static LightlessSync.Utils.FileSystemHelper; @@ -14,6 +15,7 @@ public sealed class FileCompactor : IDisposable { public const uint FSCTL_DELETE_EXTERNAL_BACKING = 0x90314U; public const ulong WOF_PROVIDER_FILE = 2UL; + public const int _maxRetries = 3; private readonly ConcurrentDictionary _pendingCompactions; private readonly ILogger _logger; @@ -29,6 +31,14 @@ public sealed class FileCompactor : IDisposable Algorithm = (int)CompressionAlgorithm.XPRESS8K, Flags = 0 }; + + [StructLayout(LayoutKind.Sequential, Pack = 1)] + private struct WOF_FILE_COMPRESSION_INFO_V1 + { + public int Algorithm; + public ulong Flags; + } + private enum CompressionAlgorithm { NO_COMPRESSION = -2, @@ -58,6 +68,10 @@ public sealed class FileCompactor : IDisposable public bool MassCompactRunning { get; private set; } public string Progress { get; private set; } = string.Empty; + /// + /// Compact the storage of the Cache Folder + /// + /// Used to check if files needs to be compressed public void CompactStorage(bool compress) { MassCompactRunning = true; @@ -74,6 +88,7 @@ public sealed class FileCompactor : IDisposable try { + // Compress or decompress files if (compress) CompactFile(file); else @@ -135,25 +150,19 @@ public sealed class FileCompactor : IDisposable { try { - var realPath = fileInfo.FullName.Replace("\"", "\\\"", StringComparison.Ordinal); - var psi = new ProcessStartInfo("stat", $"-c %b \"{realPath}\"") - { - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - }; + bool isWine = _dalamudUtilService?.IsWine ?? false; + string realPath = isWine ? ToLinuxPathIfWine(fileInfo.FullName, isWine) : fileInfo.FullName; - using var proc = Process.Start(psi) ?? throw new InvalidOperationException("Could not start stat"); - var outp = proc.StandardOutput.ReadToEnd(); - var err = proc.StandardError.ReadToEnd(); - proc.WaitForExit(); + var fileName = "stat"; + var arguments = $"-c %b \"{realPath}\""; - if (proc.ExitCode != 0) - throw new InvalidOperationException($"stat failed: {err}"); + (bool processControl, bool success) = StartProcessInfo(realPath, fileName, arguments, out Process? proc, out string stdout); - if (!long.TryParse(outp.Trim(), out var blocks)) - throw new InvalidOperationException($"invalid stat output: {outp}"); + if (!processControl && !success) + throw new InvalidOperationException($"stat failed: {proc}"); + + if (!long.TryParse(stdout.Trim(), out var blocks)) + throw new InvalidOperationException($"invalid stat output: {stdout}"); // st_blocks are always 512-byte on Linux enviroment. return blocks * 512L; @@ -287,57 +296,57 @@ public sealed class FileCompactor : IDisposable /// Decompessing state private bool DecompressBtrfsFile(string path) { + var fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); + try { - var opts = GetMountOptionsForPath(path); - if (opts.Contains("compress", StringComparison.OrdinalIgnoreCase)) + bool isWine = _dalamudUtilService?.IsWine ?? false; + string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; + + var mountOptions = GetMountOptionsForPath(realPath); + if (mountOptions.Contains("compress", StringComparison.OrdinalIgnoreCase)) { - _logger.LogWarning("Cannot safely decompress {file}: mount options include compression ({opts})", path, opts); + _logger.LogWarning( + "Cannot safely decompress {file}: filesystem mounted with compression ({opts}). " + + "Remount with 'compress=no' before running decompression.", + realPath, mountOptions); return false; } - string realPath = ToLinuxPathIfWine(path, _dalamudUtilService.IsWine); - var psi = new ProcessStartInfo + if (!IsBtrfsCompressedFile(realPath)) { - FileName = _dalamudUtilService.IsWine ? "/bin/bash" : "btrfs", - Arguments = _dalamudUtilService.IsWine - ? $"-c \"btrfs filesystem defragment -- '{EscapeSingle(realPath)}'\"" - : $"filesystem defragment -- \"{realPath}\"", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true, - WorkingDirectory = "/" - }; - - using var proc = Process.Start(psi); - if (proc == null) - { - _logger.LogWarning("Failed to start btrfs defragment for {file}", path); - return false; + _logger.LogTrace("File {file} is not compressed, skipping decompression.", realPath); + return true; } - var stdout = proc.StandardOutput.ReadToEnd(); - var stderr = proc.StandardError.ReadToEnd(); - proc.WaitForExit(); + (bool flowControl, bool value) = FileStreamOpening(realPath, ref fs); - if (proc.ExitCode != 0) + if (!flowControl) { - _logger.LogWarning("btrfs defragment failed for {file}: {err}", path, stderr); - return false; + return value; + } + + string fileName = isWine ? "/bin/bash" : "btrfs"; + string command = isWine ? $"-c \"filesystem defragment -- \"{realPath}\"\"" : $"filesystem defragment -- \"{realPath}\""; + + (bool processControl, bool success) = StartProcessInfo(realPath, fileName, command, out Process? proc, out string stdout); + if (!processControl && !success) + { + return value; } if (!string.IsNullOrWhiteSpace(stdout)) - _logger.LogTrace("btrfs defragment output {file}: {out}", path, stdout.Trim()); + _logger.LogTrace("btrfs defragment output for {file}: {stdout}", realPath, stdout.Trim()); - _logger.LogInformation("Btrfs rewritten uncompressed: {file}", path); + _logger.LogInformation("Decompressed btrfs file successfully: {file}", realPath); + + return true; } catch (Exception ex) { - _logger.LogWarning(ex, "Btrfs decompress error {file}", path); + _logger.LogWarning(ex, "Error rewriting {file} for Btrfs decompression", path); + return false; } - - return true; } /// @@ -379,23 +388,25 @@ public sealed class FileCompactor : IDisposable return true; } - private static string EscapeSingle(string p) => p.Replace("'", "'\\'", StringComparison.Ordinal); - - private static string ToLinuxPathIfWine(string path, bool isWine) + /// + /// Converts to Linux Path if its using Wine (diferent pathing system in Wine) + /// + /// Path that has to be converted + /// Extra check if using the wine enviroment + /// Converted path to be used in Linux + private string ToLinuxPathIfWine(string path, bool isWine) { - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - return path; - if (!IsProbablyWine() && !isWine) return path; - string realPath = path; + string linuxPath = path; if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) - realPath = "/" + path[3..].Replace('\\', '/'); + linuxPath = "/" + path[3..].Replace('\\', '/'); else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) - realPath = Path.Combine(Environment.GetEnvironmentVariable("HOME") ?? "/home", path[3..].Replace('\\', '/')).Replace('\\', '/'); + linuxPath = Path.Combine(Environment.GetEnvironmentVariable("HOME") ?? "/home", path[3..].Replace('\\', '/')).Replace('\\', '/'); - return realPath; + _logger.LogTrace("Detected Wine environment. Converted path for compression: {realPath}", linuxPath); + return linuxPath; } /// @@ -414,27 +425,11 @@ public sealed class FileCompactor : IDisposable Marshal.StructureToPtr(_efInfo, efInfoPtr, fDeleteOld: false); ulong length = (ulong)size; - const int maxRetries = 3; + (bool flowControl, bool value) = FileStreamOpening(path, ref fs); - for (int attempt = 0; attempt < maxRetries; attempt++) + if (!flowControl) { - try - { - fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); - break; - } - catch (IOException) - { - if (attempt == maxRetries - 1) - { - _logger.LogWarning("File still in use after {attempts} attempts, skipping compression for {file}", maxRetries, path); - return false; - } - - int delay = 150 * (attempt + 1); - _logger.LogTrace("File in use, retrying in {delay}ms for {file}", delay, path); - Thread.Sleep(delay); - } + return value; } if (fs == null) @@ -462,14 +457,14 @@ public sealed class FileCompactor : IDisposable return true; } - catch (DllNotFoundException) + catch (DllNotFoundException ex) { - _logger.LogTrace("WofUtil.dll not available; skipping NTFS compaction for {file}", path); + _logger.LogTrace(ex, "WofUtil.dll not available, this DLL is needed for compression; skipping NTFS compaction for {file}", path); return false; } - catch (EntryPointNotFoundException) + catch (EntryPointNotFoundException ex) { - _logger.LogTrace("WOF entrypoint missing (Wine/older OS); skipping NTFS compaction for {file}", path); + _logger.LogTrace(ex, "WOF entrypoint missing (Wine/older OS); skipping NTFS compaction for {file}", path); return false; } catch (Exception ex) @@ -527,37 +522,24 @@ public sealed class FileCompactor : IDisposable bool isWine = _dalamudUtilService?.IsWine ?? false; string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; - var psi = new ProcessStartInfo - { - FileName = isWine ? "/bin/bash" : "filefrag", - Arguments = isWine - ? $"-c \"filefrag -v '{EscapeSingle(realPath)}'\"" - : $"-v \"{realPath}\"", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true, - WorkingDirectory = "/" - }; + var fi = new FileInfo(realPath); - using var proc = Process.Start(psi); - if (proc == null) + if (fi == null) { - _logger.LogWarning("Failed to start filefrag for {file}", path); + _logger.LogWarning("Failed to open {file} for checking on compression; skipping", realPath); return false; } - string stdout = proc.StandardOutput.ReadToEnd(); - string stderr = proc.StandardError.ReadToEnd(); - proc.WaitForExit(); - - if (proc.ExitCode != 0 && !string.IsNullOrWhiteSpace(stderr)) + string fileName = isWine ? "/bin/bash" : "filefrag"; + string command = isWine ? $"-c \"filefrag -v '{EscapeSingle(realPath)}'\"" : $"-v \"{realPath}\""; + (bool processControl, bool success) = StartProcessInfo(realPath, fileName, command, out Process? proc, out string stdout); + if (!processControl && !success) { - _logger.LogTrace("filefrag exited with code {code}: {stderr}", proc.ExitCode, stderr); + return success; } bool compressed = stdout.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase); - _logger.LogTrace("Btrfs compression check for {file}: {compressed}", path, compressed); + _logger.LogTrace("Btrfs compression check for {file}: {compressed}", realPath, compressed); return compressed; } catch (Exception ex) @@ -574,89 +556,56 @@ public sealed class FileCompactor : IDisposable /// Compessing state private bool BtrfsCompressFile(string path) { + FileStream? fs = null; + try { bool isWine = _dalamudUtilService?.IsWine ?? false; string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; - if (isWine && IsProbablyWine()) + var fi = new FileInfo(realPath); + + if (fi == null) { - if (path.StartsWith("Z:\\", StringComparison.OrdinalIgnoreCase)) - { - realPath = "/" + path[3..].Replace('\\', '/'); - } - else if (path.StartsWith("C:\\", StringComparison.OrdinalIgnoreCase)) - { - string linuxHome = Environment.GetEnvironmentVariable("HOME") ?? "/home"; - realPath = Path.Combine(linuxHome, path[3..].Replace('\\', '/')).Replace('\\', '/'); - } - - _logger.LogTrace("Detected Wine environment. Converted path for compression: {realPath}", realPath); - } - - const int maxRetries = 3; - for (int attempt = 0; attempt < maxRetries; attempt++) - { - try - { - using var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); - break; - } - catch (IOException) - { - if (attempt == maxRetries - 1) - { - _logger.LogWarning("File still in use after {attempts} attempts; skipping btrfs compression for {file}", maxRetries, path); - return false; - } - - int delay = 150 * (attempt + 1); - _logger.LogTrace("File busy, retrying in {delay}ms for {file}", delay, path); - Thread.Sleep(delay); - } - } - - string command = $"btrfs filesystem defragment -czstd -- \"{realPath}\""; - var psi = new ProcessStartInfo - { - FileName = isWine ? "/bin/bash" : "btrfs", - Arguments = isWine ? $"-c \"{command}\"" : $"filesystem defragment -czstd -- \"{realPath}\"", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true, - WorkingDirectory = "/" - }; - - using var proc = Process.Start(psi); - if (proc == null) - { - _logger.LogWarning("Failed to start btrfs defragment for compression of {file}", path); + _logger.LogWarning("Failed to open {file} for compression; skipping", realPath); return false; } - string stdout = proc.StandardOutput.ReadToEnd(); - string stderr = proc.StandardError.ReadToEnd(); - - try - { - proc.WaitForExit(); - } - catch (Exception ex) + //Skipping small files to make compression a bit faster, its not that effective on small files. + int blockSize = GetBlockSizeForPath(realPath, _logger, isWine); + if (fi.Length < Math.Max(blockSize * 2, 128 * 1024)) { - _logger.LogTrace(ex, "Process.WaitForExit threw under Wine for {file}", path); + _logger.LogTrace("Skipping Btrfs compression for small file {file} ({size} bytes)", realPath, fi.Length); + return true; } - if (proc.ExitCode != 0) + if (IsBtrfsCompressedFile(realPath)) { - _logger.LogWarning("btrfs defragment failed for {file}: {stderr}", path, stderr); - return false; + _logger.LogTrace("File {file} already compressed (Btrfs), skipping file", realPath); + return true; + } + + (bool flowControl, bool value) = FileStreamOpening(realPath, ref fs); + + if (!flowControl) + { + return value; + } + + string fileName = isWine ? "/bin/bash" : "btrfs"; + string command = isWine ? $"-c \"btrfs filesystem defragment -czstd:1 -- \"{realPath}\"\"" : $"btrfs filesystem defragment -czstd:1 -- \"{realPath}\""; + + (bool processControl, bool success) = StartProcessInfo(realPath, fileName, command, out Process? proc, out string stdout); + if (!processControl && !success) + { + return value; } if (!string.IsNullOrWhiteSpace(stdout)) - _logger.LogTrace("btrfs defragment output for {file}: {stdout}", path, stdout.Trim()); + _logger.LogTrace("btrfs defragment output for {file}: {stdout}", realPath, stdout.Trim()); + + _logger.LogInformation("Compressed btrfs file successfully: {file}", realPath); - _logger.LogInformation("Compressed btrfs file successfully: {file}", path); return true; } catch (Exception ex) @@ -666,6 +615,84 @@ public sealed class FileCompactor : IDisposable } } + + /// + /// Trying opening file stream in certain amount of tries. + /// + /// Path where the file is located + /// Filestream used for the function + /// State of the filestream opening + private (bool flowControl, bool value) FileStreamOpening(string path, ref FileStream? fs) + { + for (int attempt = 0; attempt < _maxRetries; attempt++) + { + try + { + fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); + break; + } + catch (IOException) + { + if (attempt == _maxRetries - 1) + { + _logger.LogWarning("File still in use after {attempts} attempts, skipping compression for {file}", _maxRetries, path); + return (flowControl: false, value: false); + } + + int delay = 150 * (attempt + 1); + _logger.LogTrace("File in use, retrying in {delay}ms for {file}", delay, path); + Thread.Sleep(delay); + } + } + + return (flowControl: true, value: default); + } + + /// + /// Starts an process with given Filename and Arguments + /// + /// Path you want to use for the process (Compression is using these) + /// File of the command + /// Arguments used for the command + /// Returns process of the given command + /// Returns output of the given command + /// Returns if the process been done succesfully or not + private (bool processControl, bool success) StartProcessInfo(string path, string fileName, string arguments, out Process? proc, out string stdout) + { + var psi = new ProcessStartInfo + { + FileName = fileName, + Arguments = arguments, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + WorkingDirectory = "/" + }; + proc = Process.Start(psi); + + if (proc == null) + { + _logger.LogWarning("Failed to start {arguments} for {file}", arguments, path); + stdout = string.Empty; + return (processControl: false, success: false); + } + + stdout = proc.StandardOutput.ReadToEnd(); + string stderr = proc.StandardError.ReadToEnd(); + proc.WaitForExit(); + + if (proc.ExitCode != 0 && !string.IsNullOrWhiteSpace(stderr)) + { + _logger.LogTrace("{arguments} exited with code {code}: {stderr}", arguments, proc.ExitCode, stderr); + return (processControl: false, success: false); + } + + return (processControl: true, success: default); + } + + private static string EscapeSingle(string p) => p.Replace("'", "'\\'", StringComparison.Ordinal); + private void EnqueueCompaction(string filePath) { if (!_pendingCompactions.TryAdd(filePath, 0)) @@ -735,13 +762,6 @@ public sealed class FileCompactor : IDisposable } } - [StructLayout(LayoutKind.Sequential, Pack = 1)] - private struct WOF_FILE_COMPRESSION_INFO_V1 - { - public int Algorithm; - public ulong Flags; - } - [DllImport("kernel32.dll", SetLastError = true)] private static extern bool DeviceIoControl(SafeFileHandle hDevice, uint dwIoControlCode, IntPtr lpInBuffer, uint nInBufferSize, IntPtr lpOutBuffer, uint nOutBufferSize, out uint lpBytesReturned, IntPtr lpOverlapped); From cfc9f60176ff29bf237427572a7000154fafc72b Mon Sep 17 00:00:00 2001 From: cake Date: Mon, 3 Nov 2025 19:27:47 +0100 Subject: [PATCH 22/28] Added safe checks on enqueue. --- LightlessSync/FileCache/FileCompactor.cs | 152 ++++++++++++++++------- 1 file changed, 107 insertions(+), 45 deletions(-) diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index b9c2118..4722b1f 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -4,7 +4,6 @@ using Microsoft.Extensions.Logging; using Microsoft.Win32.SafeHandles; using System.Collections.Concurrent; using System.Diagnostics; -using System.IO; using System.Runtime.InteropServices; using System.Threading.Channels; using static LightlessSync.Utils.FileSystemHelper; @@ -140,42 +139,82 @@ public sealed class FileCompactor : IDisposable if (fsType == FilesystemType.NTFS && !_dalamudUtilService.IsWine) { - var blockSize = GetBlockSizeForPath(fileInfo.FullName, _logger, _dalamudUtilService.IsWine); - var losize = GetCompressedFileSizeW(fileInfo.FullName, out uint hosize); - var size = (long)hosize << 32 | losize; - return ((size + blockSize - 1) / blockSize) * blockSize; + (bool flowControl, long value) = GetFileSizeNTFS(fileInfo); + if (!flowControl) + { + return value; + } } if (fsType == FilesystemType.Btrfs) { - try + (bool flowControl, long value) = GetFileSizeBtrfs(fileInfo); + if (!flowControl) { - bool isWine = _dalamudUtilService?.IsWine ?? false; - string realPath = isWine ? ToLinuxPathIfWine(fileInfo.FullName, isWine) : fileInfo.FullName; - - var fileName = "stat"; - var arguments = $"-c %b \"{realPath}\""; - - (bool processControl, bool success) = StartProcessInfo(realPath, fileName, arguments, out Process? proc, out string stdout); - - if (!processControl && !success) - throw new InvalidOperationException($"stat failed: {proc}"); - - if (!long.TryParse(stdout.Trim(), out var blocks)) - throw new InvalidOperationException($"invalid stat output: {stdout}"); - - // st_blocks are always 512-byte on Linux enviroment. - return blocks * 512L; - } - catch (Exception ex) - { - _logger.LogDebug(ex, "Failed stat size for {file}, fallback to Length", fileInfo.FullName); + return value; } } return fileInfo.Length; } + /// + /// Get File Size in an Btrfs file system (Linux/Wine). + /// + /// File that you want the size from. + /// Succesful check and value of the filesize. + /// Fails on the Process in StartProcessInfo + private (bool flowControl, long value) GetFileSizeBtrfs(FileInfo fileInfo) + { + try + { + bool isWine = _dalamudUtilService?.IsWine ?? false; + string realPath = isWine ? ToLinuxPathIfWine(fileInfo.FullName, isWine) : fileInfo.FullName; + + var fileName = "stat"; + var arguments = $"-c %b \"{realPath}\""; + + (bool processControl, bool success) = StartProcessInfo(realPath, fileName, arguments, out Process? proc, out string stdout); + + if (!processControl && !success) + throw new InvalidOperationException($"stat failed: {proc}"); + + if (!long.TryParse(stdout.Trim(), out var blocks)) + throw new InvalidOperationException($"invalid stat output: {stdout}"); + + // st_blocks are always 512-byte on Linux enviroment. + return (flowControl: false, value: blocks * 512L); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Failed stat size for {file}, fallback to Length", fileInfo.FullName); + } + + return (flowControl: true, value: default); + } + + /// + /// Get File Size in an NTFS file system (Windows). + /// + /// File that you want the size from. + /// Succesful check and value of the filesize. + private (bool flowControl, long value) GetFileSizeNTFS(FileInfo fileInfo) + { + try + { + var blockSize = GetBlockSizeForPath(fileInfo.FullName, _logger, _dalamudUtilService.IsWine); + var losize = GetCompressedFileSizeW(fileInfo.FullName, out uint hosize); + var size = (long)hosize << 32 | losize; + return (flowControl: false, value: ((size + blockSize - 1) / blockSize) * blockSize); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Failed stat size for {file}, fallback to Length", fileInfo.FullName); + } + + return (flowControl: true, value: default); + } + /// /// Compressing the given path with BTRFS or NTFS file system. /// @@ -293,7 +332,7 @@ public sealed class FileCompactor : IDisposable /// Decompress an BTRFS File /// /// Path of the compressed file - /// Decompessing state + /// Decompressing state private bool DecompressBtrfsFile(string path) { var fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); @@ -353,7 +392,7 @@ public sealed class FileCompactor : IDisposable /// Decompress an NTFS File /// /// Path of the compressed file - /// Decompessing state + /// Decompressing state private bool DecompressWOFFile(string path, out FileStream fs) { fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); @@ -571,14 +610,6 @@ public sealed class FileCompactor : IDisposable return false; } - //Skipping small files to make compression a bit faster, its not that effective on small files. - int blockSize = GetBlockSizeForPath(realPath, _logger, isWine); - if (fi.Length < Math.Max(blockSize * 2, 128 * 1024)) - { - _logger.LogTrace("Skipping Btrfs compression for small file {file} ({size} bytes)", realPath, fi.Length); - return true; - } - if (IsBtrfsCompressedFile(realPath)) { _logger.LogTrace("File {file} already compressed (Btrfs), skipping file", realPath); @@ -695,21 +726,52 @@ public sealed class FileCompactor : IDisposable private void EnqueueCompaction(string filePath) { + // Safe-checks + if (string.IsNullOrWhiteSpace(filePath)) + return; + + if (!_lightlessConfigService.Current.UseCompactor) + return; + + if (!File.Exists(filePath)) + return; + if (!_pendingCompactions.TryAdd(filePath, 0)) return; - var fsType = GetFilesystemType(filePath, _dalamudUtilService.IsWine); - if (fsType != FilesystemType.NTFS && fsType != FilesystemType.Btrfs) + bool enqueued = false; + try { - _logger.LogTrace("Skip enqueue (unsupported fs) {fs} {file}", fsType, filePath); - _pendingCompactions.TryRemove(filePath, out _); - return; - } + bool isWine = _dalamudUtilService?.IsWine ?? false; + var fsType = GetFilesystemType(filePath, isWine); - if (!_compactionQueue.Writer.TryWrite(filePath)) + // If under Wine, we should skip NTFS because its not Windows but might return NTFS. + if (fsType == FilesystemType.NTFS && isWine) + { + _logger.LogTrace("Skip enqueue (NTFS under Wine) {file}", filePath); + return; + } + + // Unknown file system should be skipped. + if (fsType != FilesystemType.NTFS && fsType != FilesystemType.Btrfs) + { + _logger.LogTrace("Skip enqueue (unsupported fs) {fs} {file}", fsType, filePath); + return; + } + + if (!_compactionQueue.Writer.TryWrite(filePath)) + { + _logger.LogTrace("Skip enqueue: compaction channel is closed {file}", filePath); + return; + } + + enqueued = true; + _logger.LogTrace("Queued compaction for {file} (fs={fs})", filePath, fsType); + } + finally { - _pendingCompactions.TryRemove(filePath, out _); - _logger.LogDebug("Failed to enqueue compaction {file}", filePath); + if (!enqueued) + _pendingCompactions.TryRemove(filePath, out _); } } From b6aa2bebb18a6bfb3cd882b962b7c1d1a0a34c0d Mon Sep 17 00:00:00 2001 From: cake Date: Mon, 3 Nov 2025 19:59:12 +0100 Subject: [PATCH 23/28] Added more checks. --- LightlessSync/Services/NameplateHandler.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/LightlessSync/Services/NameplateHandler.cs b/LightlessSync/Services/NameplateHandler.cs index a28be5f..e302934 100644 --- a/LightlessSync/Services/NameplateHandler.cs +++ b/LightlessSync/Services/NameplateHandler.cs @@ -571,6 +571,7 @@ public unsafe class NameplateHandler : IMediatorSubscriber var nameplateObject = GetNameplateObject(i); return nameplateObject != null ? nameplateObject.Value.RootComponentNode : null; } + private HashSet VisibleUserIds => [.. _pairManager.GetOnlineUserPairs() .Where(u => u.IsVisible && u.PlayerCharacterId != uint.MaxValue) .Select(u => (ulong)u.PlayerCharacterId)]; From 1b686e45dc75dc485bce3d81c6d745e76115103c Mon Sep 17 00:00:00 2001 From: cake Date: Mon, 3 Nov 2025 20:19:02 +0100 Subject: [PATCH 24/28] Added more null checks and redid active broadcasting cache. --- LightlessSync/Services/NameplateHandler.cs | 146 +++++++++++---------- 1 file changed, 79 insertions(+), 67 deletions(-) diff --git a/LightlessSync/Services/NameplateHandler.cs b/LightlessSync/Services/NameplateHandler.cs index e302934..185a81d 100644 --- a/LightlessSync/Services/NameplateHandler.cs +++ b/LightlessSync/Services/NameplateHandler.cs @@ -15,8 +15,8 @@ using LightlessSync.UtilsEnum.Enum; // Created using https://github.com/PunishedPineapple/Distance as a reference, thank you! using Microsoft.Extensions.Logging; +using System.Collections.Immutable; using System.Globalization; -using System.Text; namespace LightlessSync.Services; @@ -32,10 +32,10 @@ public unsafe class NameplateHandler : IMediatorSubscriber private readonly LightlessMediator _mediator; public LightlessMediator Mediator => _mediator; - private bool mEnabled = false; + private bool _mEnabled = false; private bool _needsLabelRefresh = false; - private AddonNamePlate* mpNameplateAddon = null; - private readonly AtkTextNode*[] mTextNodes = new AtkTextNode*[AddonNamePlate.NumNamePlateObjects]; + private AddonNamePlate* _mpNameplateAddon = null; + private readonly AtkTextNode*[] _mTextNodes = new AtkTextNode*[AddonNamePlate.NumNamePlateObjects]; private readonly int[] _cachedNameplateTextWidths = new int[AddonNamePlate.NumNamePlateObjects]; private readonly int[] _cachedNameplateTextHeights = new int[AddonNamePlate.NumNamePlateObjects]; private readonly int[] _cachedNameplateContainerHeights = new int[AddonNamePlate.NumNamePlateObjects]; @@ -44,10 +44,10 @@ public unsafe class NameplateHandler : IMediatorSubscriber internal const uint mNameplateNodeIDBase = 0x7D99D500; private const string DefaultLabelText = "LightFinder"; private const SeIconChar DefaultIcon = SeIconChar.Hyadelyn; - private const int ContainerOffsetX = 50; + private const int _containerOffsetX = 50; private static readonly string DefaultIconGlyph = SeIconCharExtensions.ToIconString(DefaultIcon); - private volatile HashSet _activeBroadcastingCids = []; + private ImmutableHashSet _activeBroadcastingCids = []; public NameplateHandler(ILogger logger, IAddonLifecycle addonLifecycle, IGameGui gameGui, DalamudUtilService dalamudUtil, LightlessConfigService configService, LightlessMediator mediator, IClientState clientState, PairManager pairManager) { @@ -74,17 +74,17 @@ public unsafe class NameplateHandler : IMediatorSubscriber DisableNameplate(); DestroyNameplateNodes(); _mediator.Unsubscribe(this); - mpNameplateAddon = null; + _mpNameplateAddon = null; } internal void EnableNameplate() { - if (!mEnabled) + if (!_mEnabled) { try { _addonLifecycle.RegisterListener(AddonEvent.PostDraw, "NamePlate", NameplateDrawDetour); - mEnabled = true; + _mEnabled = true; } catch (Exception e) { @@ -96,7 +96,7 @@ public unsafe class NameplateHandler : IMediatorSubscriber internal void DisableNameplate() { - if (mEnabled) + if (_mEnabled) { try { @@ -107,7 +107,7 @@ public unsafe class NameplateHandler : IMediatorSubscriber _logger.LogError($"Unknown error while unregistering nameplate listener:\n{e}"); } - mEnabled = false; + _mEnabled = false; HideAllNameplateNodes(); } } @@ -116,15 +116,15 @@ public unsafe class NameplateHandler : IMediatorSubscriber { var pNameplateAddon = (AddonNamePlate*)args.Addon.Address; - if (mpNameplateAddon != pNameplateAddon) + if (_mpNameplateAddon != pNameplateAddon) { - for (int i = 0; i < mTextNodes.Length; ++i) mTextNodes[i] = null; + for (int i = 0; i < _mTextNodes.Length; ++i) _mTextNodes[i] = null; System.Array.Clear(_cachedNameplateTextWidths, 0, _cachedNameplateTextWidths.Length); System.Array.Clear(_cachedNameplateTextHeights, 0, _cachedNameplateTextHeights.Length); System.Array.Clear(_cachedNameplateContainerHeights, 0, _cachedNameplateContainerHeights.Length); System.Array.Fill(_cachedNameplateTextOffsets, int.MinValue); - mpNameplateAddon = pNameplateAddon; - if (mpNameplateAddon != null) CreateNameplateNodes(); + _mpNameplateAddon = pNameplateAddon; + if (_mpNameplateAddon != null) CreateNameplateNodes(); } UpdateNameplateNodes(); @@ -139,6 +139,11 @@ public unsafe class NameplateHandler : IMediatorSubscriber continue; var pNameplateResNode = nameplateObject.Value.NameContainer; + if (pNameplateResNode == null) + continue; + if (pNameplateResNode->ChildNode == null) + continue; + var pNewNode = AtkNodeHelpers.CreateOrphanTextNode(mNameplateNodeIDBase + (uint)i, TextFlags.Edge | TextFlags.Glare); if (pNewNode != null) @@ -150,7 +155,7 @@ public unsafe class NameplateHandler : IMediatorSubscriber pLastChild->PrevSiblingNode = (AtkResNode*)pNewNode; nameplateObject.Value.RootComponentNode->Component->UldManager.UpdateDrawNodeList(); pNewNode->AtkResNode.SetUseDepthBasedPriority(true); - mTextNodes[i] = pNewNode; + _mTextNodes[i] = pNewNode; } } } @@ -158,12 +163,12 @@ public unsafe class NameplateHandler : IMediatorSubscriber private void DestroyNameplateNodes() { var pCurrentNameplateAddon = (AddonNamePlate*)_gameGui.GetAddonByName("NamePlate", 1).Address; - if (mpNameplateAddon == null || mpNameplateAddon != pCurrentNameplateAddon) + if (_mpNameplateAddon == null || _mpNameplateAddon != pCurrentNameplateAddon) return; for (int i = 0; i < AddonNamePlate.NumNamePlateObjects; ++i) { - var pTextNode = mTextNodes[i]; + var pTextNode = _mTextNodes[i]; var pNameplateNode = GetNameplateComponentNode(i); if (pTextNode != null && pNameplateNode != null) { @@ -175,7 +180,7 @@ public unsafe class NameplateHandler : IMediatorSubscriber pTextNode->AtkResNode.NextSiblingNode->PrevSiblingNode = pTextNode->AtkResNode.PrevSiblingNode; pNameplateNode->Component->UldManager.UpdateDrawNodeList(); pTextNode->AtkResNode.Destroy(true); - mTextNodes[i] = null; + _mTextNodes[i] = null; } catch (Exception e) { @@ -192,7 +197,7 @@ public unsafe class NameplateHandler : IMediatorSubscriber private void HideAllNameplateNodes() { - for (int i = 0; i < mTextNodes.Length; ++i) + for (int i = 0; i < _mTextNodes.Length; ++i) { HideNameplateTextNode(i); } @@ -200,22 +205,34 @@ public unsafe class NameplateHandler : IMediatorSubscriber private void UpdateNameplateNodes() { - var framework = Framework.Instance(); - var ui3DModule = framework->GetUIModule()->GetUI3DModule(); + var currentAddon = (AddonNamePlate*)_gameGui.GetAddonByName("NamePlate", 1).Address; + if (_mpNameplateAddon == null || currentAddon == null || currentAddon != _mpNameplateAddon) + return; + var framework = Framework.Instance(); + + var ui3DModule = framework->GetUIModule()->GetUI3DModule(); if (ui3DModule == null) return; - for (int i = 0; i < ui3DModule->NamePlateObjectInfoCount; ++i) + var vec = ui3DModule->NamePlateObjectInfoPointers; + if (vec.IsEmpty) + return; + + var safeCount = System.Math.Min( + ui3DModule->NamePlateObjectInfoCount, + vec.Length + ); + + for (int i = 0; i < safeCount; ++i) { - if (ui3DModule->NamePlateObjectInfoPointers.IsEmpty) continue; + var config = _configService.Current; - var objectInfoPtr = ui3DModule->NamePlateObjectInfoPointers[i]; - - if (objectInfoPtr == null) continue; + var objectInfoPtr = vec[i]; + if (objectInfoPtr == null) + continue; var objectInfo = objectInfoPtr.Value; - if (objectInfo == null || objectInfo->GameObject == null) continue; @@ -223,62 +240,61 @@ public unsafe class NameplateHandler : IMediatorSubscriber if (nameplateIndex < 0 || nameplateIndex >= AddonNamePlate.NumNamePlateObjects) continue; - var pNode = mTextNodes[nameplateIndex]; + var pNode = _mTextNodes[nameplateIndex]; if (pNode == null) continue; - if (mpNameplateAddon == null) - continue; - + // CID gating var cid = DalamudUtilService.GetHashedCIDFromPlayerPointer((nint)objectInfo->GameObject); - if (cid == null || !_activeBroadcastingCids.Contains(cid)) { pNode->AtkResNode.ToggleVisibility(false); continue; } - if (!_configService.Current.LightfinderLabelShowOwn && (objectInfo->GameObject->GetGameObjectId() == _clientState.LocalPlayer.GameObjectId)) + var local = _clientState.LocalPlayer; + if (!config.LightfinderLabelShowOwn && local != null && + objectInfo->GameObject->GetGameObjectId() == local.GameObjectId) { pNode->AtkResNode.ToggleVisibility(false); continue; } - if (!_configService.Current.LightfinderLabelShowPaired && VisibleUserIds.Any(u => u == objectInfo->GameObject->GetGameObjectId())) - { - pNode->AtkResNode.ToggleVisibility(false); - continue; - } - - var nameplateObject = mpNameplateAddon->NamePlateObjectArray[nameplateIndex]; - nameplateObject.RootComponentNode->Component->UldManager.UpdateDrawNodeList(); - - var pNameplateIconNode = nameplateObject.MarkerIcon; - var pNameplateResNode = nameplateObject.NameContainer; - var pNameplateTextNode = nameplateObject.NameText; - bool IsVisible = pNameplateIconNode->AtkResNode.IsVisible() || (pNameplateResNode->IsVisible() && pNameplateTextNode->AtkResNode.IsVisible()) || _configService.Current.LightfinderLabelShowHidden; - pNode->AtkResNode.ToggleVisibility(IsVisible); - - if (nameplateObject.RootComponentNode == null || - nameplateObject.NameContainer == null || - nameplateObject.NameText == null) + var visibleUserIds = VisibleUserIds; + var hidePaired = !config.LightfinderLabelShowPaired; + + var goId = (ulong)objectInfo->GameObject->GetGameObjectId(); + if (hidePaired && visibleUserIds.Contains(goId)) { pNode->AtkResNode.ToggleVisibility(false); continue; } + var nameplateObject = _mpNameplateAddon->NamePlateObjectArray[nameplateIndex]; + var root = nameplateObject.RootComponentNode; var nameContainer = nameplateObject.NameContainer; var nameText = nameplateObject.NameText; + var marker = nameplateObject.MarkerIcon; - if (nameContainer == null || nameText == null) + if (root == null || nameContainer == null || nameText == null) { pNode->AtkResNode.ToggleVisibility(false); continue; } + + root->Component->UldManager.UpdateDrawNodeList(); + + bool isVisible = + ((marker != null) && marker->AtkResNode.IsVisible()) || + (nameContainer->IsVisible() && nameText->AtkResNode.IsVisible()) || + config.LightfinderLabelShowHidden; + + pNode->AtkResNode.ToggleVisibility(isVisible); + if (!isVisible) + continue; var labelColor = UIColors.Get("Lightfinder"); var edgeColor = UIColors.Get("LightfinderEdge"); - var config = _configService.Current; var scaleMultiplier = System.Math.Clamp(config.LightfinderLabelScale, 0.5f, 2.0f); var baseScale = config.LightfinderLabelUseIcon ? 1.0f : 0.5f; @@ -545,7 +561,7 @@ public unsafe class NameplateHandler : IMediatorSubscriber } private void HideNameplateTextNode(int i) { - var pNode = mTextNodes[i]; + var pNode = _mTextNodes[i]; if (pNode != null) { pNode->AtkResNode.ToggleVisibility(false); @@ -555,10 +571,10 @@ public unsafe class NameplateHandler : IMediatorSubscriber private AddonNamePlate.NamePlateObject? GetNameplateObject(int i) { if (i < AddonNamePlate.NumNamePlateObjects && - mpNameplateAddon != null && - mpNameplateAddon->NamePlateObjectArray[i].RootComponentNode != null) + _mpNameplateAddon != null && + _mpNameplateAddon->NamePlateObjectArray[i].RootComponentNode != null) { - return mpNameplateAddon->NamePlateObjectArray[i]; + return _mpNameplateAddon->NamePlateObjectArray[i]; } else { @@ -576,6 +592,7 @@ public unsafe class NameplateHandler : IMediatorSubscriber .Where(u => u.IsVisible && u.PlayerCharacterId != uint.MaxValue) .Select(u => (ulong)u.PlayerCharacterId)]; + public void FlagRefresh() { _needsLabelRefresh = true; @@ -592,18 +609,13 @@ public unsafe class NameplateHandler : IMediatorSubscriber public void UpdateBroadcastingCids(IEnumerable cids) { - var newSet = cids.ToHashSet(); - - var changed = !_activeBroadcastingCids.SetEquals(newSet); - if (!changed) + var newSet = cids.ToImmutableHashSet(StringComparer.Ordinal); + if (ReferenceEquals(_activeBroadcastingCids, newSet) || _activeBroadcastingCids.SetEquals(newSet)) return; - _activeBroadcastingCids.Clear(); - foreach (var cid in newSet) - _activeBroadcastingCids.Add(cid); - + // single atomic swap readers always see a consistent snapshot + _activeBroadcastingCids = newSet; _logger.LogInformation("Active broadcast CIDs: {Cids}", string.Join(",", _activeBroadcastingCids)); - FlagRefresh(); } From 35636f27f6b8b38e6864b6f8d4a6e8e09a857680 Mon Sep 17 00:00:00 2001 From: cake Date: Mon, 3 Nov 2025 21:47:15 +0100 Subject: [PATCH 25/28] Cleanup --- LightlessSync/Services/NameplateHandler.cs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/LightlessSync/Services/NameplateHandler.cs b/LightlessSync/Services/NameplateHandler.cs index 185a81d..5e83683 100644 --- a/LightlessSync/Services/NameplateHandler.cs +++ b/LightlessSync/Services/NameplateHandler.cs @@ -205,7 +205,7 @@ public unsafe class NameplateHandler : IMediatorSubscriber private void UpdateNameplateNodes() { - var currentAddon = (AddonNamePlate*)_gameGui.GetAddonByName("NamePlate", 1).Address; + var currentAddon = (AddonNamePlate*)_gameGui.GetAddonByName("NamePlate").Address; if (_mpNameplateAddon == null || currentAddon == null || currentAddon != _mpNameplateAddon) return; @@ -248,7 +248,7 @@ public unsafe class NameplateHandler : IMediatorSubscriber var cid = DalamudUtilService.GetHashedCIDFromPlayerPointer((nint)objectInfo->GameObject); if (cid == null || !_activeBroadcastingCids.Contains(cid)) { - pNode->AtkResNode.ToggleVisibility(false); + pNode->AtkResNode.ToggleVisibility(enable: false); continue; } @@ -256,7 +256,7 @@ public unsafe class NameplateHandler : IMediatorSubscriber if (!config.LightfinderLabelShowOwn && local != null && objectInfo->GameObject->GetGameObjectId() == local.GameObjectId) { - pNode->AtkResNode.ToggleVisibility(false); + pNode->AtkResNode.ToggleVisibility(enable: false); continue; } @@ -266,7 +266,7 @@ public unsafe class NameplateHandler : IMediatorSubscriber var goId = (ulong)objectInfo->GameObject->GetGameObjectId(); if (hidePaired && visibleUserIds.Contains(goId)) { - pNode->AtkResNode.ToggleVisibility(false); + pNode->AtkResNode.ToggleVisibility(enable: false); continue; } @@ -278,7 +278,7 @@ public unsafe class NameplateHandler : IMediatorSubscriber if (root == null || nameContainer == null || nameText == null) { - pNode->AtkResNode.ToggleVisibility(false); + pNode->AtkResNode.ToggleVisibility(enable: false); continue; } @@ -453,7 +453,7 @@ public unsafe class NameplateHandler : IMediatorSubscriber positionY += config.LightfinderLabelOffsetY; alignment = (AlignmentType)System.Math.Clamp((int)alignment, 0, 8); - pNode->AtkResNode.SetUseDepthBasedPriority(true); + pNode->AtkResNode.SetUseDepthBasedPriority(enable: true); pNode->AtkResNode.Color.A = 255; @@ -613,9 +613,8 @@ public unsafe class NameplateHandler : IMediatorSubscriber if (ReferenceEquals(_activeBroadcastingCids, newSet) || _activeBroadcastingCids.SetEquals(newSet)) return; - // single atomic swap readers always see a consistent snapshot _activeBroadcastingCids = newSet; - _logger.LogInformation("Active broadcast CIDs: {Cids}", string.Join(",", _activeBroadcastingCids)); + _logger.LogInformation("Active broadcast CIDs: {Cids}", string.Join(',', _activeBroadcastingCids)); FlagRefresh(); } From 1d672d25520f4aee9b6c2b88ee486d82e821967a Mon Sep 17 00:00:00 2001 From: azyges <229218900+azyges@users.noreply.github.com> Date: Thu, 6 Nov 2025 23:40:58 +0900 Subject: [PATCH 26/28] improve checks and add logging --- LightlessSync/Services/NameplateHandler.cs | 94 ++++++++++++++++++---- 1 file changed, 80 insertions(+), 14 deletions(-) diff --git a/LightlessSync/Services/NameplateHandler.cs b/LightlessSync/Services/NameplateHandler.cs index 5e83683..11af974 100644 --- a/LightlessSync/Services/NameplateHandler.cs +++ b/LightlessSync/Services/NameplateHandler.cs @@ -1,5 +1,6 @@ using Dalamud.Game.Addon.Lifecycle; using Dalamud.Game.Addon.Lifecycle.AddonArgTypes; +using Dalamud.Game.ClientState.Objects.Enums; using Dalamud.Game.Text; using Dalamud.Plugin.Services; using FFXIVClientStructs.FFXIV.Client.System.Framework; @@ -15,6 +16,7 @@ using LightlessSync.UtilsEnum.Enum; // Created using https://github.com/PunishedPineapple/Distance as a reference, thank you! using Microsoft.Extensions.Logging; +using System.Collections.Generic; using System.Collections.Immutable; using System.Globalization; @@ -114,6 +116,12 @@ public unsafe class NameplateHandler : IMediatorSubscriber private void NameplateDrawDetour(AddonEvent type, AddonArgs args) { + if (args.Addon.Address == nint.Zero) + { + _logger.LogWarning("Nameplate draw detour received a null addon address, skipping update."); + return; + } + var pNameplateAddon = (AddonNamePlate*)args.Addon.Address; if (_mpNameplateAddon != pNameplateAddon) @@ -138,6 +146,10 @@ public unsafe class NameplateHandler : IMediatorSubscriber if (nameplateObject == null) continue; + var rootNode = nameplateObject.Value.RootComponentNode; + if (rootNode == null || rootNode->Component == null) + continue; + var pNameplateResNode = nameplateObject.Value.NameContainer; if (pNameplateResNode == null) continue; @@ -153,7 +165,7 @@ public unsafe class NameplateHandler : IMediatorSubscriber pNewNode->AtkResNode.NextSiblingNode = pLastChild; pNewNode->AtkResNode.ParentNode = pNameplateResNode; pLastChild->PrevSiblingNode = (AtkResNode*)pNewNode; - nameplateObject.Value.RootComponentNode->Component->UldManager.UpdateDrawNodeList(); + rootNode->Component->UldManager.UpdateDrawNodeList(); pNewNode->AtkResNode.SetUseDepthBasedPriority(true); _mTextNodes[i] = pNewNode; } @@ -162,15 +174,34 @@ public unsafe class NameplateHandler : IMediatorSubscriber private void DestroyNameplateNodes() { - var pCurrentNameplateAddon = (AddonNamePlate*)_gameGui.GetAddonByName("NamePlate", 1).Address; - if (_mpNameplateAddon == null || _mpNameplateAddon != pCurrentNameplateAddon) + var currentHandle = _gameGui.GetAddonByName("NamePlate", 1); + if (currentHandle.Address == nint.Zero) + { + _logger.LogWarning("Unable to destroy nameplate nodes because the NamePlate addon is not available."); return; + } + + var pCurrentNameplateAddon = (AddonNamePlate*)currentHandle.Address; + if (_mpNameplateAddon == null) + return; + + if (_mpNameplateAddon != pCurrentNameplateAddon) + { + _logger.LogWarning("Skipping nameplate node destroy due to addon address mismatch (cached {Cached:X}, current {Current:X}).", (IntPtr)_mpNameplateAddon, (IntPtr)pCurrentNameplateAddon); + return; + } for (int i = 0; i < AddonNamePlate.NumNamePlateObjects; ++i) { var pTextNode = _mTextNodes[i]; var pNameplateNode = GetNameplateComponentNode(i); - if (pTextNode != null && pNameplateNode != null) + if (pTextNode != null && (pNameplateNode == null || pNameplateNode->Component == null)) + { + _logger.LogDebug("Skipping destroy for nameplate {Index} because its component node is unavailable.", i); + continue; + } + + if (pTextNode != null && pNameplateNode != null && pNameplateNode->Component != null) { try { @@ -205,20 +236,48 @@ public unsafe class NameplateHandler : IMediatorSubscriber private void UpdateNameplateNodes() { - var currentAddon = (AddonNamePlate*)_gameGui.GetAddonByName("NamePlate").Address; - if (_mpNameplateAddon == null || currentAddon == null || currentAddon != _mpNameplateAddon) + var currentHandle = _gameGui.GetAddonByName("NamePlate"); + if (currentHandle.Address == nint.Zero) + { + _logger.LogDebug("NamePlate addon unavailable during update, skipping label refresh."); return; + } + + var currentAddon = (AddonNamePlate*)currentHandle.Address; + if (_mpNameplateAddon == null || currentAddon == null || currentAddon != _mpNameplateAddon) + { + if (_mpNameplateAddon != null) + _logger.LogDebug("Cached NamePlate addon pointer differs from current: waiting for new hook (cached {Cached:X}, current {Current:X}).", (IntPtr)_mpNameplateAddon, (IntPtr)currentAddon); + return; + } var framework = Framework.Instance(); - - var ui3DModule = framework->GetUIModule()->GetUI3DModule(); - if (ui3DModule == null) + if (framework == null) + { + _logger.LogDebug("Framework instance unavailable during nameplate update, skipping."); return; + } + + var uiModule = framework->GetUIModule(); + if (uiModule == null) + { + _logger.LogDebug("UI module unavailable during nameplate update, skipping."); + return; + } + + var ui3DModule = uiModule->GetUI3DModule(); + if (ui3DModule == null) + { + _logger.LogDebug("UI3D module unavailable during nameplate update, skipping."); + return; + } var vec = ui3DModule->NamePlateObjectInfoPointers; if (vec.IsEmpty) return; + var visibleUserIdsSnapshot = VisibleUserIds; + var safeCount = System.Math.Min( ui3DModule->NamePlateObjectInfoCount, vec.Length @@ -244,8 +303,15 @@ public unsafe class NameplateHandler : IMediatorSubscriber if (pNode == null) continue; + var gameObject = objectInfo->GameObject; + if ((ObjectKind)gameObject->ObjectKind != ObjectKind.Player) + { + pNode->AtkResNode.ToggleVisibility(enable: false); + continue; + } + // CID gating - var cid = DalamudUtilService.GetHashedCIDFromPlayerPointer((nint)objectInfo->GameObject); + var cid = DalamudUtilService.GetHashedCIDFromPlayerPointer((nint)gameObject); if (cid == null || !_activeBroadcastingCids.Contains(cid)) { pNode->AtkResNode.ToggleVisibility(enable: false); @@ -260,11 +326,10 @@ public unsafe class NameplateHandler : IMediatorSubscriber continue; } - var visibleUserIds = VisibleUserIds; var hidePaired = !config.LightfinderLabelShowPaired; - var goId = (ulong)objectInfo->GameObject->GetGameObjectId(); - if (hidePaired && visibleUserIds.Contains(goId)) + var goId = (ulong)gameObject->GetGameObjectId(); + if (hidePaired && visibleUserIdsSnapshot.Contains(goId)) { pNode->AtkResNode.ToggleVisibility(enable: false); continue; @@ -276,8 +341,9 @@ public unsafe class NameplateHandler : IMediatorSubscriber var nameText = nameplateObject.NameText; var marker = nameplateObject.MarkerIcon; - if (root == null || nameContainer == null || nameText == null) + if (root == null || root->Component == null || nameContainer == null || nameText == null) { + _logger.LogDebug("Nameplate {Index} missing required nodes during update, skipping.", nameplateIndex); pNode->AtkResNode.ToggleVisibility(enable: false); continue; } From 557121a9b7461e71c02fa345ac1f5362fabb2ac6 Mon Sep 17 00:00:00 2001 From: cake Date: Fri, 7 Nov 2025 05:27:58 +0100 Subject: [PATCH 27/28] Added batching for the File Frag command for the iscompressed calls. --- LightlessSync/FileCache/FileCompactor.cs | 491 +++++++++++------- .../Compression/BatchFileFragService.cs | 245 +++++++++ 2 files changed, 546 insertions(+), 190 deletions(-) create mode 100644 LightlessSync/Services/Compression/BatchFileFragService.cs diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index 4722b1f..e6505b9 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -1,5 +1,6 @@ using LightlessSync.LightlessConfiguration; using LightlessSync.Services; +using LightlessSync.Services.Compression; using Microsoft.Extensions.Logging; using Microsoft.Win32.SafeHandles; using System.Collections.Concurrent; @@ -23,8 +24,12 @@ public sealed class FileCompactor : IDisposable private readonly Channel _compactionQueue; private readonly CancellationTokenSource _compactionCts = new(); - private readonly Task _compactionWorker; - + + private readonly List _workers = []; + private readonly SemaphoreSlim _globalGate; + private static readonly SemaphoreSlim _btrfsGate = new(4, 4); + private readonly BatchFilefragService _fragBatch; + private readonly WOF_FILE_COMPRESSION_INFO_V1 _efInfo = new() { Algorithm = (int)CompressionAlgorithm.XPRESS8K, @@ -57,11 +62,30 @@ public sealed class FileCompactor : IDisposable _compactionQueue = Channel.CreateUnbounded(new UnboundedChannelOptions { - SingleReader = true, + SingleReader = false, SingleWriter = false }); - _compactionWorker = Task.Factory.StartNew(() => ProcessQueueAsync(_compactionCts.Token), _compactionCts.Token, TaskCreationOptions.LongRunning,TaskScheduler.Default).Unwrap(); + int workers = Math.Clamp(Math.Min(Environment.ProcessorCount / 2, 4), 1, 8); + _globalGate = new SemaphoreSlim(workers, workers); + int workerCount = Math.Max(workers * 2, workers); + + for (int i = 0; i < workerCount; i++) + { + _workers.Add(Task.Factory.StartNew( + () => ProcessQueueWorkerAsync(_compactionCts.Token), + _compactionCts.Token, + TaskCreationOptions.LongRunning, + TaskScheduler.Default).Unwrap()); + } + + _fragBatch = new BatchFilefragService( + useShell: _dalamudUtilService.IsWine, + log: _logger, + batchSize: 128, + flushMs: 25); + + _logger.LogInformation("FileCompactor started with {workers} workers", workerCount); } public bool MassCompactRunning { get; private set; } @@ -171,18 +195,12 @@ public sealed class FileCompactor : IDisposable bool isWine = _dalamudUtilService?.IsWine ?? false; string realPath = isWine ? ToLinuxPathIfWine(fileInfo.FullName, isWine) : fileInfo.FullName; - var fileName = "stat"; - var arguments = $"-c %b \"{realPath}\""; + (bool ok, string stdout, string stderr, int code) = + RunProcessDirect("stat", ["-c", "%b", realPath]); - (bool processControl, bool success) = StartProcessInfo(realPath, fileName, arguments, out Process? proc, out string stdout); + if (!ok || !long.TryParse(stdout.Trim(), out var blocks)) + throw new InvalidOperationException($"stat failed (exit {code}): {stderr}"); - if (!processControl && !success) - throw new InvalidOperationException($"stat failed: {proc}"); - - if (!long.TryParse(stdout.Trim(), out var blocks)) - throw new InvalidOperationException($"invalid stat output: {stdout}"); - - // st_blocks are always 512-byte on Linux enviroment. return (flowControl: false, value: blocks * 512L); } catch (Exception ex) @@ -224,18 +242,22 @@ public sealed class FileCompactor : IDisposable var fi = new FileInfo(filePath); if (!fi.Exists) { - _logger.LogTrace("Skip compact: missing {file}", filePath); + _logger.LogTrace("Skip compaction: missing {file}", filePath); return; } var fsType = GetFilesystemType(filePath, _dalamudUtilService.IsWine); - _logger.LogTrace("Detected filesystem {fs} for {file} (isWine={wine})", fsType, filePath, _dalamudUtilService.IsWine); var oldSize = fi.Length; - int blockSize = GetBlockSizeForPath(fi.FullName, _logger, _dalamudUtilService.IsWine); - if (oldSize < Math.Max(blockSize, 8 * 1024)) + + // We skipping small files (128KiB) as they slow down the system a lot for BTRFS. as BTRFS has a different blocksize it requires an different calculation. + long minSizeBytes = fsType == FilesystemType.Btrfs + ? Math.Max(blockSize * 2L, 128 * 1024L) + : Math.Max(blockSize, 8 * 1024L); + + if (oldSize < minSizeBytes) { - _logger.LogTrace("Skip compact: {file} < block {block}", filePath, blockSize); + _logger.LogTrace("Skip compaction: {file} ({size} B) < threshold ({th} B)", filePath, oldSize, minSizeBytes); return; } @@ -243,7 +265,7 @@ public sealed class FileCompactor : IDisposable { if (!IsWOFCompactedFile(filePath)) { - _logger.LogDebug("NTFS compact XPRESS8K: {file}", filePath); + _logger.LogDebug("NTFS compaction XPRESS8K: {file}", filePath); if (WOFCompressFile(filePath)) { var newSize = GetFileSizeOnDisk(fi); @@ -265,7 +287,7 @@ public sealed class FileCompactor : IDisposable { if (!IsBtrfsCompressedFile(filePath)) { - _logger.LogDebug("Btrfs compress zstd: {file}", filePath); + _logger.LogDebug("Btrfs compression zstd: {file}", filePath); if (BtrfsCompressFile(filePath)) { var newSize = GetFileSizeOnDisk(fi); @@ -299,7 +321,7 @@ public sealed class FileCompactor : IDisposable { try { - bool flowControl = DecompressWOFFile(path, out FileStream fs); + bool flowControl = DecompressWOFFile(path); if (!flowControl) { return; @@ -335,10 +357,9 @@ public sealed class FileCompactor : IDisposable /// Decompressing state private bool DecompressBtrfsFile(string path) { - var fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); - try { + _btrfsGate.Wait(_compactionCts.Token); bool isWine = _dalamudUtilService?.IsWine ?? false; string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; @@ -358,27 +379,25 @@ public sealed class FileCompactor : IDisposable return true; } - (bool flowControl, bool value) = FileStreamOpening(realPath, ref fs); + if (!ProbeFileReadable(realPath)) + return false; - if (!flowControl) + (bool ok, string stdout, string stderr, int code) = + isWine + ? RunProcessShell($"btrfs filesystem defragment -- {QuoteSingle(realPath)}") + : RunProcessDirect("btrfs", ["filesystem", "defragment", "--", realPath]); + + if (!ok) { - return value; - } - - string fileName = isWine ? "/bin/bash" : "btrfs"; - string command = isWine ? $"-c \"filesystem defragment -- \"{realPath}\"\"" : $"filesystem defragment -- \"{realPath}\""; - - (bool processControl, bool success) = StartProcessInfo(realPath, fileName, command, out Process? proc, out string stdout); - if (!processControl && !success) - { - return value; + _logger.LogWarning("btrfs defragment (decompress) failed for {file} (exit {code}): {stderr}", + realPath, code, stderr); + return false; } if (!string.IsNullOrWhiteSpace(stdout)) _logger.LogTrace("btrfs defragment output for {file}: {stdout}", realPath, stdout.Trim()); - _logger.LogInformation("Decompressed btrfs file successfully: {file}", realPath); - + _logger.LogInformation("Decompressed (rewritten) Btrfs file: {file}", realPath); return true; } catch (Exception ex) @@ -386,6 +405,11 @@ public sealed class FileCompactor : IDisposable _logger.LogWarning(ex, "Error rewriting {file} for Btrfs decompression", path); return false; } + finally + { + if (_btrfsGate.CurrentCount < 4) + _btrfsGate.Release(); + } } /// @@ -393,38 +417,40 @@ public sealed class FileCompactor : IDisposable /// /// Path of the compressed file /// Decompressing state - private bool DecompressWOFFile(string path, out FileStream fs) + private bool DecompressWOFFile(string path) { - fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); - var handle = fs.SafeFileHandle; - - if (handle.IsInvalid) + if (TryIsWofExternal(path, out bool isExternal, out int algo)) { - _logger.LogWarning("Invalid handle: {file}", path); - return false; + if (!isExternal) + { + _logger.LogTrace("Already decompressed file: {file}", path); + return true; + } + var compressString = ((CompressionAlgorithm)algo).ToString(); + _logger.LogTrace("WOF compression (algo={algo}) detected for {file}", compressString, path); } - if (!DeviceIoControl(handle, FSCTL_DELETE_EXTERNAL_BACKING, - IntPtr.Zero, 0, IntPtr.Zero, 0, - out _, IntPtr.Zero)) + return WithFileHandleForWOF(path, FileAccess.ReadWrite, h => { - int err = Marshal.GetLastWin32Error(); + if (!DeviceIoControl(h, FSCTL_DELETE_EXTERNAL_BACKING, + IntPtr.Zero, 0, IntPtr.Zero, 0, + out uint _, IntPtr.Zero)) + { + int err = Marshal.GetLastWin32Error(); + // 342 error code means its been decompressed after the control, we handle it as it succesfully been decompressed. + if (err == 342) + { + _logger.LogTrace("Successfully decompressed NTFS file {file}", path); + return true; + } - if (err == 342) - { - _logger.LogTrace("File {file} not externally backed (already decompressed)", path); - } - else - { _logger.LogWarning("DeviceIoControl failed for {file} with Win32 error {err}", path, err); + return false; } - } - else - { - _logger.LogTrace("Successfully decompressed NTFS file {file}", path); - } - return true; + _logger.LogTrace("Successfully decompressed NTFS file {file}", path); + return true; + }); } /// @@ -455,7 +481,6 @@ public sealed class FileCompactor : IDisposable /// Compessing state private bool WOFCompressFile(string path) { - FileStream? fs = null; int size = Marshal.SizeOf(); IntPtr efInfoPtr = Marshal.AllocHGlobal(size); @@ -464,46 +489,28 @@ public sealed class FileCompactor : IDisposable Marshal.StructureToPtr(_efInfo, efInfoPtr, fDeleteOld: false); ulong length = (ulong)size; - (bool flowControl, bool value) = FileStreamOpening(path, ref fs); - - if (!flowControl) + return WithFileHandleForWOF(path, FileAccess.ReadWrite, h => { - return value; - } + int ret = WofSetFileDataLocation(h, WOF_PROVIDER_FILE, efInfoPtr, length); - if (fs == null) - { - _logger.LogWarning("Failed to open {file} for compression; skipping", path); - return false; - } + // 0x80070158 is the benign "already compressed/unsupported" style return + if (ret != 0 && ret != unchecked((int)0x80070158)) + { + _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X")); + return false; + } - var handle = fs.SafeFileHandle; - - if (handle.IsInvalid) - { - _logger.LogWarning("Invalid file handle for {file}", path); - return false; - } - - int ret = WofSetFileDataLocation(handle, WOF_PROVIDER_FILE, efInfoPtr, length); - - // 0x80070158 is WOF error whenever compression fails in an non-fatal way. - if (ret != 0 && ret != unchecked((int)0x80070158)) - { - _logger.LogWarning("Failed to compact {file}: {ret}", path, ret.ToString("X")); - return false; - } - - return true; + return true; + }); } catch (DllNotFoundException ex) { - _logger.LogTrace(ex, "WofUtil.dll not available, this DLL is needed for compression; skipping NTFS compaction for {file}", path); + _logger.LogTrace(ex, "WofUtil not available; skipping NTFS compaction for {file}", path); return false; } catch (EntryPointNotFoundException ex) { - _logger.LogTrace(ex, "WOF entrypoint missing (Wine/older OS); skipping NTFS compaction for {file}", path); + _logger.LogTrace(ex, "WOF entrypoint missing on this system (Wine/older OS); skipping NTFS compaction for {file}", path); return false; } catch (Exception ex) @@ -513,12 +520,8 @@ public sealed class FileCompactor : IDisposable } finally { - fs?.Dispose(); - if (efInfoPtr != IntPtr.Zero) - { Marshal.FreeHGlobal(efInfoPtr); - } } } @@ -549,6 +552,36 @@ public sealed class FileCompactor : IDisposable } } + /// + /// Checks if an File is compacted any WOF compression with an WOF backing + /// + /// Path of the file + /// State of the file, if its an external (no backing) and which algorithm if detected + private static bool TryIsWofExternal(string path, out bool isExternal, out int algorithm) + { + isExternal = false; + algorithm = 0; + try + { + uint buf = (uint)Marshal.SizeOf(); + int hr = WofIsExternalFile(path, out int ext, out _, out var info, ref buf); + if (hr == 0 && ext != 0) + { + isExternal = true; + algorithm = info.Algorithm; + } + return true; + } + catch (DllNotFoundException) + { + return false; + } + catch (EntryPointNotFoundException) + { + return false; + } + } + /// /// Checks if an File is compacted with Btrfs compression /// @@ -558,34 +591,23 @@ public sealed class FileCompactor : IDisposable { try { + _btrfsGate.Wait(_compactionCts.Token); + bool isWine = _dalamudUtilService?.IsWine ?? false; string realPath = isWine ? ToLinuxPathIfWine(path, isWine) : path; - var fi = new FileInfo(realPath); - - if (fi == null) - { - _logger.LogWarning("Failed to open {file} for checking on compression; skipping", realPath); - return false; - } - - string fileName = isWine ? "/bin/bash" : "filefrag"; - string command = isWine ? $"-c \"filefrag -v '{EscapeSingle(realPath)}'\"" : $"-v \"{realPath}\""; - (bool processControl, bool success) = StartProcessInfo(realPath, fileName, command, out Process? proc, out string stdout); - if (!processControl && !success) - { - return success; - } - - bool compressed = stdout.Contains("flags: compressed", StringComparison.OrdinalIgnoreCase); - _logger.LogTrace("Btrfs compression check for {file}: {compressed}", realPath, compressed); - return compressed; + return _fragBatch.IsCompressedAsync(realPath, _compactionCts.Token).GetAwaiter().GetResult(); } catch (Exception ex) { _logger.LogDebug(ex, "Failed to detect Btrfs compression for {file}", path); return false; } + finally + { + if (_btrfsGate.CurrentCount < 4) + _btrfsGate.Release(); + } } /// @@ -595,8 +617,6 @@ public sealed class FileCompactor : IDisposable /// Compessing state private bool BtrfsCompressFile(string path) { - FileStream? fs = null; - try { bool isWine = _dalamudUtilService?.IsWine ?? false; @@ -616,21 +636,22 @@ public sealed class FileCompactor : IDisposable return true; } - (bool flowControl, bool value) = FileStreamOpening(realPath, ref fs); + if (!ProbeFileReadable(realPath)) + return false; - if (!flowControl) + (bool ok, string stdout, string stderr, int code) = + isWine + ? RunProcessShell($"btrfs filesystem defragment -clzo -- {QuoteSingle(realPath)}") + : RunProcessDirect("btrfs", ["filesystem", "defragment", "-clzo", "--", realPath]); + + if (!ok) { - return value; + _logger.LogWarning("btrfs defragment failed for {file} (exit {code}): {stderr}", realPath, code, stderr); + return false; } - string fileName = isWine ? "/bin/bash" : "btrfs"; - string command = isWine ? $"-c \"btrfs filesystem defragment -czstd:1 -- \"{realPath}\"\"" : $"btrfs filesystem defragment -czstd:1 -- \"{realPath}\""; - - (bool processControl, bool success) = StartProcessInfo(realPath, fileName, command, out Process? proc, out string stdout); - if (!processControl && !success) - { - return value; - } + if (!string.IsNullOrWhiteSpace(stdout)) + _logger.LogTrace("btrfs output for {file}: {stdout}", realPath, stdout.Trim()); if (!string.IsNullOrWhiteSpace(stdout)) _logger.LogTrace("btrfs defragment output for {file}: {stdout}", realPath, stdout.Trim()); @@ -648,82 +669,171 @@ public sealed class FileCompactor : IDisposable /// - /// Trying opening file stream in certain amount of tries. + /// Probe file if its readable for certain amount of tries. /// /// Path where the file is located /// Filestream used for the function /// State of the filestream opening - private (bool flowControl, bool value) FileStreamOpening(string path, ref FileStream? fs) + private bool ProbeFileReadable(string path) { for (int attempt = 0; attempt < _maxRetries; attempt++) { try { - fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read); - break; + using var _ = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete); + return true; } - catch (IOException) + catch (IOException ex) { if (attempt == _maxRetries - 1) { - _logger.LogWarning("File still in use after {attempts} attempts, skipping compression for {file}", _maxRetries, path); - return (flowControl: false, value: false); + _logger.LogWarning(ex, "File still in use after {attempts} attempts, skipping {file}", _maxRetries, path); + return false; + } + int delay = 150 * (attempt + 1); + _logger.LogTrace(ex, "File busy, retrying in {delay}ms for {file}", delay, path); + Thread.Sleep(delay); + } + } + return false; + } + + /// + /// Attempt opening file stream for WOF functions + /// + /// File that has to be accessed + /// Permissions for the file + /// Access of the file stream for the WOF function to handle. + /// State of the attempt for the file + private bool WithFileHandleForWOF(string path, FileAccess access, Func body) + { + const FileShare share = FileShare.ReadWrite | FileShare.Delete; + + for (int attempt = 0; attempt < _maxRetries; attempt++) + { + try + { + using var fs = new FileStream(path, FileMode.Open, access, share); + + var handle = fs.SafeFileHandle; + if (handle.IsInvalid) + { + _logger.LogWarning("Invalid file handle for {file}", path); + return false; + } + + return body(handle); + } + catch (IOException ex) + { + if (attempt == _maxRetries - 1) + { + _logger.LogWarning(ex, "File still in use after {attempts} attempts, skipping {file}", _maxRetries, path); + return false; } int delay = 150 * (attempt + 1); - _logger.LogTrace("File in use, retrying in {delay}ms for {file}", delay, path); + _logger.LogTrace(ex, "File busy, retrying in {delay}ms for {file}", delay, path); Thread.Sleep(delay); } } - return (flowControl: true, value: default); + return false; } /// - /// Starts an process with given Filename and Arguments + /// Runs an nonshell process meant for Linux/Wine enviroments /// - /// Path you want to use for the process (Compression is using these) - /// File of the command - /// Arguments used for the command - /// Returns process of the given command - /// Returns output of the given command - /// Returns if the process been done succesfully or not - private (bool processControl, bool success) StartProcessInfo(string path, string fileName, string arguments, out Process? proc, out string stdout) + /// File that has to be excuted + /// Arguments meant for the file/command + /// Working directory used to execute the file with/without arguments + /// Timeout timer for the process + /// State of the process, output of the process and error with exit code + private (bool ok, string stdout, string stderr, int exitCode) RunProcessDirect(string fileName, IEnumerable args, string? workingDir = null, int timeoutMs = 60000) { - var psi = new ProcessStartInfo + var psi = new ProcessStartInfo(fileName) { - FileName = fileName, - Arguments = arguments, RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, - CreateNoWindow = true, - WorkingDirectory = "/" + CreateNoWindow = true }; - proc = Process.Start(psi); + if (!string.IsNullOrEmpty(workingDir)) psi.WorkingDirectory = workingDir; - if (proc == null) + foreach (var a in args) psi.ArgumentList.Add(a); + + using var proc = Process.Start(psi); + if (proc is null) return (false, "", "failed to start process", -1); + + var outTask = proc.StandardOutput.ReadToEndAsync(_compactionCts.Token); + var errTask = proc.StandardError.ReadToEndAsync(_compactionCts.Token); + + if (!proc.WaitForExit(timeoutMs)) { - _logger.LogWarning("Failed to start {arguments} for {file}", arguments, path); - stdout = string.Empty; - return (processControl: false, success: false); + try + { + proc.Kill(entireProcessTree: true); + } + catch + { + // Ignore this catch on the dispose + } + + Task.WaitAll([outTask, errTask], 1000, _compactionCts.Token); + return (false, outTask.Result, "timeout", -1); } - stdout = proc.StandardOutput.ReadToEnd(); - string stderr = proc.StandardError.ReadToEnd(); - proc.WaitForExit(); - - if (proc.ExitCode != 0 && !string.IsNullOrWhiteSpace(stderr)) - { - _logger.LogTrace("{arguments} exited with code {code}: {stderr}", arguments, proc.ExitCode, stderr); - return (processControl: false, success: false); - } - - return (processControl: true, success: default); + Task.WaitAll(outTask, errTask); + return (proc.ExitCode == 0, outTask.Result, errTask.Result, proc.ExitCode); } - private static string EscapeSingle(string p) => p.Replace("'", "'\\'", StringComparison.Ordinal); + /// + /// Runs an shell using '/bin/bash'/ command meant for Linux/Wine enviroments + /// + /// Command that has to be excuted + /// Timeout timer for the process + /// State of the process, output of the process and error with exit code + private (bool ok, string stdout, string stderr, int exitCode) RunProcessShell(string command, int timeoutMs = 60000) + { + var psi = new ProcessStartInfo("/bin/bash") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + psi.ArgumentList.Add("-c"); + psi.ArgumentList.Add(command); + using var proc = Process.Start(psi); + if (proc is null) return (false, "", "failed to start /bin/bash", -1); + + var outTask = proc.StandardOutput.ReadToEndAsync(_compactionCts.Token); + var errTask = proc.StandardError.ReadToEndAsync(_compactionCts.Token); + + if (!proc.WaitForExit(timeoutMs)) + { + try + { + proc.Kill(entireProcessTree: true); + } + catch + { + // Ignore this catch on the dispose + } + + Task.WaitAll([outTask, errTask], 1000, _compactionCts.Token); + return (false, outTask.Result, "timeout", -1); + } + + Task.WaitAll(outTask, errTask); + return (proc.ExitCode == 0, outTask.Result, errTask.Result, proc.ExitCode); + } + + /// + /// Enqueues the compaction/decompation of an filepath. + /// + /// Filepath that will be enqueued private void EnqueueCompaction(string filePath) { // Safe-checks @@ -759,9 +869,10 @@ public sealed class FileCompactor : IDisposable return; } + // Channel got closed, skip enqueue on file if (!_compactionQueue.Writer.TryWrite(filePath)) { - _logger.LogTrace("Skip enqueue: compaction channel is closed {file}", filePath); + _logger.LogTrace("Skip enqueue: compaction channel is/got closed {file}", filePath); return; } @@ -775,7 +886,11 @@ public sealed class FileCompactor : IDisposable } } - private async Task ProcessQueueAsync(CancellationToken token) + /// + /// Process the queue with, meant for a worker/thread + /// + /// Cancellation token for the worker whenever it needs to be stopped + private async Task ProcessQueueWorkerAsync(CancellationToken token) { try { @@ -785,28 +900,20 @@ public sealed class FileCompactor : IDisposable { try { - if (token.IsCancellationRequested) - { - return; - } + token.ThrowIfCancellationRequested(); + await _globalGate.WaitAsync(token).ConfigureAwait(false); - if (!_lightlessConfigService.Current.UseCompactor) + try { - continue; + if (_lightlessConfigService.Current.UseCompactor && File.Exists(filePath)) + CompactFile(filePath); } - - if (!File.Exists(filePath)) + finally { - _logger.LogTrace("Skip compact (missing) {file}", filePath); - continue; + _globalGate.Release(); } - - CompactFile(filePath); - } - catch (OperationCanceledException) - { - return; } + catch (OperationCanceledException) { return; } catch (Exception ex) { _logger.LogWarning(ex, "Error compacting file {file}", filePath); @@ -818,9 +925,9 @@ public sealed class FileCompactor : IDisposable } } } - catch (OperationCanceledException) - { - _logger.LogDebug("Compaction queue cancelled"); + catch (OperationCanceledException) + { + // Shutting down worker, this exception is expected } } @@ -836,17 +943,21 @@ public sealed class FileCompactor : IDisposable [DllImport("WofUtil.dll", SetLastError = true)] private static extern int WofSetFileDataLocation(SafeFileHandle FileHandle, ulong Provider, IntPtr ExternalFileInfo, ulong Length); + private static string QuoteSingle(string s) => "'" + s.Replace("'", "'\\''", StringComparison.Ordinal) + "'"; + public void Dispose() { + _fragBatch?.Dispose() _compactionQueue.Writer.TryComplete(); _compactionCts.Cancel(); + try { - _compactionWorker.Wait(TimeSpan.FromSeconds(5)); + Task.WaitAll([.. _workers.Where(t => t != null)], TimeSpan.FromSeconds(5)); } catch { - //ignore on catch ^^ + // Ignore this catch on the dispose } finally { diff --git a/LightlessSync/Services/Compression/BatchFileFragService.cs b/LightlessSync/Services/Compression/BatchFileFragService.cs new file mode 100644 index 0000000..ae5bb71 --- /dev/null +++ b/LightlessSync/Services/Compression/BatchFileFragService.cs @@ -0,0 +1,245 @@ +using Microsoft.Extensions.Logging; +using System.Diagnostics; +using System.Text.RegularExpressions; +using System.Threading.Channels; + +namespace LightlessSync.Services.Compression +{ + /// + /// This batch service is made for the File Frag command, because of each file needing to use this command. + /// It's better to combine into one big command in batches then doing each command on each compressed call. + /// + public sealed partial class BatchFilefragService : IDisposable + { + private readonly Channel<(string path, TaskCompletionSource tcs)> _ch; + private readonly Task _worker; + private readonly bool _useShell; + private readonly ILogger _log; + private readonly int _batchSize; + private readonly TimeSpan _flushDelay; + private readonly CancellationTokenSource _cts = new(); + + public BatchFilefragService(bool useShell, ILogger log, int batchSize = 128, int flushMs = 25) + { + _useShell = useShell; + _log = log; + _batchSize = Math.Max(8, batchSize); + _flushDelay = TimeSpan.FromMilliseconds(Math.Max(5, flushMs)); + _ch = Channel.CreateUnbounded<(string, TaskCompletionSource)>(new UnboundedChannelOptions { SingleReader = true, SingleWriter = false }); + _worker = Task.Run(ProcessAsync, _cts.Token); + } + + /// + /// Checks if the file is compressed using Btrfs using tasks + /// + /// Linux/Wine path given for the file. + /// Cancellation Token + /// If it was compressed or not + public Task IsCompressedAsync(string linuxPath, CancellationToken ct = default) + { + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + if (!_ch.Writer.TryWrite((linuxPath, tcs))) + { + tcs.TrySetResult(false); + return tcs.Task; + } + + if (ct.CanBeCanceled) + { + var reg = ct.Register(() => tcs.TrySetCanceled(ct)); + _ = tcs.Task.ContinueWith(_ => reg.Dispose(), CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); + } + + return tcs.Task; + } + + /// + /// Process the pending compression tasks asynchronously + /// + /// Task + private async Task ProcessAsync() + { + var reader = _ch.Reader; + var pending = new List<(string path, TaskCompletionSource tcs)>(_batchSize); + + try + { + while (await reader.WaitToReadAsync(_cts.Token).ConfigureAwait(false)) + { + if (!reader.TryRead(out var first)) continue; + pending.Add(first); + + var flushAt = DateTime.UtcNow + _flushDelay; + while (pending.Count < _batchSize && DateTime.UtcNow < flushAt) + { + if (reader.TryRead(out var item)) + { + pending.Add(item); + continue; + } + + if ((flushAt - DateTime.UtcNow) <= TimeSpan.Zero) break; + try + { + await Task.Delay(TimeSpan.FromMilliseconds(5), _cts.Token).ConfigureAwait(false); + } + catch + { + break; + } + } + + try + { + var map = await RunBatchAsync(pending.Select(p => p.path)).ConfigureAwait(false); + foreach (var (path, tcs) in pending) + { + tcs.TrySetResult(map.TryGetValue(path, out var c) && c); + } + } + catch (Exception ex) + { + _log.LogDebug(ex, "filefrag batch failed. falling back to false"); + foreach (var (_, tcs) in pending) + { + tcs.TrySetResult(false); + } + } + finally + { + pending.Clear(); + } + } + } + catch (OperationCanceledException) + { + //Shutting down worker, exception called + } + } + + /// + /// Running the batch of each file in the queue in one file frag command. + /// + /// Paths that are needed for the command building for the batch return + /// Path of the file and if it went correctly + /// Failing to start filefrag on the system if this exception is found + private async Task> RunBatchAsync(IEnumerable paths) + { + var list = paths.Distinct(StringComparer.Ordinal).ToList(); + var result = list.ToDictionary(p => p, _ => false, StringComparer.Ordinal); + + ProcessStartInfo psi; + if (_useShell) + { + var inner = "filefrag -v -- " + string.Join(' ', list.Select(QuoteSingle)); + psi = new ProcessStartInfo + { + FileName = "/bin/bash", + Arguments = "-c " + QuoteDouble(inner), + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + WorkingDirectory = "/" + }; + } + else + { + psi = new ProcessStartInfo + { + FileName = "filefrag", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + psi.ArgumentList.Add("-v"); + psi.ArgumentList.Add("--"); + foreach (var p in list) psi.ArgumentList.Add(p); + } + + using var proc = Process.Start(psi) ?? throw new InvalidOperationException("Failed to start filefrag"); + var stdoutTask = proc.StandardOutput.ReadToEndAsync(_cts.Token); + var stderrTask = proc.StandardError.ReadToEndAsync(_cts.Token); + await Task.WhenAll(stdoutTask, stderrTask).ConfigureAwait(false); + try + { + await proc.WaitForExitAsync(_cts.Token).ConfigureAwait(false); + } + catch (Exception ex) + { + _log.LogWarning(ex, "Error in the batch frag service. proc = {proc}", proc); + } + + var stdout = await stdoutTask.ConfigureAwait(false); + var stderr = await stderrTask.ConfigureAwait(false); + + if (proc.ExitCode != 0 && !string.IsNullOrWhiteSpace(stderr)) + _log.LogTrace("filefrag exited {code}: {err}", proc.ExitCode, stderr.Trim()); + + ParseFilefrag(stdout, result); + return result; + } + + /// + /// Parsing the string given from the File Frag command into mapping + /// + /// Output of the process from the File Frag + /// Mapping of the processed files + private static void ParseFilefrag(string output, Dictionary map) + { + var reHeaderColon = ColonRegex(); + var reHeaderSize = SizeRegex(); + + string? current = null; + using var sr = new StringReader(output); + for (string? line = sr.ReadLine(); line != null; line = sr.ReadLine()) + { + var m1 = reHeaderColon.Match(line); + if (m1.Success) { current = m1.Groups[1].Value; continue; } + + var m2 = reHeaderSize.Match(line); + if (m2.Success) { current = m2.Groups[1].Value; continue; } + + if (current is not null && line.Contains("flags:", StringComparison.OrdinalIgnoreCase) && + line.Contains("compressed", StringComparison.OrdinalIgnoreCase) && map.ContainsKey(current)) + { + map[current] = true; + } + } + } + + private static string QuoteSingle(string s) => "'" + s.Replace("'", "'\\''", StringComparison.Ordinal) + "'"; + private static string QuoteDouble(string s) => "\"" + s.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal).Replace("$", "\\$", StringComparison.Ordinal).Replace("`", "\\`", StringComparison.Ordinal) + "\""; + + /// + /// Regex of the File Size return on the Linux/Wine systems, giving back the amount + /// + /// Regex of the File Size + [GeneratedRegex(@"^File size of (/.+?) is ", RegexOptions.ExplicitCapture | RegexOptions.CultureInvariant,matchTimeoutMilliseconds: 500)] + private static partial Regex SizeRegex(); + + /// + /// Regex on colons return on the Linux/Wine systems + /// + /// Regex of the colons in the given path + [GeneratedRegex(@"^(/.+?):\s", RegexOptions.ExplicitCapture | RegexOptions.CultureInvariant, matchTimeoutMilliseconds: 500)] + private static partial Regex ColonRegex(); + + public void Dispose() + { + _ch.Writer.TryComplete(); + _cts.Cancel(); + try + { + _worker.Wait(TimeSpan.FromSeconds(2), _cts.Token); + } + catch + { + // Ignore the catch in dispose + } + _cts.Dispose(); + } + } +} \ No newline at end of file From e9082ab8d0c5f79323eed1037ee396e05ab9c410 Mon Sep 17 00:00:00 2001 From: cake Date: Fri, 7 Nov 2025 06:07:34 +0100 Subject: [PATCH 28/28] forget semicolomn.. --- LightlessSync/FileCache/FileCompactor.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LightlessSync/FileCache/FileCompactor.cs b/LightlessSync/FileCache/FileCompactor.cs index e6505b9..be89c1f 100644 --- a/LightlessSync/FileCache/FileCompactor.cs +++ b/LightlessSync/FileCache/FileCompactor.cs @@ -947,7 +947,7 @@ public sealed class FileCompactor : IDisposable public void Dispose() { - _fragBatch?.Dispose() + _fragBatch?.Dispose(); _compactionQueue.Writer.TryComplete(); _compactionCts.Cancel();