From 7c9ea505b0d2e9561654bb357be53549cc672c2c Mon Sep 17 00:00:00 2001 From: Amatsugu Date: Sun, 25 Jan 2026 19:13:58 -0500 Subject: [PATCH] added metadata loading to scanner --- AZKiServer/AZKi Server.csproj | 6 +- AZKiServer/Models/MediaEntry.cs | 14 ++-- AZKiServer/Protos/types.proto | 18 +++++ AZKiServer/Services/FileScannerService.cs | 81 +++++++++++++++++++++-- AZKiServer/Services/MediaService.cs | 37 +++++++++-- 5 files changed, 137 insertions(+), 19 deletions(-) diff --git a/AZKiServer/AZKi Server.csproj b/AZKiServer/AZKi Server.csproj index 9ad9d00..04a42b5 100644 --- a/AZKiServer/AZKi Server.csproj +++ b/AZKiServer/AZKi Server.csproj @@ -12,10 +12,12 @@ - + + - + + diff --git a/AZKiServer/Models/MediaEntry.cs b/AZKiServer/Models/MediaEntry.cs index caa7c55..3f7eee0 100644 --- a/AZKiServer/Models/MediaEntry.cs +++ b/AZKiServer/Models/MediaEntry.cs @@ -10,15 +10,18 @@ namespace AZKiServer.Models; public partial class MediaEntry { + public const int CUR_VERSION = 1; [BsonId] public ObjectId Id { get; set; } public int Version { get; set; } public MediaType Type { get; set; } public required string Filepath { get; set; } public DateTime Date { get; set; } + public DateTime EndDate { get; set; } public int CameraId { get; set; } + public required MediaMetadata Metadata { get; set; } - public static Maybe Parse(string relativePath) + public static Maybe Parse(string relativePath, MediaMetadata metadata) { var filename = Path.GetFileName(relativePath); @@ -32,13 +35,15 @@ public partial class MediaEntry var cam = match.Groups["cam"]; var date = match.Groups["date"]; var ext = match.Groups["ext"]; - + var startDate = ParseDate(date.Value); return new MediaEntry { - Version = 0, + Version = CUR_VERSION, CameraId = int.Parse(cam.Value), Filepath = relativePath, - Date = ParseDate(date.Value), + Date = startDate, + EndDate = startDate.AddSeconds(metadata.Duration), + Metadata = metadata, Type = ext.Value switch { "mp4" => MediaType.Video, @@ -68,6 +73,7 @@ public partial class MediaEntry private static partial Regex FileParser(); } +public record MediaMetadata(int Duration, int Height, int Width); [Flags] public enum MediaType diff --git a/AZKiServer/Protos/types.proto b/AZKiServer/Protos/types.proto index 8c30a20..684e97a 100644 --- a/AZKiServer/Protos/types.proto +++ b/AZKiServer/Protos/types.proto @@ -30,4 +30,22 @@ message MediaEntry { string filePath = 4; int32 cameraId = 5; google.protobuf.Timestamp date = 6; + MediaMetadata metadata = 7; +} + +message MediaMetadata { + int32 duration = 1; + int32 height = 2; + int32 width = 3; +} + +message PlaybackInfo{ + google.protobuf.Timestamp date = 1; + repeated MediaChannel channels = 2; +} + +message MediaChannel{ + int32 cameraId = 1; + repeated MediaEntry images = 2; + repeated MediaEntry videos = 3; } \ No newline at end of file diff --git a/AZKiServer/Services/FileScannerService.cs b/AZKiServer/Services/FileScannerService.cs index d766506..ccc989c 100644 --- a/AZKiServer/Services/FileScannerService.cs +++ b/AZKiServer/Services/FileScannerService.cs @@ -1,11 +1,19 @@  using AZKiServer.Models; +using FFMpegCore; + +using MaybeError; +using MaybeError.Errors; + +using SixLabors.ImageSharp; + namespace AZKiServer.Services; public class FileScannerService(MediaService mediaService, IConfiguration config, ILogger logger) : IHostedService, IDisposable { private Timer? _timer; + private bool _isRunning; public void Dispose() { @@ -19,7 +27,11 @@ public class FileScannerService(MediaService mediaService, IConfiguration config return Task.CompletedTask; _timer = new Timer((_) => { + if (_isRunning) + return; + _isRunning = true; ScanFilesAsync(path).Wait(); + _isRunning = false; }, null, TimeSpan.FromMinutes(0), TimeSpan.FromHours(1)); return Task.CompletedTask; } @@ -31,37 +43,94 @@ public class FileScannerService(MediaService mediaService, IConfiguration config } - private async Task ScanFilesAsync(string path) + private async Task ScanFilesAsync(string path, CancellationToken cancellationToken = default) { logger.LogInformation("Scanning Files"); try { var files = Directory.GetFiles(path, "*", SearchOption.AllDirectories); - var existingFiles = await mediaService.GetExistingFilePathsAsync(); + var existingFiles = await mediaService.GetExistingFilePathsAsync(cancellationToken); var entries = new List(); + var upgradeEntries = new List(); foreach (var filePath in files) { + if (cancellationToken.IsCancellationRequested) + break; var relativePath = Path.GetRelativePath(path, filePath); if (relativePath[0] == '.') //Ignore hidden folders continue; - if (existingFiles.Contains(relativePath)) + var isUpgrade = false; + if (existingFiles.TryGetValue(relativePath, out var version)) + { + if(version < MediaEntry.CUR_VERSION) + continue; + isUpgrade = true; + } + var metadata = ReadMetadata(filePath); + if(metadata.HasError) + { + logger.LogError(metadata.Error.GetException(), $"Failed to get metadata for file: {filePath}"); continue; - var entry = MediaEntry.Parse(relativePath); + } + var entry = MediaEntry.Parse(relativePath, metadata); if(entry.HasError) { logger.LogError(entry.Error.GetException(), "Failed to parse file data"); continue; } - entries.Add(entry); + if(isUpgrade) + upgradeEntries.Add(entry); + else + entries.Add(entry); } + cancellationToken.ThrowIfCancellationRequested(); if(entries.Count > 0) { - await mediaService.AddMediaBulkAsync(entries); + await mediaService.AddMediaBulkAsync(entries, cancellationToken); logger.LogInformation("Added {count} file entries", entries.Count); } + if (upgradeEntries.Count > 0) + { + await mediaService.DeleteAllEntriesAsync(upgradeEntries.Select(e => e.Filepath), cancellationToken); + await mediaService.AddMediaBulkAsync(upgradeEntries, cancellationToken); + logger.LogInformation("Upgraded {count} file entries", entries.Count); + } } catch (Exception ex) { logger.LogError(ex, "Failed to read directory contents"); } } + + private static Maybe ReadMetadata(string filePath) + { + var ext = Path.GetExtension(filePath); + return ext switch + { + ".jpg" or ".png" or ".jpeg" => ReadImageMetadata(filePath), + ".mp4" => ReadVideoMetadata(filePath), + _ => throw new NotSupportedException($"Files of type {ext} are not supported") + }; + } + + private static Maybe ReadImageMetadata(string filePath) + { + try + { + var info = Image.Identify(filePath); + return new MediaMetadata(0, info.Height, info.Width); + } + catch (Exception ex) + { + return ex; + } + } + + + private static Maybe ReadVideoMetadata(string filePath) + { + var info = FFProbe.Analyse(filePath); + if (info.PrimaryVideoStream == null) + return new Error($"Could not find a primirary video stream in file."); + return new MediaMetadata((int)info.Duration.TotalSeconds, info.PrimaryVideoStream.Height, info.PrimaryVideoStream.Width); + } } diff --git a/AZKiServer/Services/MediaService.cs b/AZKiServer/Services/MediaService.cs index 4a5477d..1aa7e33 100644 --- a/AZKiServer/Services/MediaService.cs +++ b/AZKiServer/Services/MediaService.cs @@ -1,5 +1,7 @@ using AZKiServer.Models; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; using MongoDB.Driver; using System.Collections.Frozen; @@ -9,10 +11,15 @@ namespace AZKiServer.Services; public class MediaService(IMongoDatabase db) { public readonly IMongoCollection _entries = db.GetCollection("media"); - public async Task> GetExistingFilePathsAsync(CancellationToken cancellationToken = default) + + [BsonIgnoreExtraElements] + private record MediaInfo(string FilePath, int Version); + public async Task> GetExistingFilePathsAsync(CancellationToken cancellationToken = default) { - var files = await _entries.Find("{}").Project(m => m.Filepath).ToListAsync(cancellationToken); - return files.ToFrozenSet(); + var files = await _entries.Find("{}").As().ToListAsync(cancellationToken); + if (files.Count == 0) + return FrozenDictionary.Empty; + return files.ToFrozenDictionary(m => m.FilePath, m => m.Version); } public async Task AddMediaBulkAsync(List entries, CancellationToken cancellationToken = default) @@ -20,7 +27,7 @@ public class MediaService(IMongoDatabase db) await _entries.InsertManyAsync(entries, cancellationToken: cancellationToken); } - public async Task> GetEntriesInRangeAsync(MediaType mediaType, DateTime from, DateTime to) + public async Task> GetEntriesInRangeAsync(MediaType mediaType, DateTime from, DateTime to, CancellationToken cancellationToken = default) { var filter = Builders.Filter .And([ @@ -29,11 +36,25 @@ public class MediaService(IMongoDatabase db) Builders.Filter.Lte(m => m.Date, to), ]); - return _entries.Find(filter).ToList(); - - + return await _entries.Find(filter).ToListAsync(cancellationToken); } + public async Task DeleteAllEntriesAsync(IEnumerable ids, CancellationToken cancellationToken = default) + { + await _entries.DeleteManyAsync(e => ids.Contains(e.Id), cancellationToken); + } + + public async Task DeleteAllEntriesAsync(IEnumerable paths, CancellationToken cancellationToken = default) + { + await _entries.DeleteManyAsync(e => paths.Contains(e.Filepath), cancellationToken); + } + +#if DEBUG + public async Task DeleteAllEntriesAsync(CancellationToken cancellationToken = default) { + await _entries.DeleteManyAsync("{}", cancellationToken); + } +#endif + public class IndexCreation : BackgroundService { protected override Task ExecuteAsync(CancellationToken stoppingToken) @@ -41,4 +62,6 @@ public class MediaService(IMongoDatabase db) throw new NotImplementedException(); } } + + }