diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4b53fa3..3e49ae8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,4 +16,5 @@ jobs: - uses: actions/setup-dotnet@v6 with: dotnet-version: 10.0.x - - run: dotnet test MarkItDownWeb.slnx --configuration Release + - run: dotnet build MarkItDownWeb.slnx --configuration Release --warnaserror + - run: dotnet test MarkItDownWeb.slnx --configuration Release --no-build diff --git a/README.md b/README.md index ec2a8c7..623632c 100644 --- a/README.md +++ b/README.md @@ -1,94 +1,86 @@ # FileToMarkdown -Aplicación web local para convertir documentos a Markdown. Selecciona un archivo, revisa el resultado en el navegador y descarga un `.md` con el nombre del documento original. +Biblioteca local para convertir documentos a Markdown, buscar su contenido y consultar fragmentos con referencias a su archivo y sección de origen. -## Captura +**Primer uso en escritorio** -![Conversión de un documento y su vista previa Markdown con historial local](docs/screenshots/local-conversion-flow.png) +![Biblioteca local en escritorio](docs/screenshots/local-document-library.png) + +**Primer uso en móvil** + +![Biblioteca local en móvil](docs/screenshots/local-document-library-mobile.png) ## Funciones -- Interfaz web responsive con vista previa, copia y descarga del Markdown. -- Historial local de conversiones con búsqueda y reapertura de resultados. -- Conversión de PDF, DOCX, PPTX, XLSX, HTML y CSV mediante [Microsoft MarkItDown](https://github.com/microsoft/markitdown). -- Base de datos SQLite y archivos de resultado guardados en el equipo. -- Validación de extensión, tamaño máximo de 100 MiB y firma de archivos PDF/Office. +- Importa hasta 10 archivos por lote; cada archivo puede ocupar hasta 100 MiB. El estado de conversión se muestra por archivo y un error no detiene los demás. +- Convierte PDF, DOCX, PPTX, XLSX, HTML y CSV con [Microsoft MarkItDown](https://github.com/microsoft/markitdown). +- Previsualiza, copia y descarga Markdown. El historial local permite reabrir o borrar conversiones. +- Indexa el Markdown en fragmentos de hasta 1.200 caracteres y busca nombres de archivo y contenido en SQLite, sin distinguir mayúsculas ni acentos. +- Consulta la biblioteca con respuestas extractivas: muestra el fragmento coincidente y cita el archivo y la sección. Si no encuentra evidencia, lo indica explícitamente. -La aplicación se ejecuta en tu equipo. Los archivos se envían al proceso web local y a MarkItDown; esta configuración no proporciona autenticación, así que mantenla accesible solo desde tu propio equipo y no la expongas a una red pública. +La búsqueda y las consultas son locales y no necesitan claves ni conexión a un proveedor de IA. `IAnswerGenerator` permite añadir otro generador, pero esta versión solo registra el generador extractivo local y no envía documentos fuera del equipo. ## Requisitos - [.NET SDK 10](https://dotnet.microsoft.com/download/dotnet/10.0) -- Python y `pip` +- Python 3.12 o posterior y `pip` para Microsoft MarkItDown -## Ejecutar en local +## Instalación y ejecución -Desde la raíz del repositorio: +Desde la raíz del repositorio, crea un entorno e instala MarkItDown con sus dependencias opcionales: -```bash -python -m venv .venv +```powershell +py -3.12 -m venv .venv +.\.venv\Scripts\python.exe -m pip install --upgrade pip +.\.venv\Scripts\python.exe -m pip install 'markitdown[pdf,docx,pptx,xlsx]' ``` -Activa el entorno virtual y luego instala MarkItDown: +Antes de iniciar la aplicación, configura el ejecutable de Python del entorno virtual. Así MarkItDown y sus dependencias quedan aislados del Python global: -```bash -# Windows PowerShell -.venv\Scripts\Activate.ps1 -python -m pip install --upgrade pip -python -m pip install 'markitdown[all]' +```powershell +$env:MarkItDown__PythonPath = (Resolve-Path .venv\Scripts\python.exe).Path +dotnet run --project src/Web/MarkItDownWeb.Web.csproj --launch-profile http ``` -```bash -# Linux / macOS -source .venv/bin/activate -python -m pip install --upgrade pip -python -m pip install 'markitdown[all]' -``` +Abre [http://localhost:5241](http://localhost:5241). Si prefieres usar una instalación global, establece `MarkItDown__PythonPath` con la ruta completa a ese ejecutable antes de iniciar la app. -Inicia la interfaz: +## Uso -```bash -dotnet run --project src/Web/MarkItDownWeb.Web.csproj --launch-profile http -``` +1. Arrastra archivos a la zona de carga o pulsa **Elegir documentos**. Se procesan de uno en uno; el estado o error aparece junto al nombre de cada archivo. +2. Busca por nombre o contenido desde la barra superior. La lista central muestra tipo, fecha y estado. +3. Selecciona un documento para abrir su Markdown en el panel de detalle. Desde ahí puedes copiar, descargar o borrar. +4. Usa **Pregunta a la biblioteca** para consultar fragmentos relacionados con sus citas. Sin coincidencias, la app informa que no encontró respaldo. En móvil, el detalle queda debajo de la lista. -Abre [http://localhost:5241](http://localhost:5241). La ruta del ejecutable de Python se puede configurar con `MarkItDown__PythonPath`; por defecto se usa `python` del entorno activo. Por ejemplo, en PowerShell: +## Almacenamiento y privacidad -```powershell -$env:MarkItDown__PythonPath = (Get-Command python).Source -dotnet run --project src/Web/MarkItDownWeb.Web.csproj --launch-profile http -``` +SQLite conserva los metadatos y los fragmentos indexados. Los resultados Markdown se guardan localmente en `Storage/Markdown`; el documento original solo existe en un archivo temporal durante la conversión y se elimina al terminar. Al borrar una conversión también se borran sus fragmentos por clave foránea y su archivo Markdown. Al iniciar, la app indexa conversiones existentes que todavía no tengan fragmentos. -## Comprobar cambios +Las rutas se configuran en `src/Web/appsettings.json` (`ConnectionStrings:DefaultConnection`, `Storage:UploadsFolder` y `Storage:MarkdownFolder`). La aplicación no incluye autenticación; mantenla accesible solo desde tu equipo y no la expongas a una red pública. -```bash -dotnet test MarkItDownWeb.slnx --configuration Release -``` +## Arquitectura -GitHub Actions ejecuta la misma comprobación al abrir o actualizar un pull request y al hacer push a `master`. +```text +src/ + Domain/ Conversiones y fragmentos indexados + Application/ Conversión, división Markdown, búsqueda y respuesta extractiva + Infrastructure/ MarkItDown, almacenamiento local y persistencia SQLite + Web/ Interfaz y endpoints ASP.NET Core MVC +tests/ Pruebas de conversión, búsqueda, citas, validación y borrado +``` -## Uso +La tabla `DocumentChunks` se crea al iniciar. Esto conserva las bases SQLite existentes sin modificar su tabla de historial; los fragmentos se eliminan en cascada al borrar una conversión. -1. Elige un PDF, DOCX, PPTX, XLSX, HTML o CSV de hasta 100 MiB. -2. Pulsa **Convertir a Markdown** y revisa el contenido generado. -3. Cópialo o descárgalo como `.md`. -4. Abre una conversión anterior desde el historial. Los datos permanecen en el equipo. +## Formatos y límites verificados -## Datos locales +La aplicación acepta `.pdf`, `.docx`, `.pptx`, `.xlsx`, `.html` y `.csv`, limita cada archivo a 100 MiB y rechaza extensiones no permitidas. También comprueba las firmas PDF/ZIP de PDF y documentos Office; HTML y CSV no tienen una comprobación de firma. MarkItDown y sus dependencias instaladas determinan qué contenido puede extraerse de cada archivo. -La base de datos SQLite se crea automáticamente. Los archivos temporales se eliminan al terminar la conversión; el historial conserva los resultados Markdown y los metadatos. Las carpetas se pueden cambiar desde `src/Web/appsettings.json` (`Storage:UploadsFolder`, `Storage:MarkdownFolder` y `ConnectionStrings:DefaultConnection`). No se conserva el documento original después de convertirlo. +Las pruebas del repositorio verifican validación, indexación, búsqueda, citas, abstención y eliminación con un conversor simulado. También se comprobó manualmente la conversión e indexación de archivos PDF, DOCX, PPTX, XLSX, HTML y CSV. La extracción depende de MarkItDown; los PDF no reciben número de página de forma uniforme. La app cita los marcadores de página que existan en el Markdown y, en su ausencia, cita encabezados/secciones o `Contenido`. La búsqueda es léxica; no interpreta sinónimos ni genera resúmenes. No incluye OCR ni cuentas multiusuario. -## Estructura +## Verificación -```text -src/ - Domain/ Entidades y tipos de conversión - Application/ Flujo de conversión e interfaces - Infrastructure/ MarkItDown, almacenamiento y SQLite - Web/ Interfaz ASP.NET Core MVC +```powershell +dotnet build MarkItDownWeb.slnx --configuration Release --warnaserror +dotnet test MarkItDownWeb.slnx --configuration Release ``` -## Limitaciones conocidas - -- La calidad del Markdown depende del formato y del contenido del documento original. -- MarkItDown puede necesitar dependencias adicionales para ciertos formatos; consulta su documentación si la instalación completa no está disponible en tu plataforma. -- Esta versión está pensada para uso local individual; no incluye cuentas, aislamiento entre usuarios ni almacenamiento en la nube. +GitHub Actions ejecuta build con warnings como errores y las pruebas al abrir o actualizar un pull request y al hacer push a `master`. diff --git a/docs/screenshots/local-document-library-mobile.png b/docs/screenshots/local-document-library-mobile.png new file mode 100644 index 0000000..d835ffa Binary files /dev/null and b/docs/screenshots/local-document-library-mobile.png differ diff --git a/docs/screenshots/local-document-library.png b/docs/screenshots/local-document-library.png new file mode 100644 index 0000000..7003344 Binary files /dev/null and b/docs/screenshots/local-document-library.png differ diff --git a/src/Application/DTOs/LibraryAnswerDto.cs b/src/Application/DTOs/LibraryAnswerDto.cs new file mode 100644 index 0000000..dd255af --- /dev/null +++ b/src/Application/DTOs/LibraryAnswerDto.cs @@ -0,0 +1,8 @@ +namespace MarkItDownWeb.Application.DTOs; + +public sealed class LibraryAnswerDto +{ + public string Answer { get; init; } = string.Empty; + public bool HasEvidence { get; init; } + public IReadOnlyList Citations { get; init; } = []; +} diff --git a/src/Application/DTOs/LibrarySearchHit.cs b/src/Application/DTOs/LibrarySearchHit.cs new file mode 100644 index 0000000..8388da4 --- /dev/null +++ b/src/Application/DTOs/LibrarySearchHit.cs @@ -0,0 +1,10 @@ +namespace MarkItDownWeb.Application.DTOs; + +public sealed class LibrarySearchHit +{ + public Guid ConversionId { get; init; } + public string FileName { get; init; } = string.Empty; + public string Section { get; init; } = string.Empty; + public string Content { get; init; } = string.Empty; + public int MatchCount { get; init; } +} diff --git a/src/Application/Interfaces/IAnswerGenerator.cs b/src/Application/Interfaces/IAnswerGenerator.cs new file mode 100644 index 0000000..c2ba1ce --- /dev/null +++ b/src/Application/Interfaces/IAnswerGenerator.cs @@ -0,0 +1,8 @@ +using MarkItDownWeb.Application.DTOs; + +namespace MarkItDownWeb.Application.Interfaces; + +public interface IAnswerGenerator +{ + LibraryAnswerDto CreateAnswer(IReadOnlyList evidence); +} diff --git a/src/Application/Interfaces/IConversionRepository.cs b/src/Application/Interfaces/IConversionRepository.cs index 077faf8..f121c5f 100644 --- a/src/Application/Interfaces/IConversionRepository.cs +++ b/src/Application/Interfaces/IConversionRepository.cs @@ -1,3 +1,4 @@ +using MarkItDownWeb.Application.DTOs; using MarkItDownWeb.Domain.Entities; namespace MarkItDownWeb.Application.Interfaces; @@ -7,6 +8,10 @@ public interface IConversionRepository Task AddAsync(Conversion conversion, CancellationToken ct = default); Task GetByIdAsync(Guid id, CancellationToken ct = default); Task> GetHistoryAsync(string? search, DateTime? date, CancellationToken ct = default); + Task> GetCompletedWithoutIndexAsync(CancellationToken ct = default); + Task AddChunksAsync(IReadOnlyCollection chunks, CancellationToken ct = default); + Task BackfillSearchIndexesAsync(CancellationToken ct = default); Task DeleteAsync(Guid id, CancellationToken ct = default); + Task> SearchContentAsync(IReadOnlyCollection terms, int limit, CancellationToken ct = default); Task SaveChangesAsync(CancellationToken ct = default); } diff --git a/src/Application/Interfaces/IFileStorageService.cs b/src/Application/Interfaces/IFileStorageService.cs index ed35e39..d98b5e3 100644 --- a/src/Application/Interfaces/IFileStorageService.cs +++ b/src/Application/Interfaces/IFileStorageService.cs @@ -6,4 +6,5 @@ public interface IFileStorageService Task SaveMarkdownAsync(Guid conversionId, string markdown, CancellationToken ct = default); Task ReadMarkdownAsync(string markdownFilePath, CancellationToken ct = default); void DeleteTemporary(string filePath); + void DeleteMarkdown(string filePath); } diff --git a/src/Application/Services/ConversionService.cs b/src/Application/Services/ConversionService.cs index 12a38ee..caffb23 100644 --- a/src/Application/Services/ConversionService.cs +++ b/src/Application/Services/ConversionService.cs @@ -50,6 +50,7 @@ public async Task ConvertAsync(Stream fileStream, string fi var markdownPath = await _fileStorageService.SaveMarkdownAsync(conversion.Id, markdown, ct); conversion.MarkAsCompleted(markdownPath, stopwatch.Elapsed.TotalSeconds); + conversion.Chunks.AddRange(MarkdownChunker.Create(conversion.Id, markdown, conversion.FileName)); await _repository.AddAsync(conversion, ct); await _repository.SaveChangesAsync(ct); @@ -112,7 +113,42 @@ public Task> GetHistoryAsync(string? search, DateTim Status = c.Status.ToString() }).ToList(), ct); - public Task DeleteAsync(Guid id, CancellationToken ct = default) => _repository.DeleteAsync(id, ct); + public async Task IndexExistingAsync(CancellationToken ct = default) + { + var conversions = await _repository.GetCompletedWithoutIndexAsync(ct); + foreach (var conversion in conversions) + { + if (string.IsNullOrWhiteSpace(conversion.MarkdownFilePath)) + continue; + + try + { + var markdown = await _fileStorageService.ReadMarkdownAsync(conversion.MarkdownFilePath, ct); + var chunks = MarkdownChunker.Create(conversion.Id, markdown, conversion.FileName); + await _repository.AddChunksAsync(chunks, ct); + } + catch (FileNotFoundException) + { + // A missing Markdown file must not stop the local app from starting. + } + } + + await _repository.BackfillSearchIndexesAsync(ct); + await _repository.SaveChangesAsync(ct); + } + + public async Task DeleteAsync(Guid id, CancellationToken ct = default) + { + var conversion = await _repository.GetByIdAsync(id, ct); + if (conversion is null) + return; + + await _repository.DeleteAsync(id, ct); + await _repository.SaveChangesAsync(ct); + + if (!string.IsNullOrWhiteSpace(conversion.MarkdownFilePath)) + _fileStorageService.DeleteMarkdown(conversion.MarkdownFilePath); + } private static void ValidateFile(string fileName, long fileSize) { diff --git a/src/Application/Services/LibraryService.cs b/src/Application/Services/LibraryService.cs new file mode 100644 index 0000000..ef0447e --- /dev/null +++ b/src/Application/Services/LibraryService.cs @@ -0,0 +1,58 @@ +using System.Text.RegularExpressions; +using MarkItDownWeb.Application.DTOs; +using MarkItDownWeb.Application.Interfaces; + +namespace MarkItDownWeb.Application.Services; + +public sealed partial class LibraryService +{ + private const int MaxQueryLength = 300; + private static readonly HashSet StopWords = new(StringComparer.OrdinalIgnoreCase) + { + "a", "an", "and", "are", "as", "at", "be", "by", "do", "does", "for", "from", "how", "in", "is", "it", "of", "on", "or", "the", "to", "was", "were", "what", "when", "where", "which", "who", "why", + "con", "cuál", "cuáles", "cual", "cuales", "cómo", "como", "de", "del", "el", "ella", "en", "es", "esta", "este", "la", "las", "lo", "los", "más", "mas", "para", "por", "qué", "que", "se", "sobre", "son", "un", "una", "y" + }; + private readonly IConversionRepository _repository; + private readonly IAnswerGenerator _answerGenerator; + + public LibraryService(IConversionRepository repository, IAnswerGenerator answerGenerator) + { + _repository = repository; + _answerGenerator = answerGenerator; + } + + [GeneratedRegex(@"[\p{L}\p{N}]{2,}", RegexOptions.CultureInvariant)] + private static partial Regex TermRegex(); + + public async Task> SearchAsync(string? query, CancellationToken ct = default) + { + var terms = GetTerms(query); + return terms.Length == 0 + ? [] + : await _repository.SearchContentAsync(terms, 20, ct); + } + + public async Task AskAsync(string? question, CancellationToken ct = default) + { + var terms = GetTerms(question); + if (terms.Length == 0) + return _answerGenerator.CreateAnswer([]); + + var evidence = await _repository.SearchContentAsync(terms, 8, ct); + return _answerGenerator.CreateAnswer(evidence); + } + + private static string[] GetTerms(string? query) + { + if (string.IsNullOrWhiteSpace(query)) + return []; + + var normalizedQuery = SearchTextNormalizer.NormalizeTerm(query[..Math.Min(query.Length, MaxQueryLength)]); + return TermRegex().Matches(normalizedQuery) + .Select(match => match.Value) + .Where(term => !StopWords.Contains(term)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .Take(8) + .ToArray(); + } +} diff --git a/src/Application/Services/LocalExtractiveAnswerGenerator.cs b/src/Application/Services/LocalExtractiveAnswerGenerator.cs new file mode 100644 index 0000000..0d93caa --- /dev/null +++ b/src/Application/Services/LocalExtractiveAnswerGenerator.cs @@ -0,0 +1,27 @@ +using MarkItDownWeb.Application.DTOs; +using MarkItDownWeb.Application.Interfaces; + +namespace MarkItDownWeb.Application.Services; + +public sealed class LocalExtractiveAnswerGenerator : IAnswerGenerator +{ + public LibraryAnswerDto CreateAnswer(IReadOnlyList evidence) + { + if (evidence.Count == 0) + { + return new LibraryAnswerDto + { + Answer = "No encontré contenido que respalde una respuesta en los documentos de la biblioteca.", + HasEvidence = false + }; + } + + var citations = evidence.Take(4).ToArray(); + return new LibraryAnswerDto + { + Answer = $"El fragmento más relevante dice: «{citations[0].Content}»", + HasEvidence = true, + Citations = citations + }; + } +} diff --git a/src/Application/Services/MarkdownChunker.cs b/src/Application/Services/MarkdownChunker.cs new file mode 100644 index 0000000..5aa2f18 --- /dev/null +++ b/src/Application/Services/MarkdownChunker.cs @@ -0,0 +1,101 @@ +using System.Text.RegularExpressions; +using MarkItDownWeb.Domain.Entities; + +namespace MarkItDownWeb.Application.Services; + +internal static partial class MarkdownChunker +{ + private const int MaxChunkLength = 1200; + + [GeneratedRegex(@"^(#{1,6})\s+(.+?)\s*#*\s*$", RegexOptions.Multiline | RegexOptions.CultureInvariant)] + private static partial Regex HeadingRegex(); + + [GeneratedRegex(@"(?:|^\s*#{1,6}\s+(?:Page|Página)\s+(\d+)\s*$)", RegexOptions.Multiline | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex PageRegex(); + + public static IReadOnlyList Create(Guid conversionId, string markdown, string fileName) + { + var chunks = new List(); + var headings = new List<(int Level, string Text)>(); + var lines = markdown.Replace("\r\n", "\n", StringComparison.Ordinal).Split('\n'); + var buffer = new List(); + var section = "Contenido"; + string? page = null; + var ordinal = 0; + + void Flush() + { + var text = string.Join('\n', buffer).Trim(); + buffer.Clear(); + if (text.Length == 0) + return; + + foreach (var part in SplitText(text, MaxChunkLength)) + { + chunks.Add(new DocumentChunk + { + ConversionId = conversionId, + Ordinal = ordinal++, + Section = section.Length <= 500 ? section : section[..500], + Content = part, + SearchIndex = SearchTextNormalizer.CreateIndex(fileName, part) + }); + } + } + + foreach (var line in lines) + { + var pageMatch = PageRegex().Match(line); + if (pageMatch.Success) + { + Flush(); + var pageNumber = pageMatch.Groups[1].Success ? pageMatch.Groups[1].Value : pageMatch.Groups[2].Value; + page = $"Página {pageNumber}"; + headings.Clear(); + section = page; + continue; + } + + var headingMatch = HeadingRegex().Match(line); + if (headingMatch.Success) + { + Flush(); + var level = headingMatch.Groups[1].Length; + var title = headingMatch.Groups[2].Value.Trim(); + while (headings.Count > 0 && headings[^1].Level >= level) + headings.RemoveAt(headings.Count - 1); + headings.Add((level, title)); + var headingPath = string.Join(" › ", headings.Select(item => item.Text)); + section = page is null ? headingPath : $"{page} › {headingPath}"; + continue; + } + + if (string.IsNullOrWhiteSpace(line) && buffer.Count > 0) + { + Flush(); + continue; + } + + buffer.Add(line); + } + + Flush(); + return chunks; + } + + private static IEnumerable SplitText(string text, int maxLength) + { + while (text.Length > maxLength) + { + var splitAt = text.LastIndexOf(' ', maxLength); + if (splitAt < maxLength / 2) + splitAt = maxLength; + + yield return text[..splitAt].Trim(); + text = text[splitAt..].Trim(); + } + + if (text.Length > 0) + yield return text; + } +} diff --git a/src/Application/Services/SearchTextNormalizer.cs b/src/Application/Services/SearchTextNormalizer.cs new file mode 100644 index 0000000..87595f5 --- /dev/null +++ b/src/Application/Services/SearchTextNormalizer.cs @@ -0,0 +1,41 @@ +using System.Globalization; +using System.Text; +using System.Text.RegularExpressions; + +namespace MarkItDownWeb.Application.Services; + +public static partial class SearchTextNormalizer +{ + [GeneratedRegex(@"[\p{L}\p{N}]{2,}", RegexOptions.CultureInvariant)] + private static partial Regex WordRegex(); + + public static string NormalizeTerm(string value) => Normalize(value); + + public static string CreateIndex(params string?[] values) + { + var tokens = values + .Where(value => !string.IsNullOrWhiteSpace(value)) + .SelectMany(value => WordRegex().Matches(Normalize(value!)).Select(match => match.Value)); + + return $" {string.Join(' ', tokens)} "; + } + + private static string Normalize(string value) + { + var builder = new StringBuilder(value.Length); + foreach (var rune in value.Normalize(NormalizationForm.FormD).EnumerateRunes()) + { + var category = Rune.GetUnicodeCategory(rune); + if (category is UnicodeCategory.NonSpacingMark + or UnicodeCategory.SpacingCombiningMark + or UnicodeCategory.EnclosingMark) + { + continue; + } + + builder.Append(Rune.ToLowerInvariant(rune).ToString()); + } + + return builder.ToString().Normalize(NormalizationForm.FormC); + } +} diff --git a/src/Domain/Entities/Conversion.cs b/src/Domain/Entities/Conversion.cs index 2ada337..1dc443b 100644 --- a/src/Domain/Entities/Conversion.cs +++ b/src/Domain/Entities/Conversion.cs @@ -12,6 +12,7 @@ public class Conversion public double? ConversionTimeSeconds { get; set; } public ConversionStatus Status { get; set; } = ConversionStatus.Pending; public string? ErrorMessage { get; set; } + public List Chunks { get; set; } = []; public void MarkAsCompleted(string markdownFilePath, double elapsedSeconds) { diff --git a/src/Domain/Entities/DocumentChunk.cs b/src/Domain/Entities/DocumentChunk.cs new file mode 100644 index 0000000..6587627 --- /dev/null +++ b/src/Domain/Entities/DocumentChunk.cs @@ -0,0 +1,12 @@ +namespace MarkItDownWeb.Domain.Entities; + +public class DocumentChunk +{ + public Guid Id { get; set; } = Guid.NewGuid(); + public Guid ConversionId { get; set; } + public int Ordinal { get; set; } + public string Section { get; set; } = "Contenido"; + public string Content { get; set; } = string.Empty; + public string SearchIndex { get; set; } = " "; + public Conversion Conversion { get; set; } = null!; +} diff --git a/src/Infrastructure/Persistence/AppDbContext.cs b/src/Infrastructure/Persistence/AppDbContext.cs index 370e228..677ca37 100644 --- a/src/Infrastructure/Persistence/AppDbContext.cs +++ b/src/Infrastructure/Persistence/AppDbContext.cs @@ -8,6 +8,7 @@ public class AppDbContext : DbContext public AppDbContext(DbContextOptions options) : base(options) { } public DbSet Conversions => Set(); + public DbSet DocumentChunks => Set(); protected override void OnModelCreating(ModelBuilder modelBuilder) { @@ -20,6 +21,19 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) entity.Property(c => c.Status).HasConversion().HasMaxLength(20); entity.Property(c => c.FileType).HasConversion().HasMaxLength(20); entity.HasIndex(c => c.CreatedAt); + entity.HasMany(c => c.Chunks) + .WithOne(chunk => chunk.Conversion) + .HasForeignKey(chunk => chunk.ConversionId) + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity(entity => + { + entity.HasKey(chunk => chunk.Id); + entity.Property(chunk => chunk.Section).IsRequired().HasMaxLength(500); + entity.Property(chunk => chunk.Content).IsRequired().HasMaxLength(1200); + entity.Property(chunk => chunk.SearchIndex).IsRequired().HasMaxLength(1500); + entity.HasIndex(chunk => new { chunk.ConversionId, chunk.Ordinal }).IsUnique(); }); } } diff --git a/src/Infrastructure/Persistence/ConversionRepository.cs b/src/Infrastructure/Persistence/ConversionRepository.cs index 9e3ec91..46cfdc6 100644 --- a/src/Infrastructure/Persistence/ConversionRepository.cs +++ b/src/Infrastructure/Persistence/ConversionRepository.cs @@ -1,6 +1,9 @@ +using MarkItDownWeb.Application.DTOs; using MarkItDownWeb.Application.Interfaces; using MarkItDownWeb.Domain.Entities; using Microsoft.EntityFrameworkCore; +using MarkItDownWeb.Application.Services; +using Microsoft.Data.Sqlite; namespace MarkItDownWeb.Infrastructure.Persistence; @@ -29,6 +32,62 @@ public async Task> GetHistoryAsync(string? search, DateTime? da return await query.OrderByDescending(c => c.CreatedAt).ToListAsync(ct); } + public Task> GetCompletedWithoutIndexAsync(CancellationToken ct = default) + => _context.Conversions + .Include(conversion => conversion.Chunks) + .Where(conversion => conversion.Status == ConversionStatus.Completed && !conversion.Chunks.Any()) + .ToListAsync(ct); + + public Task AddChunksAsync(IReadOnlyCollection chunks, CancellationToken ct = default) + => _context.DocumentChunks.AddRangeAsync(chunks, ct); + + public async Task BackfillSearchIndexesAsync(CancellationToken ct = default) + { + var chunks = await _context.DocumentChunks + .Include(chunk => chunk.Conversion) + .Where(chunk => chunk.SearchIndex == " ") + .ToListAsync(ct); + + foreach (var chunk in chunks) + chunk.SearchIndex = SearchTextNormalizer.CreateIndex(chunk.Conversion.FileName, chunk.Content); + } + + public async Task> SearchContentAsync( + IReadOnlyCollection terms, + int limit, + CancellationToken ct = default) + { + if (terms.Count == 0 || limit <= 0) + return []; + + var matchTerms = terms + .Select((_, index) => $"instr(chunk.\"SearchIndex\", $term{index}) > 0") + .ToArray(); + var score = string.Join(" + ", matchTerms.Select(condition => $"CASE WHEN {condition} THEN 1 ELSE 0 END")); + var where = string.Join(" OR ", matchTerms); + var sql = $""" + SELECT + chunk."ConversionId" AS "ConversionId", + conversion."FileName" AS "FileName", + chunk."Section" AS "Section", + chunk."Content" AS "Content", + ({score}) AS "MatchCount" + FROM "DocumentChunks" AS chunk + INNER JOIN "Conversions" AS conversion ON conversion."Id" = chunk."ConversionId" + WHERE conversion."Status" = 'Completed' AND ({where}) + ORDER BY "MatchCount" DESC, conversion."FileName" COLLATE NOCASE ASC, conversion."CreatedAt" DESC, chunk."Ordinal" ASC + LIMIT $limit; + """; + var parameters = terms + .Select((term, index) => (object)new SqliteParameter($"$term{index}", $" {term} ")) + .Append(new SqliteParameter("$limit", limit)) + .ToArray(); + + return await _context.Database + .SqlQueryRaw(sql, parameters) + .ToListAsync(ct); + } + public async Task DeleteAsync(Guid id, CancellationToken ct = default) { var entity = await _context.Conversions.FindAsync(new object[] { id }, ct); diff --git a/src/Infrastructure/Persistence/LocalLibrarySchema.cs b/src/Infrastructure/Persistence/LocalLibrarySchema.cs new file mode 100644 index 0000000..6848577 --- /dev/null +++ b/src/Infrastructure/Persistence/LocalLibrarySchema.cs @@ -0,0 +1,54 @@ +using Microsoft.EntityFrameworkCore; + +namespace MarkItDownWeb.Infrastructure.Persistence; + +public static class LocalLibrarySchema +{ + public static void EnsureCreated(AppDbContext context) + { + context.Database.EnsureCreated(); + context.Database.ExecuteSqlRaw(""" + CREATE TABLE IF NOT EXISTS "DocumentChunks" ( + "Id" TEXT NOT NULL CONSTRAINT "PK_DocumentChunks" PRIMARY KEY, + "ConversionId" TEXT NOT NULL, + "Ordinal" INTEGER NOT NULL, + "Section" TEXT NOT NULL, + "Content" TEXT NOT NULL, + "SearchIndex" TEXT NOT NULL DEFAULT ' ', + CONSTRAINT "FK_DocumentChunks_Conversions_ConversionId" + FOREIGN KEY ("ConversionId") REFERENCES "Conversions" ("Id") ON DELETE CASCADE + ); + """); + if (!HasColumn(context, "DocumentChunks", "SearchIndex")) + context.Database.ExecuteSqlRaw("""ALTER TABLE "DocumentChunks" ADD COLUMN "SearchIndex" TEXT NOT NULL DEFAULT ' ';"""); + + context.Database.ExecuteSqlRaw("""CREATE UNIQUE INDEX IF NOT EXISTS "IX_DocumentChunks_ConversionId_Ordinal" ON "DocumentChunks" ("ConversionId", "Ordinal");"""); + } + + private static bool HasColumn(AppDbContext context, string table, string column) + { + var connection = context.Database.GetDbConnection(); + var closeAfterRead = connection.State != System.Data.ConnectionState.Open; + if (closeAfterRead) + context.Database.OpenConnection(); + + try + { + using var command = connection.CreateCommand(); + command.CommandText = $"PRAGMA table_info(\"{table}\");"; + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + if (string.Equals(reader.GetString(1), column, StringComparison.Ordinal)) + return true; + } + + return false; + } + finally + { + if (closeAfterRead) + context.Database.CloseConnection(); + } + } +} diff --git a/src/Infrastructure/Services/FileStorageService.cs b/src/Infrastructure/Services/FileStorageService.cs index b6c67f9..9da40f6 100644 --- a/src/Infrastructure/Services/FileStorageService.cs +++ b/src/Infrastructure/Services/FileStorageService.cs @@ -42,4 +42,10 @@ public void DeleteTemporary(string filePath) if (File.Exists(filePath)) File.Delete(filePath); } + + public void DeleteMarkdown(string filePath) + { + if (File.Exists(filePath)) + File.Delete(filePath); + } } diff --git a/src/Web/Controllers/ConversionController.cs b/src/Web/Controllers/ConversionController.cs index a25a66f..3629161 100644 --- a/src/Web/Controllers/ConversionController.cs +++ b/src/Web/Controllers/ConversionController.cs @@ -1,4 +1,5 @@ using MarkItDownWeb.Application.Services; +using MarkItDownWeb.Application.DTOs; using MarkItDownWeb.Web.Models; using Microsoft.AspNetCore.Mvc; @@ -7,9 +8,13 @@ namespace MarkItDownWeb.Web.Controllers; public class ConversionController : Controller { private readonly ConversionService _conversionService; + private readonly LibraryService _libraryService; - public ConversionController(ConversionService conversionService) - => _conversionService = conversionService; + public ConversionController(ConversionService conversionService, LibraryService libraryService) + { + _conversionService = conversionService; + _libraryService = libraryService; + } [HttpGet] public IActionResult Index() => View(new ConversionViewModel()); @@ -55,6 +60,14 @@ public async Task Item(Guid id, CancellationToken ct) } } + [HttpGet] + public async Task Search(string? query, CancellationToken ct) + => Json(await _libraryService.SearchAsync(query, ct)); + + [HttpPost] + public async Task Ask([FromBody] LibraryQuestionRequest request, CancellationToken ct) + => Json(await _libraryService.AskAsync(request.Question, ct)); + [HttpPost] public async Task Delete(Guid id, CancellationToken ct) { @@ -62,3 +75,5 @@ public async Task Delete(Guid id, CancellationToken ct) return Ok(); } } + +public sealed record LibraryQuestionRequest(string? Question); diff --git a/src/Web/Program.cs b/src/Web/Program.cs index 41b9afd..9c451e0 100644 --- a/src/Web/Program.cs +++ b/src/Web/Program.cs @@ -15,13 +15,17 @@ builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); var app = builder.Build(); using (var scope = app.Services.CreateScope()) { var db = scope.ServiceProvider.GetRequiredService(); - db.Database.EnsureCreated(); + LocalLibrarySchema.EnsureCreated(db); + var conversionService = scope.ServiceProvider.GetRequiredService(); + await conversionService.IndexExistingAsync(); } app.UseStaticFiles(); diff --git a/src/Web/Views/Conversion/Index.cshtml b/src/Web/Views/Conversion/Index.cshtml index a2ba4c6..8c0ac28 100644 --- a/src/Web/Views/Conversion/Index.cshtml +++ b/src/Web/Views/Conversion/Index.cshtml @@ -1,59 +1,116 @@ @model MarkItDownWeb.Web.Models.ConversionViewModel @{ - ViewData["Title"] = "Convertir archivos a Markdown"; + ViewData["Title"] = "Biblioteca local - MarkItDown"; } -
-
-

CONVERSIÓN LOCAL

-

Convierte tus documentos a Markdown

-

Sube un archivo, revisa el resultado y descarga tu .md.

+
+
+
+

FILETOMARKDOWN · BIBLIOTECA LOCAL

+

Convierte documentos a Markdown

+

Convierte tus archivos y guarda los resultados localmente en este equipo.

+
+ Solo en este equipo
-
-
-
- - -
PDF, Word, PowerPoint, Excel, HTML o CSV · máximo 100 MB
-
- - -
-
+ + + +
+ +

Empieza con un documento

+

Suelta aquí hasta 10 archivos para convertirlos a Markdown y consultarlos desde este equipo.

+ + +

PDF, Word, PowerPoint, Excel, HTML o CSV · hasta 100 MiB por archivo

+
+ + -
-
+
-
+ @section Scripts { diff --git a/src/Web/wwwroot/css/site.css b/src/Web/wwwroot/css/site.css index 436c966..70b2e0e 100644 --- a/src/Web/wwwroot/css/site.css +++ b/src/Web/wwwroot/css/site.css @@ -1,118 +1,427 @@ html { font-size: 14px; + min-height: 100%; } @media (min-width: 768px) { - html { - font-size: 16px; - } + html { font-size: 16px; } } -.btn:focus, .btn:active:focus, .btn-link.nav-link:focus, .form-control:focus, .form-check-input:focus { - box-shadow: 0 0 0 0.1rem white, 0 0 0 0.25rem #258cfb; +body { + min-height: 100vh; + margin: 0; + background: #f5f7fb; + color: #172638; } -html { - position: relative; - min-height: 100%; +.navbar { + min-height: 58px; + border-bottom: 1px solid #e8edf3; + background: #fff; } -body { - margin-bottom: 60px; - background: #f5f7fa; - color: #17212f; +.navbar-brand { + color: #183454; + font-weight: 700; + letter-spacing: -.02em; } -.navbar { - background: #fff; +.library-shell { + width: min(1440px, calc(100% - 3rem)); + margin: 2.5rem auto 4rem; } -.conversion-shell { - max-width: 1440px; - margin: 2.25rem auto; +.library-heading { + display: flex; + align-items: flex-end; + justify-content: space-between; + gap: 2rem; + margin-bottom: 1.75rem; } -.page-heading h1 { - max-width: 760px; - font-size: clamp(1.8rem, 3vw, 2.5rem); +.library-heading h1 { + margin-bottom: .45rem; + font-size: clamp(1.8rem, 3vw, 2.4rem); font-weight: 700; - letter-spacing: -0.035em; + letter-spacing: -.04em; } .eyebrow { - color: #64748b; - font-size: .7rem; - font-weight: 700; - letter-spacing: .12em; + color: #718096; + font-size: .68rem; + font-weight: 750; + letter-spacing: .13em; +} + +.local-badge { + flex: 0 0 auto; + display: inline-flex; + align-items: center; + gap: .45rem; + padding: .48rem .75rem; + border: 1px solid #d5e9db; + border-radius: 999px; + background: #f1faf4; + color: #287346; + font-size: .76rem; + font-weight: 650; + white-space: nowrap; +} + +.library-toolbar { + display: flex; + align-items: center; + gap: 1rem; + padding: 1rem; + border: 1px solid #e4eaf1; + border-radius: 14px; + background: #fff; + box-shadow: 0 8px 28px rgb(15 23 42 / 3%); +} + +.global-search { + position: relative; + flex: 1; +} + +.global-search .form-control { + min-height: 44px; + padding-left: 2.7rem; + border-color: #e0e7ef; + background: #f9fbfd; +} + +.search-glyph { + position: absolute; + z-index: 1; + top: 50%; + left: .9rem; + color: #728198; + font-size: 1.45rem; + line-height: 1; + transform: translateY(-57%); +} + +.import-hint { + margin: .5rem 0 1.25rem 1rem; + color: #718096; +} + +.first-run-panel { + display: grid; + min-height: 340px; + margin: 0 auto 1rem; + padding: 3rem 1.5rem; + place-items: center; + align-content: center; + gap: .75rem; + border: 2px dashed #cbd7e5; + border-radius: 16px; + background: #fff; + color: #65768b; + text-align: center; + transition: border-color .15s ease, background-color .15s ease; +} + +.first-run-panel.is-dragging { + border-color: #0d6efd; + background: #f4f8ff; +} + +.first-run-icon { + display: grid; + width: 56px; + height: 56px; + place-items: center; + border-radius: 16px; + background: #edf4ff; + color: #2366bd; + font-size: 2rem; +} + +.first-run-panel h2 { margin: .35rem 0 0; color: #26394f; font-size: 1.25rem; font-weight: 650; } +.first-run-panel > p:not(.first-run-hint) { max-width: 440px; margin: 0; font-size: .9rem; line-height: 1.55; } +.first-run-panel .btn { min-height: 44px; margin-top: .25rem; } +.first-run-hint { margin-top: .2rem; font-size: .76rem; } +.first-run-status { max-width: 520px; color: #526781; font-size: .82rem; } + +#libraryToolbar[hidden], #supportedFormats[hidden], #workspaceLayout[hidden], .first-run-panel[hidden] { + display: none !important; +} + +.import-progress-global { margin: 0 0 1rem; } + +.library-layout { + align-items: flex-start; } .app-card { - border: 1px solid #e4e9f0; + overflow: hidden; + border: 1px solid #e4eaf1; border-radius: 14px; - box-shadow: 0 8px 28px rgb(15 23 42 / 4%); + background: #fff; + box-shadow: 0 8px 28px rgb(15 23 42 / 3%); +} + +.library-card, .detail-card, .ask-card { + padding: 1.2rem 1.25rem; +} + +.card-heading { + display: flex; + align-items: center; + justify-content: space-between; + gap: 1rem; + margin-bottom: 1rem; +} + +.count-pill { + display: inline-block; + margin-left: .35rem; + padding: .15rem .5rem; + border-radius: 99px; + background: #edf3fa; + color: #395774; + font-size: .75rem; + vertical-align: middle; } .status-text { - font-size: .9rem; + min-height: 1.3rem; + font-size: .78rem; + text-align: right; } -.markdown-output { - min-height: 360px; - max-height: 65vh; - overflow: auto; - margin: 0; - padding: 1rem; - border: 1px solid #e2e8f0; - border-radius: 9px; - background: #f8fafc; - color: #243247; - font: .875rem/1.6 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; - white-space: pre-wrap; - overflow-wrap: anywhere; +.document-table { + min-width: 660px; + border-top: 1px solid #edf1f5; + color: #304156; } -.history-list { - max-height: 65vh; - overflow-y: auto; +.document-table thead th { + padding: .75rem .55rem; + border-bottom-color: #e9eef4; + color: #78879a; + font-size: .67rem; + font-weight: 700; + letter-spacing: .06em; + text-transform: uppercase; + white-space: nowrap; } -.history-entry { - display: grid; - width: 100%; - gap: .25rem; - padding: .4rem 0; +.document-table tbody td { + padding: .8rem .55rem; + border-bottom-color: #f0f3f7; + vertical-align: middle; +} + +.document-table tbody tr:last-child td { border-bottom: 0; } + +.document-name-button { + max-width: 260px; + padding: 0; + overflow: hidden; border: 0; background: transparent; - color: inherit; + color: #183f68; + font-weight: 650; text-align: left; + text-overflow: ellipsis; + white-space: nowrap; } -.history-entry:not(:disabled):hover .history-file-name { +.document-name-button:not(:disabled):hover, +.evidence-open:hover { color: #0d6efd; + text-decoration: underline; } -.history-entry:disabled { - color: #94a3b8; - cursor: not-allowed; +.document-name-button:disabled { color: #8794a5; } + +.file-type { + display: inline-block; + padding: .22rem .42rem; + border: 1px solid #e4eaf1; + border-radius: 5px; + background: #f8fafc; + color: #68788d; + font-size: .65rem; + font-weight: 700; } -.history-file-name { - overflow: hidden; - font-weight: 600; - text-overflow: ellipsis; +.document-date { + color: #718096 !important; + font-size: .78rem; + white-space: nowrap; +} + +.document-state { + display: inline-flex; + align-items: center; + padding: .24rem .5rem; + border-radius: 99px; + font-size: .7rem; + font-weight: 650; white-space: nowrap; } -.history-date { - color: #64748b; +.document-state.is-complete { background: #edf8f0; color: #287346; } +.document-state.is-failed { background: #fff1f0; color: #b44740; } + +.document-actions { text-align: right; } + +.icon-action { + padding: .25rem .35rem; + border: 0; + background: transparent; + color: #8390a0; font-size: .75rem; } -.form-floating > .form-control-plaintext::placeholder, .form-floating > .form-control::placeholder { - color: var(--bs-secondary-color); - text-align: end; +.icon-action:hover { color: #aa3037; text-decoration: underline; } + +.empty-state { + padding: 2.5rem 1rem 2.75rem; + color: #68788d; + text-align: center; +} + +.empty-icon { + display: grid; + width: 44px; + height: 44px; + margin: 0 auto .75rem; + place-items: center; + border-radius: 12px; + background: #f1f5f9; + color: #6b7d91; + font-size: 1.4rem; +} + +.empty-state h3 { color: #26394f; font-size: 1rem; font-weight: 650; } +.empty-state p { max-width: 390px; margin: 0 auto 1rem; font-size: .85rem; line-height: 1.55; } + +.import-progress { + margin: 0 0 1rem; + padding: .75rem; + border: 1px solid #edf1f5; + border-radius: 9px; + background: #fafbfd; +} + +.import-results { list-style: none; padding: 0; } + +.import-result { + padding: .45rem .65rem; + border-left: 3px solid #9aabbd; + color: #596a7f; + font-size: .8rem; + overflow-wrap: anywhere; +} + +.import-result + .import-result { margin-top: .3rem; } +.import-result.is-success { border-color: #198754; color: #267347; } +.import-result.is-error { border-color: #dc3545; color: #a52d38; } + +.search-results-panel { + margin-top: 1rem; + padding-top: 1rem; + border-top: 1px solid #edf1f5; +} + +.library-results, .answer-panel { + max-height: 38vh; + overflow: auto; +} + +.evidence-item { padding: .8rem 0; border-color: #eef2f6; } +.evidence-open { + padding: 0; + border: 0; + background: transparent; + color: #1c4b78; + font-size: .82rem; + font-weight: 650; + text-align: left; +} + +.evidence-excerpt { + display: -webkit-box; + overflow: hidden; + color: #5a6a7e; + font-size: .8rem; + line-height: 1.5; + overflow-wrap: anywhere; + -webkit-box-orient: vertical; + -webkit-line-clamp: 4; +} + +.detail-column { display: grid; gap: 1rem; } + +.detail-card { min-width: 0; } +.detail-actions { display: flex; flex-wrap: wrap; justify-content: flex-end; gap: .35rem; } +.detail-actions .btn { white-space: nowrap; } +.detail-empty { display: grid; min-height: 240px; place-items: center; padding: 1rem; border: 1px dashed #dce4ed; border-radius: 9px; color: #8290a0; font-size: .85rem; text-align: center; } + +.markdown-output { + min-height: 240px; + max-height: 62vh; + overflow: auto; + margin: 0; + padding: 1rem; + border: 1px solid #e2e8f0; + border-radius: 9px; + background: #f8fafc; + color: #34445a; + font: .82rem/1.6 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + white-space: pre-wrap; + overflow-wrap: anywhere; +} + +.ask-card h2 { color: #24374d; } +.ask-form { display: flex; gap: .55rem; } +.ask-form .form-control { min-width: 0; } +.answer-panel { padding: .8rem; border-radius: 9px; background: #f8fafc; } +.answer-text { color: #24374d; font-size: .85rem; font-weight: 650; line-height: 1.5; } + +.btn:focus-visible, +.form-control:focus-visible, +.document-name-button:focus-visible, +.evidence-open:focus-visible, +.icon-action:focus-visible { + outline: 3px solid rgb(13 110 253 / 35%); + outline-offset: 2px; + box-shadow: none; +} + +@media (max-width: 991.98px) { + .library-shell { width: min(calc(100% - 2rem), 760px); margin-top: 1.5rem; } + .detail-column { margin-top: 0; } + .detail-empty { min-height: 170px; } } -.form-floating > .form-control-plaintext:focus::placeholder, .form-floating > .form-control:focus::placeholder { - text-align: start; +@media (max-width: 575.98px) { + .library-shell { width: calc(100% - 1rem); margin: 1.15rem auto 2rem; } + .library-heading { align-items: flex-start; flex-direction: column; gap: .85rem; margin-bottom: 1rem; } + .library-heading h1 { font-size: 1.75rem; } + .library-toolbar { align-items: stretch; flex-direction: column; padding: .75rem; gap: .6rem; } + .library-toolbar .btn { width: 100%; } + .import-hint { margin: .4rem .4rem 1rem; } + .first-run-panel { min-height: 300px; padding: 2rem 1rem; } + .first-run-panel h2 { font-size: 1.1rem; } + .library-card, .detail-card, .ask-card { padding: .95rem; } + .card-heading { align-items: flex-start; } + .status-text { max-width: 45%; } + .document-table { min-width: 100%; table-layout: fixed; } + .document-table th:nth-child(2), .document-table td:nth-child(2), + .document-table th:nth-child(3), .document-table td:nth-child(3) { display: none; } + .document-table th:nth-child(1), .document-table td:nth-child(1) { width: 58%; } + .document-table th:nth-child(4), .document-table td:nth-child(4) { width: 22%; } + .document-table th:nth-child(5), .document-table td:nth-child(5) { width: 20%; } + .document-name-button { max-width: 100%; } + .detail-actions { gap: .25rem; } + .detail-actions .btn { padding: .3rem .45rem; font-size: .7rem; } + .ask-form { flex-direction: column; } + .ask-form .btn { width: 100%; } + .markdown-output { min-height: 190px; max-height: 52vh; font-size: .75rem; } } diff --git a/src/Web/wwwroot/js/conversion.js b/src/Web/wwwroot/js/conversion.js index f51bb3f..b709be2 100644 --- a/src/Web/wwwroot/js/conversion.js +++ b/src/Web/wwwroot/js/conversion.js @@ -1,70 +1,314 @@ const fileInput = document.getElementById('fileInput'); -const btnConvert = document.getElementById('btnConvert'); -const btnCopy = document.getElementById('btnCopy'); -const btnDownload = document.getElementById('btnDownload'); -const btnClear = document.getElementById('btnClear'); +const btnImport = document.getElementById('btnImport'); +const btnFirstRunImport = document.getElementById('btnFirstRunImport'); +const firstRunPanel = document.getElementById('firstRunPanel'); +const firstRunStatus = document.getElementById('firstRunStatus'); +const workspaceLayout = document.getElementById('workspaceLayout'); +const libraryToolbar = document.getElementById('libraryToolbar'); +const supportedFormats = document.getElementById('supportedFormats'); +const pageTitle = document.getElementById('pageTitle'); +const pageDescription = document.getElementById('pageDescription'); const statusText = document.getElementById('statusText'); +const importProgress = document.getElementById('importProgress'); +const importResults = document.getElementById('importResults'); const markdownOutput = document.getElementById('markdownOutput'); const resultHeading = document.getElementById('resultHeading'); +const detailEmpty = document.getElementById('detailEmpty'); const historyList = document.getElementById('historyList'); const historyEmpty = document.getElementById('historyEmpty'); -const searchHistory = document.getElementById('searchHistory'); +const historyEmptyTitle = document.getElementById('historyEmptyTitle'); +const historyEmptyText = document.getElementById('historyEmptyText'); +const documentCount = document.getElementById('documentCount'); +const documentTableHead = document.getElementById('documentTableHead'); +const librarySearch = document.getElementById('librarySearch'); +const libraryResultsPanel = document.getElementById('libraryResultsPanel'); +const libraryResults = document.getElementById('libraryResults'); +const librarySearchEmpty = document.getElementById('librarySearchEmpty'); +const btnCopy = document.getElementById('btnCopy'); +const btnDownload = document.getElementById('btnDownload'); +const btnDeleteCurrent = document.getElementById('btnDeleteCurrent'); +const askForm = document.getElementById('askForm'); +const questionInput = document.getElementById('questionInput'); +const btnAsk = document.getElementById('btnAsk'); +const answerPanel = document.getElementById('answerPanel'); let currentFileName = ''; +let currentConversionId = null; +let historyRequestId = 0; +let librarySearchRequestId = 0; +let detailRequestId = 0; +let isImporting = false; -function setMarkdown(markdown, fileName = '') { - markdownOutput.textContent = markdown || ''; +function updateLibraryState(itemCount, search = '') { + const isFirstRun = itemCount === 0 && !search.trim(); + firstRunPanel.hidden = !isFirstRun; + workspaceLayout.hidden = isFirstRun; + libraryToolbar.hidden = isFirstRun; + supportedFormats.hidden = isFirstRun; + pageTitle.textContent = isFirstRun ? 'Convierte documentos a Markdown' : 'Consulta tus documentos'; + pageDescription.textContent = isFirstRun + ? 'Convierte tus archivos y guarda los resultados localmente en este equipo.' + : 'Busca contenido y vuelve a tus archivos desde un espacio privado en este equipo.'; +} + +function setMarkdown(markdown, fileName = '', conversionId = null) { + detailRequestId++; currentFileName = fileName; - resultHeading.textContent = fileName ? `${fileName} → Markdown` : 'Markdown generado'; + currentConversionId = conversionId; + markdownOutput.textContent = markdown || ''; const hasMarkdown = markdownOutput.textContent.length > 0; + resultHeading.textContent = fileName || 'Selecciona un documento'; + markdownOutput.hidden = !hasMarkdown; + detailEmpty.hidden = hasMarkdown; + detailEmpty.textContent = fileName && !hasMarkdown + ? 'Esta conversión no tiene Markdown disponible.' + : 'El Markdown del documento seleccionado aparecerá aquí.'; btnCopy.disabled = !hasMarkdown; btnDownload.disabled = !hasMarkdown; + btnDeleteCurrent.disabled = !conversionId; } async function readResponse(response) { const body = await response.json().catch(() => ({})); - if (!response.ok) { - throw new Error(body.error || 'No se pudo completar la solicitud.'); - } + if (!response.ok) throw new Error(body.error || 'No se pudo completar la solicitud.'); return body; } -btnConvert.addEventListener('click', async () => { - const file = fileInput.files[0]; - if (!file) { - statusText.textContent = 'Selecciona un archivo primero.'; - fileInput.focus(); - return; - } +function extensionOf(fileName) { + return (fileName.split('.').pop() || '').toUpperCase(); +} - const formData = new FormData(); - formData.append('file', file); - statusText.textContent = 'Convirtiendo...'; - btnConvert.disabled = true; +function formatDate(value) { + return new Date(value).toLocaleDateString(undefined, { year: 'numeric', month: 'short', day: 'numeric' }); +} + +function createHistoryRow(item) { + const row = document.createElement('tr'); + const fileCell = document.createElement('td'); + const open = document.createElement('button'); + open.type = 'button'; + open.className = 'document-name-button'; + open.textContent = item.fileName; + open.disabled = item.status !== 'Completed'; + open.setAttribute('aria-label', `Abrir vista previa de ${item.fileName}`); + open.addEventListener('click', () => openHistoryItem(item.id)); + fileCell.appendChild(open); + + const typeCell = document.createElement('td'); + const typeBadge = document.createElement('span'); + typeBadge.className = 'file-type'; + typeBadge.textContent = extensionOf(item.fileName) || 'FILE'; + typeCell.appendChild(typeBadge); + const dateCell = document.createElement('td'); + dateCell.className = 'document-date'; + dateCell.textContent = formatDate(item.createdAt); + + const stateCell = document.createElement('td'); + const state = document.createElement('span'); + state.className = `document-state ${item.status === 'Completed' ? 'is-complete' : 'is-failed'}`; + state.textContent = item.status === 'Completed' ? 'Listo' : 'Error'; + stateCell.appendChild(state); + + const actionCell = document.createElement('td'); + actionCell.className = 'document-actions'; + const remove = document.createElement('button'); + remove.type = 'button'; + remove.className = 'icon-action'; + remove.textContent = 'Borrar'; + remove.setAttribute('aria-label', `Borrar ${item.fileName}`); + remove.addEventListener('click', () => deleteHistoryItem(item.id, item.fileName)); + actionCell.appendChild(remove); + row.append(fileCell, typeCell, dateCell, stateCell, actionCell); + return row; +} + +async function loadHistory(search = '') { + const requestId = ++historyRequestId; try { - const response = await fetch('/Conversion/Convert', { method: 'POST', body: formData }); + const response = await fetch(`/Conversion/History?search=${encodeURIComponent(search)}`); + const items = await readResponse(response); + if (requestId !== historyRequestId) return; + historyList.replaceChildren(...items.map(createHistoryRow)); + documentCount.textContent = items.length; + documentTableHead.hidden = items.length === 0; + historyEmpty.hidden = items.length > 0; + historyEmptyTitle.textContent = search ? 'No hay coincidencias' : 'Tu biblioteca está vacía'; + historyEmptyText.textContent = search + ? 'No hay archivos con ese nombre. Prueba otra palabra o borra la búsqueda; las coincidencias de contenido aparecen abajo.' + : 'Importa un documento para convertirlo, buscar su contenido y consultarlo aquí.'; + updateLibraryState(items.length, search); + } catch { + if (requestId !== historyRequestId) return; + updateLibraryState(1, search); + historyEmpty.hidden = false; + historyEmptyTitle.textContent = 'No se pudo cargar la biblioteca'; + historyEmptyText.textContent = 'Comprueba que la aplicación siga abierta e inténtalo de nuevo.'; + } +} + +async function openHistoryItem(id) { + const requestId = ++detailRequestId; + statusText.textContent = 'Abriendo documento...'; + try { + const response = await fetch(`/Conversion/Item?id=${encodeURIComponent(id)}`); const result = await readResponse(response); - if (!result.success) { - throw new Error(result.errorMessage || 'No se pudo convertir el archivo.'); - } + if (requestId !== detailRequestId) return; + setMarkdown(result.markdown, result.fileName, result.conversionId); + statusText.textContent = 'Vista previa actualizada.'; + } catch (error) { + if (requestId !== detailRequestId) return; + statusText.textContent = error.message; + } +} - setMarkdown(result.markdown, result.fileName || file.name); - statusText.textContent = `Listo en ${Number(result.conversionTimeSeconds).toFixed(2)} s.`; - await loadHistory(searchHistory.value); +async function deleteHistoryItem(id, fileName) { + if (!window.confirm(`¿Borrar ${fileName} y su Markdown de este equipo?`)) return; + try { + const response = await fetch(`/Conversion/Delete?id=${encodeURIComponent(id)}`, { method: 'POST' }); + await readResponse(response); + if (currentConversionId === id) setMarkdown(''); + statusText.textContent = `${fileName} y su índice se borraron del equipo.`; + await Promise.all([loadHistory(librarySearch.value), loadLibraryResults(librarySearch.value)]); } catch (error) { - statusText.textContent = error.message || 'Error de conexión. Comprueba que la aplicación siga abierta.'; - } finally { - btnConvert.disabled = false; + statusText.textContent = error.message || `No se pudo borrar ${fileName}.`; + } +} + +function renderEvidence(target, hit) { + const item = document.createElement('li'); + item.className = 'list-group-item evidence-item'; + const open = document.createElement('button'); + open.type = 'button'; + open.className = 'evidence-open'; + open.textContent = `${hit.fileName} · ${hit.section}`; + open.addEventListener('click', () => openHistoryItem(hit.conversionId)); + const excerpt = document.createElement('p'); + excerpt.className = 'evidence-excerpt mb-0 mt-1'; + excerpt.textContent = hit.content; + item.append(open, excerpt); + target.appendChild(item); +} + +async function loadLibraryResults(query = '') { + const requestId = ++librarySearchRequestId; + if (!query.trim()) { + libraryResultsPanel.hidden = true; + libraryResults.replaceChildren(); + return; + } + libraryResultsPanel.hidden = false; + try { + const response = await fetch(`/Conversion/Search?query=${encodeURIComponent(query)}`); + const items = await readResponse(response); + if (requestId !== librarySearchRequestId) return; + libraryResults.replaceChildren(); + items.forEach(hit => renderEvidence(libraryResults, hit)); + librarySearchEmpty.hidden = items.length > 0; + librarySearchEmpty.textContent = 'No encontré fragmentos para esta búsqueda.'; + } catch { + if (requestId !== librarySearchRequestId) return; + librarySearchEmpty.hidden = false; + librarySearchEmpty.textContent = 'No se pudo buscar en la biblioteca.'; + } +} + +async function importFiles(files) { + if (files.length === 0) return; + if (isImporting) { + statusText.textContent = 'Espera a que termine el lote actual antes de importar más archivos.'; + firstRunStatus.textContent = statusText.textContent; + firstRunStatus.hidden = false; + return; + } + if (files.length > 10) { + statusText.textContent = 'Selecciona hasta 10 archivos por lote.'; + firstRunStatus.textContent = statusText.textContent; + firstRunStatus.hidden = false; + return; + } + + isImporting = true; + importProgress.hidden = false; + importResults.replaceChildren(); + btnImport.disabled = true; + btnFirstRunImport.disabled = true; + let lastResult = null; + firstRunStatus.textContent = `Importando ${files.length} ${files.length === 1 ? 'archivo' : 'archivos'}...`; + firstRunStatus.hidden = false; + + for (const file of files) { + const entry = document.createElement('li'); + entry.className = 'import-result'; + entry.textContent = `${file.name} · En espera`; + importResults.appendChild(entry); + entry.textContent = `${file.name} · Convirtiendo...`; + try { + const formData = new FormData(); + formData.append('file', file); + const response = await fetch('/Conversion/Convert', { method: 'POST', body: formData }); + const result = await readResponse(response); + if (!result.success) throw new Error(result.errorMessage || 'No se pudo convertir.'); + lastResult = result; + entry.classList.add('is-success'); + entry.textContent = `${file.name} · Importado`; + } catch (error) { + entry.classList.add('is-error'); + entry.textContent = `${file.name} · ${error.message || 'No se pudo convertir.'}`; + } + } + + if (lastResult) { + setMarkdown(lastResult.markdown, lastResult.fileName, lastResult.conversionId); + statusText.textContent = 'Importación completada.'; + } else { + statusText.textContent = 'No se pudo importar ningún archivo.'; + } + await Promise.all([loadHistory(librarySearch.value), loadLibraryResults(librarySearch.value)]); + firstRunStatus.textContent = statusText.textContent; + firstRunStatus.hidden = !firstRunStatus.textContent; + fileInput.value = ''; + isImporting = false; + btnImport.disabled = false; + btnFirstRunImport.disabled = false; + if (lastResult) + resultHeading.focus({ preventScroll: true }); + else if (!firstRunPanel.hidden) + firstRunStatus.focus({ preventScroll: true }); + else + statusText.focus({ preventScroll: true }); +} + +btnImport.addEventListener('click', () => fileInput.click()); +btnFirstRunImport.addEventListener('click', () => fileInput.click()); +fileInput.addEventListener('change', () => importFiles([...fileInput.files])); + +firstRunPanel.addEventListener('dragover', event => { + event.preventDefault(); + firstRunPanel.classList.add('is-dragging'); +}); +firstRunPanel.addEventListener('dragleave', event => { + if (!firstRunPanel.contains(event.relatedTarget)) + firstRunPanel.classList.remove('is-dragging'); +}); +firstRunPanel.addEventListener('drop', event => { + event.preventDefault(); + firstRunPanel.classList.remove('is-dragging'); + if (isImporting) { + statusText.textContent = 'Espera a que termine el lote actual antes de importar más archivos.'; + firstRunStatus.textContent = statusText.textContent; + firstRunStatus.hidden = false; + return; } + importFiles([...event.dataTransfer.files]); }); btnCopy.addEventListener('click', async () => { try { await navigator.clipboard.writeText(markdownOutput.textContent); - statusText.textContent = 'Markdown copiado al portapapeles.'; + statusText.textContent = 'Markdown copiado.'; } catch { - statusText.textContent = 'No se pudo acceder al portapapeles en este navegador.'; + statusText.textContent = 'El navegador no permitió copiar al portapapeles.'; } }); @@ -83,75 +327,65 @@ btnDownload.addEventListener('click', () => { statusText.textContent = `Descarga iniciada: ${safeName}.md`; }); -btnClear.addEventListener('click', () => { - setMarkdown(''); - fileInput.value = ''; - statusText.textContent = ''; - fileInput.focus(); +btnDeleteCurrent.addEventListener('click', () => { + if (currentConversionId) deleteHistoryItem(currentConversionId, currentFileName); }); -async function loadHistory(search = '') { +askForm.addEventListener('submit', async event => { + event.preventDefault(); + btnAsk.disabled = true; + btnAsk.textContent = 'Buscando...'; + answerPanel.hidden = false; + answerPanel.replaceChildren(); + const loading = document.createElement('p'); + loading.className = 'text-muted small mb-0'; + loading.textContent = 'Buscando evidencia local...'; + answerPanel.appendChild(loading); try { - const response = await fetch(`/Conversion/History?search=${encodeURIComponent(search)}`); - const items = await readResponse(response); - historyList.replaceChildren(); - - items.forEach(item => { - const entry = document.createElement('li'); - entry.className = 'list-group-item px-0'; - - const button = document.createElement('button'); - button.type = 'button'; - button.className = 'history-entry'; - button.disabled = item.status !== 'Completed'; - button.setAttribute('aria-label', `Abrir ${item.fileName}`); - - const name = document.createElement('span'); - name.className = 'history-file-name'; - name.textContent = item.fileName; - - const detail = document.createElement('span'); - detail.className = 'history-date'; - detail.textContent = item.status === 'Completed' - ? new Date(item.createdAt).toLocaleString() - : 'No se pudo convertir'; - - button.append(name, detail); - button.addEventListener('click', () => openHistoryItem(item.id)); - entry.appendChild(button); - historyList.appendChild(entry); + const response = await fetch('/Conversion/Ask', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ question: questionInput.value }) }); - - historyEmpty.hidden = items.length > 0; - historyEmpty.textContent = search ? 'No hay resultados para esa búsqueda.' : 'Aún no hay conversiones guardadas.'; - } catch { - historyEmpty.hidden = false; - historyEmpty.textContent = 'No se pudo cargar el historial.'; - } -} - -async function openHistoryItem(id) { - statusText.textContent = 'Abriendo conversión...'; - try { - const response = await fetch(`/Conversion/Item?id=${encodeURIComponent(id)}`); const result = await readResponse(response); - setMarkdown(result.markdown, result.fileName); - statusText.textContent = result.markdown ? 'Conversión cargada desde el historial.' : 'Esta conversión no tiene Markdown disponible.'; - } catch (error) { - statusText.textContent = error.message; + answerPanel.replaceChildren(); + const answer = document.createElement('p'); + answer.className = result.hasEvidence ? 'answer-text' : 'answer-text text-muted'; + answer.textContent = result.answer; + answerPanel.appendChild(answer); + if (result.citations?.length) { + const citations = document.createElement('ul'); + citations.className = 'list-group list-group-flush'; + result.citations.forEach(hit => renderEvidence(citations, hit)); + answerPanel.appendChild(citations); + } + } catch { + answerPanel.textContent = 'No se pudo consultar la biblioteca local.'; + } finally { + btnAsk.disabled = false; + btnAsk.textContent = 'Consultar'; } -} - -let historySearchTimer; -searchHistory.addEventListener('input', event => { - clearTimeout(historySearchTimer); - historySearchTimer = setTimeout(() => loadHistory(event.target.value), 200); }); -fileInput.addEventListener('change', () => { - if (fileInput.files[0]) { - statusText.textContent = `${fileInput.files[0].name} seleccionado.`; - } +let searchTimer; +librarySearch.addEventListener('input', event => { + historyRequestId++; + librarySearchRequestId++; + const query = event.target.value; + historyList.replaceChildren(); + documentTableHead.hidden = true; + historyEmpty.hidden = false; + historyEmptyTitle.textContent = query.trim() ? 'Buscando...' : 'Cargando biblioteca...'; + historyEmptyText.textContent = query.trim() ? 'Actualizando coincidencias.' : 'Actualizando documentos guardados.'; + libraryResultsPanel.hidden = !query.trim(); + libraryResults.replaceChildren(); + librarySearchEmpty.hidden = !query.trim(); + librarySearchEmpty.textContent = 'Buscando fragmentos...'; + clearTimeout(searchTimer); + searchTimer = setTimeout(() => { + loadHistory(query); + loadLibraryResults(query); + }, 220); }); loadHistory(); diff --git a/tests/MarkItDownWeb.Tests/ConversionServiceTests.cs b/tests/MarkItDownWeb.Tests/ConversionServiceTests.cs index bfaa3d6..c685410 100644 --- a/tests/MarkItDownWeb.Tests/ConversionServiceTests.cs +++ b/tests/MarkItDownWeb.Tests/ConversionServiceTests.cs @@ -4,6 +4,7 @@ using MarkItDownWeb.Infrastructure.Persistence; using Microsoft.Data.Sqlite; using Microsoft.EntityFrameworkCore; +using System.Text; using Xunit; namespace MarkItDownWeb.Tests; @@ -78,6 +79,220 @@ public async Task ConvertAsync_records_failed_conversion_and_removes_temporary_f Assert.Equal("Failed", historyItem.Status); } + [Fact] + public async Task Search_returns_matching_excerpt_with_its_section_and_delete_removes_the_index() + { + await using var database = await TestDatabase.CreateAsync(); + var storage = new FakeFileStorageService(); + var converter = new FakeMarkItDownService(_ => Task.FromResult( + "# Budget\n\nThe annual maintenance cost is 420 dollars.\n\n## Schedule\n\nRenew the plan in October.")); + var conversions = new ConversionService(converter, storage, database.Repository); + + await using var input = new MemoryStream([0x61, 0x2C, 0x62]); + var conversion = await conversions.ConvertAsync(input, "budget.csv", input.Length); + var library = new LibraryService(database.Repository, new LocalExtractiveAnswerGenerator()); + + var answer = await library.AskAsync("What is the annual maintenance cost?"); + + Assert.True(answer.HasEvidence); + var citation = Assert.Single(answer.Citations); + Assert.Equal("budget.csv", citation.FileName); + Assert.Equal("Budget", citation.Section); + Assert.Contains("420 dollars", citation.Content); + + await conversions.DeleteAsync(conversion.ConversionId); + + Assert.Empty(await library.SearchAsync("maintenance")); + Assert.Empty(storage.MarkdownFiles); + } + + [Fact] + public async Task Ask_returns_clear_abstention_when_no_document_matches() + { + await using var database = await TestDatabase.CreateAsync(); + var library = new LibraryService(database.Repository, new LocalExtractiveAnswerGenerator()); + + var answer = await library.AskAsync("What is the launch date?"); + + Assert.False(answer.HasEvidence); + Assert.Empty(answer.Citations); + Assert.Contains("No encontré", answer.Answer); + } + + [Fact] + public async Task Search_uses_page_number_when_markdown_preserves_page_markers() + { + await using var database = await TestDatabase.CreateAsync(); + var storage = new FakeFileStorageService(); + var converter = new FakeMarkItDownService(_ => Task.FromResult( + "\nThe risk review is scheduled for next week.")); + var conversions = new ConversionService(converter, storage, database.Repository); + await using var input = new MemoryStream([0x61, 0x2C, 0x62]); + await conversions.ConvertAsync(input, "review.csv", input.Length); + + var library = new LibraryService(database.Repository, new LocalExtractiveAnswerGenerator()); + var hits = await library.SearchAsync("risk"); + + Assert.Equal("Página 4", Assert.Single(hits).Section); + } + + [Fact] + public async Task Search_ignores_case_and_diacritics_in_document_content() + { + await using var database = await TestDatabase.CreateAsync(); + var storage = new FakeFileStorageService(); + var converter = new FakeMarkItDownService(_ => Task.FromResult("La CANCIÓN está en el CAFÉ.")); + var conversions = new ConversionService(converter, storage, database.Repository); + await using var input = new MemoryStream([0x61, 0x2C, 0x62]); + await conversions.ConvertAsync(input, "música.csv", input.Length); + + var library = new LibraryService(database.Repository, new LocalExtractiveAnswerGenerator()); + + Assert.Single(await library.SearchAsync("canción")); + Assert.Single(await library.SearchAsync("cafe")); + Assert.Single(await library.SearchAsync("canción".Normalize(NormalizationForm.FormD))); + } + + [Fact] + public async Task Search_does_not_drop_whole_word_hits_behind_prefix_matches() + { + await using var database = await TestDatabase.CreateAsync(); + var exact = new Conversion + { + FileName = "budget.csv", + FileType = FileType.Csv, + Status = ConversionStatus.Completed, + CreatedAt = DateTime.UtcNow.AddDays(-1), + Chunks = + [ + new DocumentChunk + { + Section = "Contenido", + Content = "The annual budget is approved.", + SearchIndex = " the annual budget is approved " + } + ] + }; + await database.Repository.AddAsync(exact); + + for (var index = 0; index < 80; index++) + { + var content = $"The budgetary note {index} was updated."; + var similar = new Conversion + { + FileName = $"budgetary-{index}.csv", + FileType = FileType.Csv, + Status = ConversionStatus.Completed, + CreatedAt = DateTime.UtcNow.AddMinutes(index), + Chunks = + [ + new DocumentChunk + { + Section = "Contenido", + Content = content, + SearchIndex = SearchTextNormalizer.CreateIndex(content) + } + ] + }; + await database.Repository.AddAsync(similar); + } + + await database.Repository.SaveChangesAsync(); + var library = new LibraryService(database.Repository, new LocalExtractiveAnswerGenerator()); + + var hit = Assert.Single(await library.SearchAsync("budget")); + + Assert.Equal("budget.csv", hit.FileName); + Assert.Contains("annual budget", hit.Content); + } + + [Fact] + public async Task Search_ranks_older_chunks_that_match_more_query_terms() + { + await using var database = await TestDatabase.CreateAsync(); + var completeMatch = new Conversion + { + FileName = "archive.csv", + FileType = FileType.Csv, + Status = ConversionStatus.Completed, + CreatedAt = DateTime.UtcNow.AddDays(-1), + Chunks = + [ + new DocumentChunk + { + Section = "Contenido", + Content = "Apollo migration completed in 2026.", + SearchIndex = SearchTextNormalizer.CreateIndex("Apollo migration completed in 2026.") + } + ] + }; + await database.Repository.AddAsync(completeMatch); + + var terms = new[] { "apollo", "migration", "2026" }; + for (var termIndex = 0; termIndex < terms.Length; termIndex++) + { + for (var itemIndex = 0; itemIndex < 80; itemIndex++) + { + var content = $"{terms[termIndex]} note {itemIndex}"; + var conversion = new Conversion + { + FileName = $"note-{termIndex}-{itemIndex}.csv", + FileType = FileType.Csv, + Status = ConversionStatus.Completed, + CreatedAt = DateTime.UtcNow.AddMinutes(termIndex * 80 + itemIndex), + Chunks = + [ + new DocumentChunk + { + Section = "Contenido", + Content = content, + SearchIndex = SearchTextNormalizer.CreateIndex(content) + } + ] + }; + await database.Repository.AddAsync(conversion); + } + } + + await database.Repository.SaveChangesAsync(); + var library = new LibraryService(database.Repository, new LocalExtractiveAnswerGenerator()); + + var first = (await library.SearchAsync("Apollo migration 2026")).First(); + + Assert.Equal("archive.csv", first.FileName); + Assert.Equal(3, first.MatchCount); + } + + [Fact] + public async Task Startup_indexing_adds_searchable_chunks_to_existing_history() + { + await using var database = await TestDatabase.CreateAsync(); + var storage = new FakeFileStorageService(); + storage.SeedMarkdown("Markdown/legacy.md", "# Legacy report\n\nRetain this historical content."); + var conversion = new Conversion + { + FileName = "legacy.csv", + FileType = FileType.Csv, + MarkdownFilePath = "Markdown/legacy.md", + FileSizeBytes = 42, + Status = ConversionStatus.Completed + }; + await database.Repository.AddAsync(conversion); + await database.Repository.SaveChangesAsync(); + + var service = new ConversionService( + new FakeMarkItDownService(_ => Task.FromResult("unused")), storage, database.Repository); + database.ClearTracking(); + await service.IndexExistingAsync(); + var library = new LibraryService(database.Repository, new LocalExtractiveAnswerGenerator()); + + var hit = Assert.Single(await library.SearchAsync("historical")); + + Assert.Equal(conversion.Id, hit.ConversionId); + Assert.Equal("Legacy report", hit.Section); + Assert.Contains("Retain this historical content", hit.Content); + } + private sealed class FakeMarkItDownService(Func> convert) : IMarkItDownService { public int CallCount { get; private set; } @@ -94,6 +309,9 @@ private sealed class FakeFileStorageService : IFileStorageService private readonly Dictionary _markdownFiles = new(); public HashSet TemporaryFiles { get; } = new(); + public Dictionary MarkdownFiles => _markdownFiles; + + public void SeedMarkdown(string path, string content) => _markdownFiles[path] = content; public async Task SaveTemporaryAsync( Stream fileStream, @@ -117,6 +335,7 @@ public Task ReadMarkdownAsync(string markdownFilePath, CancellationToken => Task.FromResult(_markdownFiles[markdownFilePath]); public void DeleteTemporary(string filePath) => TemporaryFiles.Remove(filePath); + public void DeleteMarkdown(string filePath) => _markdownFiles.Remove(filePath); } private sealed class TestDatabase : IAsyncDisposable @@ -132,6 +351,7 @@ private TestDatabase(SqliteConnection connection, AppDbContext context) } public IConversionRepository Repository { get; } + public void ClearTracking() => _context.ChangeTracker.Clear(); public static async Task CreateAsync() { diff --git a/tests/MarkItDownWeb.Tests/LocalLibrarySchemaTests.cs b/tests/MarkItDownWeb.Tests/LocalLibrarySchemaTests.cs new file mode 100644 index 0000000..f7991c4 --- /dev/null +++ b/tests/MarkItDownWeb.Tests/LocalLibrarySchemaTests.cs @@ -0,0 +1,74 @@ +using MarkItDownWeb.Application.Interfaces; +using MarkItDownWeb.Domain.Entities; +using MarkItDownWeb.Infrastructure.Persistence; +using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; +using Xunit; + +namespace MarkItDownWeb.Tests; + +public class LocalLibrarySchemaTests +{ + [Fact] + public async Task EnsureCreated_adds_the_index_to_an_existing_database_and_preserves_history() + { + await using var connection = new SqliteConnection("Data Source=:memory:"); + await connection.OpenAsync(); + var options = new DbContextOptionsBuilder().UseSqlite(connection).Options; + var id = Guid.NewGuid(); + + await using (var legacy = new AppDbContext(options)) + { + await legacy.Database.ExecuteSqlRawAsync(""" + CREATE TABLE "Conversions" ( + "Id" TEXT NOT NULL PRIMARY KEY, + "FileName" TEXT NOT NULL, + "FileType" TEXT NOT NULL, + "OriginalFilePath" TEXT NOT NULL, + "MarkdownFilePath" TEXT NULL, + "FileSizeBytes" INTEGER NOT NULL, + "CreatedAt" TEXT NOT NULL, + "ConversionTimeSeconds" REAL NULL, + "Status" TEXT NOT NULL, + "ErrorMessage" TEXT NULL + ); + """); + await legacy.Database.ExecuteSqlInterpolatedAsync($""" + INSERT INTO "Conversions" VALUES + ({id}, 'legacy.csv', 'Csv', '', 'legacy.md', 42, '2026-10-03 00:00:00', 0.1, 'Completed', NULL); + """); + await legacy.Database.ExecuteSqlRawAsync(""" + CREATE TABLE "DocumentChunks" ( + "Id" TEXT NOT NULL CONSTRAINT "PK_DocumentChunks" PRIMARY KEY, + "ConversionId" TEXT NOT NULL, + "Ordinal" INTEGER NOT NULL, + "Section" TEXT NOT NULL, + "Content" TEXT NOT NULL, + CONSTRAINT "FK_DocumentChunks_Conversions_ConversionId" + FOREIGN KEY ("ConversionId") REFERENCES "Conversions" ("Id") ON DELETE CASCADE + ); + """); + var chunkId = Guid.NewGuid(); + await legacy.Database.ExecuteSqlInterpolatedAsync($""" + INSERT INTO "DocumentChunks" VALUES + ({chunkId}, {id}, 0, 'Old document', 'Preserved searchable text'); + """); + + LocalLibrarySchema.EnsureCreated(legacy); + + var preserved = await legacy.Conversions.SingleAsync(); + Assert.Equal(id, preserved.Id); + Assert.Equal("legacy.csv", preserved.FileName); + IConversionRepository repository = new ConversionRepository(legacy); + await repository.BackfillSearchIndexesAsync(); + await repository.SaveChangesAsync(); + var hits = await repository.SearchContentAsync(["searchable"], 5); + Assert.Equal(id, Assert.Single(hits).ConversionId); + + await repository.DeleteAsync(id); + await repository.SaveChangesAsync(); + Assert.Equal(0, await legacy.DocumentChunks.CountAsync()); + Assert.Equal(0, await legacy.Conversions.CountAsync()); + } + } +}