feat: support full task index

This commit is contained in:
quorafind 2025-04-07 09:23:47 +08:00
parent 2f656e7774
commit 895a46e29f
21 changed files with 4265 additions and 3 deletions

View file

@ -68,6 +68,7 @@ interface Task {
class TaskParser {
// Regular expressions for Tasks plugin syntax
private readonly startDateRegex = /📅 (\d{4}-\d{2}-\d{2})/;
private readonly completedDateRegex = /✅ (\d{4}-\d{2}-\d{2})/;
private readonly dueDateRegex = /⏳ (\d{4}-\d{2}-\d{2})/;
private readonly scheduledDateRegex = /⏰ (\d{4}-\d{2}-\d{2})/;
private readonly recurrenceRegex = /🔁 (.*?)(?=\s|$)/;

View file

@ -17,6 +17,7 @@ esbuild
},
minify: prod ? true : false,
entryPoints: ["src/index.ts"],
plugins: [inlineWorkerPlugin({ workerName: "Datacore Indexer" })],
bundle: true,
external: [
"obsidian",

View file

@ -1,7 +1,7 @@
{
"id": "obsidian-task-progress-bar",
"name": "Task Genius",
"version": "6.2.2",
"version": "7.0.0",
"minAppVersion": "0.15.2",
"description": "Comprehensive task management that includes progress bars, task status cycling, and advanced task tracking features.",
"author": "Boninall",

View file

@ -1,6 +1,6 @@
{
"name": "obsidian-task-progress-bar",
"version": "6.2.2",
"version": "7.0.0",
"description": "Comprehensive task management plugin for Obsidian with progress bars, task status cycling, and advanced task tracking features.",
"main": "main.js",
"scripts": {
@ -42,11 +42,15 @@
"codemirror": "^6.0.0",
"cross-env": "^7.0.3",
"esbuild": "0.13.12",
"esbuild-plugin-inline-worker": "https://github.com/mitschabaude/esbuild-plugin-inline-worker",
"monkey-around": "^3.0.0",
"obsidian": "^1.8.7",
"obsidian-typings": "^2.39.0",
"regexp-match-indices": "^1.0.2",
"tslib": "2.4.0",
"typescript": "4.7.3"
},
"dependencies": {
"uuid": "^11.1.0"
}
}

View file

@ -7,6 +7,10 @@ settings:
importers:
.:
dependencies:
uuid:
specifier: ^11.1.0
version: 11.1.0
devDependencies:
'@codemirror/autocomplete':
specifier: ^6.0.0
@ -59,6 +63,9 @@ importers:
esbuild:
specifier: 0.13.12
version: 0.13.12
esbuild-plugin-inline-worker:
specifier: https://github.com/mitschabaude/esbuild-plugin-inline-worker
version: https://codeload.github.com/mitschabaude/esbuild-plugin-inline-worker/tar.gz/d1aaffc721a62a3fe33f59f8f69b462c7dd05f45(esbuild@0.13.12)
monkey-around:
specifier: ^3.0.0
version: 3.0.0
@ -623,6 +630,9 @@ packages:
colord@2.9.3:
resolution: {integrity: sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw==}
commondir@1.0.1:
resolution: {integrity: sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==}
concat-map@0.0.1:
resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==}
@ -776,6 +786,12 @@ packages:
cpu: [x64]
os: [openbsd]
esbuild-plugin-inline-worker@https://codeload.github.com/mitschabaude/esbuild-plugin-inline-worker/tar.gz/d1aaffc721a62a3fe33f59f8f69b462c7dd05f45:
resolution: {tarball: https://codeload.github.com/mitschabaude/esbuild-plugin-inline-worker/tar.gz/d1aaffc721a62a3fe33f59f8f69b462c7dd05f45}
version: 0.1.1
peerDependencies:
esbuild: latest
esbuild-sunos-64@0.13.12:
resolution: {integrity: sha512-jBsF+e0woK3miKI8ufGWKG3o3rY9DpHvCVRn5eburMIIE+2c+y3IZ1srsthKyKI6kkXLvV4Cf/E7w56kLipMXw==}
cpu: [x64]
@ -881,6 +897,14 @@ packages:
resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==}
engines: {node: '>=8'}
find-cache-dir@3.3.2:
resolution: {integrity: sha512-wXZV5emFEjrridIgED11OoUKLxiYjAcqot/NJdAkOhlJ+vGzwhOAfcG5OX1jP+S0PcjEn8bdMJv+g2jwQ3Onig==}
engines: {node: '>=8'}
find-up@4.1.0:
resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==}
engines: {node: '>=8'}
find-up@5.0.0:
resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==}
engines: {node: '>=10'}
@ -1048,6 +1072,10 @@ packages:
resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==}
engines: {node: '>= 0.8.0'}
locate-path@5.0.0:
resolution: {integrity: sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==}
engines: {node: '>=8'}
locate-path@6.0.0:
resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==}
engines: {node: '>=10'}
@ -1059,6 +1087,10 @@ packages:
resolution: {integrity: sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==}
engines: {node: '>=8'}
make-dir@3.1.0:
resolution: {integrity: sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==}
engines: {node: '>=8'}
matcher@3.0.0:
resolution: {integrity: sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==}
engines: {node: '>=10'}
@ -1149,14 +1181,26 @@ packages:
resolution: {integrity: sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg==}
engines: {node: '>=8'}
p-limit@2.3.0:
resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==}
engines: {node: '>=6'}
p-limit@3.1.0:
resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==}
engines: {node: '>=10'}
p-locate@4.1.0:
resolution: {integrity: sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==}
engines: {node: '>=8'}
p-locate@5.0.0:
resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==}
engines: {node: '>=10'}
p-try@2.2.0:
resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==}
engines: {node: '>=6'}
parent-module@1.0.1:
resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==}
engines: {node: '>=6'}
@ -1187,6 +1231,10 @@ packages:
pixi.js@7.2.4:
resolution: {integrity: sha512-nBH60meoLnHxoMFz17HoMxXS4uJpG5jwIdL+Gx2S11TzWgP3iKF+/WLOTrkSdyuQoQSdIBxVqpnYii0Wiox15A==}
pkg-dir@4.2.0:
resolution: {integrity: sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==}
engines: {node: '>=8'}
prelude-ls@1.2.1:
resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==}
engines: {node: '>= 0.8.0'}
@ -1368,6 +1416,10 @@ packages:
resolution: {integrity: sha512-oCwdVC7mTuWiPyjLUz/COz5TLk6wgp0RCsN+wHZ2Ekneac9w8uuV0njcbbie2ME+Vs+d6duwmYuR3HgQXs1fOg==}
engines: {node: '>= 0.4'}
uuid@11.1.0:
resolution: {integrity: sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==}
hasBin: true
w3c-keyname@2.2.8:
resolution: {integrity: sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==}
@ -2041,6 +2093,8 @@ snapshots:
colord@2.9.3: {}
commondir@1.0.1: {}
concat-map@0.0.1: {}
crelt@1.0.6: {}
@ -2164,6 +2218,11 @@ snapshots:
esbuild-openbsd-64@0.13.12:
optional: true
esbuild-plugin-inline-worker@https://codeload.github.com/mitschabaude/esbuild-plugin-inline-worker/tar.gz/d1aaffc721a62a3fe33f59f8f69b462c7dd05f45(esbuild@0.13.12):
dependencies:
esbuild: 0.13.12
find-cache-dir: 3.3.2
esbuild-sunos-64@0.13.12:
optional: true
@ -2315,6 +2374,17 @@ snapshots:
dependencies:
to-regex-range: 5.0.1
find-cache-dir@3.3.2:
dependencies:
commondir: 1.0.1
make-dir: 3.1.0
pkg-dir: 4.2.0
find-up@4.1.0:
dependencies:
locate-path: 5.0.0
path-exists: 4.0.0
find-up@5.0.0:
dependencies:
locate-path: 6.0.0
@ -2506,6 +2576,10 @@ snapshots:
prelude-ls: 1.2.1
type-check: 0.4.0
locate-path@5.0.0:
dependencies:
p-locate: 4.1.0
locate-path@6.0.0:
dependencies:
p-locate: 5.0.0
@ -2514,6 +2588,10 @@ snapshots:
lowercase-keys@2.0.0: {}
make-dir@3.1.0:
dependencies:
semver: 6.3.1
matcher@3.0.0:
dependencies:
escape-string-regexp: 4.0.0
@ -2593,14 +2671,24 @@ snapshots:
p-cancelable@2.1.1: {}
p-limit@2.3.0:
dependencies:
p-try: 2.2.0
p-limit@3.1.0:
dependencies:
yocto-queue: 0.1.0
p-locate@4.1.0:
dependencies:
p-limit: 2.3.0
p-locate@5.0.0:
dependencies:
p-limit: 3.1.0
p-try@2.2.0: {}
parent-module@1.0.1:
dependencies:
callsites: 3.1.0
@ -2652,6 +2740,10 @@ snapshots:
transitivePeerDependencies:
- '@pixi/utils'
pkg-dir@4.2.0:
dependencies:
find-up: 4.1.0
prelude-ls@1.2.1: {}
progress@2.0.3: {}
@ -2817,6 +2909,8 @@ snapshots:
punycode: 1.4.1
qs: 6.14.0
uuid@11.1.0: {}
w3c-keyname@2.2.8: {}
which@2.0.2:

0
src/pages/TaskView.ts Normal file
View file

40
src/utils/TODO.md Normal file
View file

@ -0,0 +1,40 @@
# Refactoring TODO List
## Task Index Refactoring
### High Priority Tasks
- [ ] Remove non-task related indexing from importer.ts
- [ ] Optimize task data indexing for high performance
- [ ] Refactor worker implementation to focus only on task-related data
- [ ] Implement incremental updates for task data
- [ ] Redesign data structures for task metadata based on task-view.md
### Data Structure Optimization
- [ ] Simplify task cache structure to include only task-relevant fields
- [ ] Design efficient indexing for task filtering and querying
- [ ] Implement optimized data structures for tag and date indexing
- [ ] Reduce memory footprint of task index
### Worker Optimization
- [ ] Refactor web worker implementation for better performance
- [ ] Implement batch processing of task updates
- [ ] Add support for priority-based task processing
- [ ] Improve throttling mechanism for background processing
### Future Improvements
- [ ] Design a metadata import strategy compatible with task-view.md
- [ ] Implement LRU cache for frequently accessed tasks
- [ ] Add support for efficient task dependency tracking
- [ ] Create API for external plugin integration
## Architecture Improvements
- [ ] Separate task parsing from task indexing logic
- [ ] Create dedicated service for task data management
- [ ] Implement event-based system for task updates
- [ ] Add telemetry for performance monitoring (opt-in)
## Technical Debt
- [ ] Fix type errors in current implementation
- [ ] Add comprehensive unit tests for task indexing
- [ ] Document all public APIs and interfaces
- [ ] Create migration path for future index structure changes

View file

@ -0,0 +1,904 @@
/**
* High-performance task indexer implementation
*/
import {
App,
Component,
FileStats,
MetadataCache,
TFile,
Vault,
} from "obsidian";
import {
SortingCriteria,
Task,
TaskCache,
TaskFilter,
TaskIndexer as TaskIndexerInterface,
TaskParserConfig,
} from "../types/TaskIndex";
import { TaskParser } from "./TaskParser";
/**
* Utility to format a date for index keys (YYYY-MM-DD)
*/
function formatDateForIndex(date: number): string {
const d = new Date(date);
return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(
2,
"0"
)}-${String(d.getDate()).padStart(2, "0")}`;
}
/**
* Implementation of the task indexer that focuses only on task-related data
*/
export class TaskIndexer extends Component implements TaskIndexerInterface {
private taskCache: TaskCache;
private parser: TaskParser;
private lastIndexTime: Map<string, number> = new Map();
// Queue for throttling file indexing
private indexQueue: TFile[] = [];
private isProcessingQueue = false;
constructor(
private app: App,
private vault: Vault,
private metadataCache: MetadataCache,
config?: Partial<TaskParserConfig>
) {
super();
this.taskCache = this.initEmptyCache();
this.parser = new TaskParser(config);
// Setup file change listeners for incremental updates
this.setupEventListeners();
}
/**
* Initialize an empty task cache
*/
private initEmptyCache(): TaskCache {
return {
tasks: new Map<string, Task>(),
files: new Map<string, Set<string>>(),
tags: new Map<string, Set<string>>(),
projects: new Map<string, Set<string>>(),
contexts: new Map<string, Set<string>>(),
dueDate: new Map<string, Set<string>>(),
startDate: new Map<string, Set<string>>(),
scheduledDate: new Map<string, Set<string>>(),
completed: new Map<boolean, Set<string>>(),
priority: new Map<number, Set<string>>(),
};
}
/**
* Setup file change event listeners
*/
private setupEventListeners(): void {
// Watch for file modifications
this.registerEvent(
this.vault.on("modify", (file) => {
if (file instanceof TFile && file.extension === "md") {
this.queueFileForIndexing(file);
}
})
);
// Watch for file deletions
this.registerEvent(
this.vault.on("delete", (file) => {
if (file instanceof TFile && file.extension === "md") {
this.removeFileFromIndex(file);
}
})
);
// Watch for new files
this.registerEvent(
this.vault.on("create", (file) => {
if (file instanceof TFile && file.extension === "md") {
this.queueFileForIndexing(file);
}
})
);
}
/**
* Queue a file for indexing with throttling
*/
private queueFileForIndexing(file: TFile): void {
if (!this.indexQueue.some((f) => f.path === file.path)) {
this.indexQueue.push(file);
}
if (!this.isProcessingQueue) {
this.processIndexQueue();
}
}
/**
* Process the file index queue with throttling
*/
private async processIndexQueue(): Promise<void> {
if (this.indexQueue.length === 0) {
this.isProcessingQueue = false;
return;
}
this.isProcessingQueue = true;
const file = this.indexQueue.shift();
if (file) {
await this.indexFile(file);
// Process next file after a small delay
setTimeout(() => this.processIndexQueue(), 50);
} else {
this.isProcessingQueue = false;
}
}
/**
* Initialize the task indexer
*/
public async initialize(): Promise<void> {
// Start with an empty cache
this.taskCache = this.initEmptyCache();
// Get all markdown files
const files = this.vault.getMarkdownFiles();
// Index the files in batches to avoid UI freezing
const batchSize = 20;
for (let i = 0; i < files.length; i += batchSize) {
const batch = files.slice(i, i + batchSize);
await Promise.all(batch.map((file) => this.indexFile(file)));
// Yield to main thread after each batch
await new Promise((resolve) => setTimeout(resolve, 0));
}
console.log(
`Task indexing complete: ${this.taskCache.tasks.size} tasks found`
);
}
/**
* Get the current task cache
*/
public getCache(): TaskCache {
return this.taskCache;
}
/**
* Index all files in the vault
*/
public async indexAllFiles(): Promise<void> {
await this.initialize();
}
/**
* Index a single file
*/
public async indexFile(file: TFile): Promise<void> {
try {
// Skip if file has not been modified since last indexing
const fileStats = await this.vault.adapter.stat(file.path);
const lastMtime = this.lastIndexTime.get(file.path);
if (lastMtime && fileStats?.mtime && fileStats.mtime <= lastMtime) {
return;
}
// Read file content
const fileContent = await this.vault.cachedRead(file);
const metadata = this.metadataCache.getFileCache(file);
// Parse tasks
const tasks = await this.parser.parseTasksFromFile(
file,
fileContent,
metadata ?? undefined
);
// Remove existing tasks for this file first
this.removeFileFromIndex(file);
// Update cache with new tasks
const fileTaskIds = new Set<string>();
for (const task of tasks) {
// Store task in main task map
this.taskCache.tasks.set(task.id, task);
fileTaskIds.add(task.id);
// Update all indexes
this.updateIndexMaps(task);
}
// Update file index
this.taskCache.files.set(file.path, fileTaskIds);
this.lastIndexTime.set(file.path, Date.now());
} catch (error) {
console.error(`Error indexing file ${file.path}:`, error);
}
}
/**
* Update index for a modified file - just an alias for indexFile
*/
public async updateIndex(file: TFile): Promise<void> {
await this.indexFile(file);
}
/**
* Remove a file from the index
*/
private removeFileFromIndex(file: TFile): void {
const taskIds = this.taskCache.files.get(file.path);
if (!taskIds) return;
// Remove each task from all indexes
for (const taskId of taskIds) {
const task = this.taskCache.tasks.get(taskId);
if (task) {
this.removeTaskFromIndexes(task);
}
// Remove from main task map
this.taskCache.tasks.delete(taskId);
}
// Remove from file index
this.taskCache.files.delete(file.path);
this.lastIndexTime.delete(file.path);
}
/**
* Update all index maps for a task
*/
private updateIndexMaps(task: Task): void {
// Update completed status index
let completedTasks =
this.taskCache.completed.get(task.completed) || new Set();
completedTasks.add(task.id);
this.taskCache.completed.set(task.completed, completedTasks);
// Update tag index
for (const tag of task.tags) {
let tagTasks = this.taskCache.tags.get(tag) || new Set();
tagTasks.add(task.id);
this.taskCache.tags.set(tag, tagTasks);
}
// Update project index
if (task.project) {
let projectTasks =
this.taskCache.projects.get(task.project) || new Set();
projectTasks.add(task.id);
this.taskCache.projects.set(task.project, projectTasks);
}
// Update context index
if (task.context) {
let contextTasks =
this.taskCache.contexts.get(task.context) || new Set();
contextTasks.add(task.id);
this.taskCache.contexts.set(task.context, contextTasks);
}
// Update date indexes
if (task.dueDate) {
const dateStr = formatDateForIndex(task.dueDate);
let dueTasks = this.taskCache.dueDate.get(dateStr) || new Set();
dueTasks.add(task.id);
this.taskCache.dueDate.set(dateStr, dueTasks);
}
if (task.startDate) {
const dateStr = formatDateForIndex(task.startDate);
let startTasks = this.taskCache.startDate.get(dateStr) || new Set();
startTasks.add(task.id);
this.taskCache.startDate.set(dateStr, startTasks);
}
if (task.scheduledDate) {
const dateStr = formatDateForIndex(task.scheduledDate);
let scheduledTasks =
this.taskCache.scheduledDate.get(dateStr) || new Set();
scheduledTasks.add(task.id);
this.taskCache.scheduledDate.set(dateStr, scheduledTasks);
}
// Update priority index
if (task.priority !== undefined) {
let priorityTasks =
this.taskCache.priority.get(task.priority) || new Set();
priorityTasks.add(task.id);
this.taskCache.priority.set(task.priority, priorityTasks);
}
}
/**
* Remove a task from all indexes
*/
private removeTaskFromIndexes(task: Task): void {
// Remove from completed index
const completedTasks = this.taskCache.completed.get(task.completed);
if (completedTasks) {
completedTasks.delete(task.id);
if (completedTasks.size === 0) {
this.taskCache.completed.delete(task.completed);
}
}
// Remove from tag index
for (const tag of task.tags) {
const tagTasks = this.taskCache.tags.get(tag);
if (tagTasks) {
tagTasks.delete(task.id);
if (tagTasks.size === 0) {
this.taskCache.tags.delete(tag);
}
}
}
// Remove from project index
if (task.project) {
const projectTasks = this.taskCache.projects.get(task.project);
if (projectTasks) {
projectTasks.delete(task.id);
if (projectTasks.size === 0) {
this.taskCache.projects.delete(task.project);
}
}
}
// Remove from context index
if (task.context) {
const contextTasks = this.taskCache.contexts.get(task.context);
if (contextTasks) {
contextTasks.delete(task.id);
if (contextTasks.size === 0) {
this.taskCache.contexts.delete(task.context);
}
}
}
// Remove from date indexes
if (task.dueDate) {
const dateStr = formatDateForIndex(task.dueDate);
const dueTasks = this.taskCache.dueDate.get(dateStr);
if (dueTasks) {
dueTasks.delete(task.id);
if (dueTasks.size === 0) {
this.taskCache.dueDate.delete(dateStr);
}
}
}
if (task.startDate) {
const dateStr = formatDateForIndex(task.startDate);
const startTasks = this.taskCache.startDate.get(dateStr);
if (startTasks) {
startTasks.delete(task.id);
if (startTasks.size === 0) {
this.taskCache.startDate.delete(dateStr);
}
}
}
if (task.scheduledDate) {
const dateStr = formatDateForIndex(task.scheduledDate);
const scheduledTasks = this.taskCache.scheduledDate.get(dateStr);
if (scheduledTasks) {
scheduledTasks.delete(task.id);
if (scheduledTasks.size === 0) {
this.taskCache.scheduledDate.delete(dateStr);
}
}
}
// Remove from priority index
if (task.priority !== undefined) {
const priorityTasks = this.taskCache.priority.get(task.priority);
if (priorityTasks) {
priorityTasks.delete(task.id);
if (priorityTasks.size === 0) {
this.taskCache.priority.delete(task.priority);
}
}
}
}
/**
* Query tasks based on filters and sorting criteria
*/
public queryTasks(
filters: TaskFilter[],
sortBy: SortingCriteria[] = []
): Task[] {
if (filters.length === 0 && this.taskCache.tasks.size < 1000) {
// If no filters and small task count, just return all tasks
const allTasks = Array.from(this.taskCache.tasks.values());
return this.applySorting(allTasks, sortBy);
}
// Start with a null set to indicate we haven't applied any filters yet
let resultTaskIds: Set<string> | null = null;
// Apply each filter
for (const filter of filters) {
const filteredIds = this.applyFilter(filter);
if (resultTaskIds === null) {
// First filter
resultTaskIds = filteredIds;
} else if (filter.conjunction === "OR") {
// Union sets (OR)
filteredIds.forEach((id) => resultTaskIds!.add(id));
} else {
// Intersection (AND is default)
resultTaskIds = new Set(
[...resultTaskIds].filter((id) => filteredIds.has(id))
);
}
}
// If we have no filters, include all tasks
if (resultTaskIds === null) {
resultTaskIds = new Set(this.taskCache.tasks.keys());
}
// Convert to task array
const tasks = Array.from(resultTaskIds)
.map((id) => this.taskCache.tasks.get(id)!)
.filter((task) => task !== undefined);
// Apply sorting
return this.applySorting(tasks, sortBy);
}
/**
* Apply a filter to the task cache
*/
private applyFilter(filter: TaskFilter): Set<string> {
switch (filter.type) {
case "tag":
return this.filterByTag(filter);
case "project":
return this.filterByProject(filter);
case "context":
return this.filterByContext(filter);
case "status":
return this.filterByStatus(filter);
case "priority":
return this.filterByPriority(filter);
case "dueDate":
return this.filterByDueDate(filter);
case "startDate":
return this.filterByStartDate(filter);
case "scheduledDate":
return this.filterByScheduledDate(filter);
default:
console.warn(`Unsupported filter type: ${filter.type}`);
return new Set();
}
}
/**
* Filter tasks by tag
*/
private filterByTag(filter: TaskFilter): Set<string> {
if (filter.operator === "contains") {
return this.taskCache.tags.get(filter.value as string) || new Set();
} else if (filter.operator === "!=") {
// Get all task IDs
const allTaskIds = new Set(this.taskCache.tasks.keys());
// Get tasks with the specified tag
const tagTaskIds =
this.taskCache.tags.get(filter.value as string) || new Set();
// Return tasks that don't have the tag
return new Set([...allTaskIds].filter((id) => !tagTaskIds.has(id)));
}
return new Set();
}
/**
* Filter tasks by project
*/
private filterByProject(filter: TaskFilter): Set<string> {
if (filter.operator === "=") {
return (
this.taskCache.projects.get(filter.value as string) || new Set()
);
} else if (filter.operator === "!=") {
// Get all task IDs
const allTaskIds = new Set(this.taskCache.tasks.keys());
// Get tasks with the specified project
const projectTaskIds =
this.taskCache.projects.get(filter.value as string) ||
new Set();
// Return tasks that don't have the project
return new Set(
[...allTaskIds].filter((id) => !projectTaskIds.has(id))
);
} else if (filter.operator === "empty") {
// Get all task IDs
const allTaskIds = new Set(this.taskCache.tasks.keys());
// Get all tasks with any project
const tasksWithProject = new Set<string>();
for (const projectTasks of this.taskCache.projects.values()) {
for (const taskId of projectTasks) {
tasksWithProject.add(taskId);
}
}
// Return tasks without a project
return new Set(
[...allTaskIds].filter((id) => !tasksWithProject.has(id))
);
}
return new Set();
}
/**
* Filter tasks by context
*/
private filterByContext(filter: TaskFilter): Set<string> {
if (filter.operator === "=") {
return (
this.taskCache.contexts.get(filter.value as string) || new Set()
);
} else if (filter.operator === "!=") {
// Get all task IDs
const allTaskIds = new Set(this.taskCache.tasks.keys());
// Get tasks with the specified context
const contextTaskIds =
this.taskCache.contexts.get(filter.value as string) ||
new Set();
// Return tasks that don't have the context
return new Set(
[...allTaskIds].filter((id) => !contextTaskIds.has(id))
);
} else if (filter.operator === "empty") {
// Get all task IDs
const allTaskIds = new Set(this.taskCache.tasks.keys());
// Get all tasks with any context
const tasksWithContext = new Set<string>();
for (const contextTasks of this.taskCache.contexts.values()) {
for (const taskId of contextTasks) {
tasksWithContext.add(taskId);
}
}
// Return tasks without a context
return new Set(
[...allTaskIds].filter((id) => !tasksWithContext.has(id))
);
}
return new Set();
}
/**
* Filter tasks by status (completed or not)
*/
private filterByStatus(filter: TaskFilter): Set<string> {
if (filter.operator === "=") {
return (
this.taskCache.completed.get(filter.value as boolean) ||
new Set()
);
}
return new Set();
}
/**
* Filter tasks by priority
*/
private filterByPriority(filter: TaskFilter): Set<string> {
if (filter.operator === "=") {
return (
this.taskCache.priority.get(filter.value as number) || new Set()
);
} else if (filter.operator === ">") {
// Get tasks with priority higher than the specified value
const result = new Set<string>();
for (const [
priority,
taskIds,
] of this.taskCache.priority.entries()) {
if (priority > (filter.value as number)) {
for (const taskId of taskIds) {
result.add(taskId);
}
}
}
return result;
} else if (filter.operator === "<") {
// Get tasks with priority lower than the specified value
const result = new Set<string>();
for (const [
priority,
taskIds,
] of this.taskCache.priority.entries()) {
if (priority < (filter.value as number)) {
for (const taskId of taskIds) {
result.add(taskId);
}
}
}
return result;
}
return new Set();
}
/**
* Filter tasks by due date
*/
private filterByDueDate(filter: TaskFilter): Set<string> {
if (filter.operator === "=") {
// Exact match on date string (YYYY-MM-DD)
return (
this.taskCache.dueDate.get(filter.value as string) || new Set()
);
} else if (
filter.operator === "before" ||
filter.operator === "after"
) {
// Convert value to Date if it's a string
let compareDate: Date;
if (typeof filter.value === "string") {
compareDate = new Date(filter.value);
} else {
compareDate = new Date(filter.value as number);
}
// Get all tasks with due dates
const result = new Set<string>();
for (const [dateStr, taskIds] of this.taskCache.dueDate.entries()) {
const date = new Date(dateStr);
if (
(filter.operator === "before" && date < compareDate) ||
(filter.operator === "after" && date > compareDate)
) {
for (const taskId of taskIds) {
result.add(taskId);
}
}
}
return result;
} else if (filter.operator === "empty") {
// Get all task IDs
const allTaskIds = new Set(this.taskCache.tasks.keys());
// Get all tasks with any due date
const tasksWithDueDate = new Set<string>();
for (const dueTasks of this.taskCache.dueDate.values()) {
for (const taskId of dueTasks) {
tasksWithDueDate.add(taskId);
}
}
// Return tasks without a due date
return new Set(
[...allTaskIds].filter((id) => !tasksWithDueDate.has(id))
);
}
return new Set();
}
/**
* Filter tasks by start date
*/
private filterByStartDate(filter: TaskFilter): Set<string> {
// Similar implementation to filterByDueDate
if (filter.operator === "=") {
return (
this.taskCache.startDate.get(filter.value as string) ||
new Set()
);
} else if (
filter.operator === "before" ||
filter.operator === "after"
) {
let compareDate: Date;
if (typeof filter.value === "string") {
compareDate = new Date(filter.value);
} else {
compareDate = new Date(filter.value as number);
}
const result = new Set<string>();
for (const [
dateStr,
taskIds,
] of this.taskCache.startDate.entries()) {
const date = new Date(dateStr);
if (
(filter.operator === "before" && date < compareDate) ||
(filter.operator === "after" && date > compareDate)
) {
for (const taskId of taskIds) {
result.add(taskId);
}
}
}
return result;
} else if (filter.operator === "empty") {
const allTaskIds = new Set(this.taskCache.tasks.keys());
const tasksWithStartDate = new Set<string>();
for (const startTasks of this.taskCache.startDate.values()) {
for (const taskId of startTasks) {
tasksWithStartDate.add(taskId);
}
}
return new Set(
[...allTaskIds].filter((id) => !tasksWithStartDate.has(id))
);
}
return new Set();
}
/**
* Filter tasks by scheduled date
*/
private filterByScheduledDate(filter: TaskFilter): Set<string> {
// Similar implementation to filterByDueDate
if (filter.operator === "=") {
return (
this.taskCache.scheduledDate.get(filter.value as string) ||
new Set()
);
} else if (
filter.operator === "before" ||
filter.operator === "after"
) {
let compareDate: Date;
if (typeof filter.value === "string") {
compareDate = new Date(filter.value);
} else {
compareDate = new Date(filter.value as number);
}
const result = new Set<string>();
for (const [
dateStr,
taskIds,
] of this.taskCache.scheduledDate.entries()) {
const date = new Date(dateStr);
if (
(filter.operator === "before" && date < compareDate) ||
(filter.operator === "after" && date > compareDate)
) {
for (const taskId of taskIds) {
result.add(taskId);
}
}
}
return result;
} else if (filter.operator === "empty") {
const allTaskIds = new Set(this.taskCache.tasks.keys());
const tasksWithScheduledDate = new Set<string>();
for (const scheduledTasks of this.taskCache.scheduledDate.values()) {
for (const taskId of scheduledTasks) {
tasksWithScheduledDate.add(taskId);
}
}
return new Set(
[...allTaskIds].filter((id) => !tasksWithScheduledDate.has(id))
);
}
return new Set();
}
/**
* Apply sorting to tasks
*/
private applySorting(tasks: Task[], sortBy: SortingCriteria[]): Task[] {
if (sortBy.length === 0) {
// Default sorting: priority desc, due date asc
return [...tasks].sort((a, b) => {
// First by priority (high to low)
const priorityA = a.priority || 0;
const priorityB = b.priority || 0;
if (priorityA !== priorityB) {
return priorityB - priorityA;
}
// Then by due date (earliest first)
const dueDateA = a.dueDate || Number.MAX_SAFE_INTEGER;
const dueDateB = b.dueDate || Number.MAX_SAFE_INTEGER;
return dueDateA - dueDateB;
});
}
return [...tasks].sort((a, b) => {
for (const { field, direction } of sortBy) {
const valueA = a[field];
const valueB = b[field];
// Handle undefined values
if (valueA === undefined && valueB === undefined) {
continue;
} else if (valueA === undefined) {
return direction === "asc" ? 1 : -1;
} else if (valueB === undefined) {
return direction === "asc" ? -1 : 1;
}
// Compare values
if (valueA !== valueB) {
const multiplier = direction === "asc" ? 1 : -1;
if (
typeof valueA === "string" &&
typeof valueB === "string"
) {
return valueA.localeCompare(valueB) * multiplier;
} else if (
typeof valueA === "number" &&
typeof valueB === "number"
) {
return (valueA - valueB) * multiplier;
} else if (
valueA instanceof Date &&
valueB instanceof Date
) {
return (
(valueA.getTime() - valueB.getTime()) * multiplier
);
} else {
// Convert to string and compare as fallback
return (
String(valueA).localeCompare(String(valueB)) *
multiplier
);
}
}
}
return 0;
});
}
/**
* Get task by ID
*/
public getTaskById(id: string): Task | undefined {
return this.taskCache.tasks.get(id);
}
/**
* Create a new task
*/
public async createTask(taskData: Partial<Task>): Promise<Task> {
throw new Error("Not implemented");
}
/**
* Update an existing task
*/
public async updateTask(task: Task): Promise<void> {
throw new Error("Not implemented");
}
/**
* Delete a task
*/
public async deleteTask(taskId: string): Promise<void> {
throw new Error("Not implemented");
}
}

View file

@ -0,0 +1,257 @@
/**
* Optimized task parser focused on task data only
*/
import { CachedMetadata, FileStats, TFile } from "obsidian";
import { Task, TaskParserConfig } from "../types/TaskIndex";
import { v4 as uuidv4 } from "uuid";
/**
* Default configuration for the task parser
*/
export const DEFAULT_TASK_PARSER_CONFIG: TaskParserConfig = {
taskRegex: /^([\s>]*- \[(.)\])\s*(.*)$/m,
startDateFormat: "📅 YYYY-MM-DD",
dueDateFormat: "⏳ YYYY-MM-DD",
scheduledDateFormat: "⏰ YYYY-MM-DD",
projectPrefix: "#project/",
contextPrefix: "@",
priorityMarkers: {
"🔺": 5, // Highest
"⏫": 4, // High
"🔼": 3, // Medium
"🔽": 2, // Low
"⏬️": 1, // Lowest
"[#A]": 4, // High (letter format)
"[#B]": 3, // Medium (letter format)
"[#C]": 2, // Low (letter format)
},
};
export class TaskParser {
// Regular expressions for parsing task components
private readonly startDateRegex = /📅 (\d{4}-\d{2}-\d{2})/;
private readonly completedDateRegex = /✅ (\d{4}-\d{2}-\d{2})/;
private readonly dueDateRegex = /⏳ (\d{4}-\d{2}-\d{2})/;
private readonly scheduledDateRegex = /⏰ (\d{4}-\d{2}-\d{2})/;
private readonly recurrenceRegex = /🔁 (.*?)(?=\s|$)/;
private readonly tagRegex = /#[\w\/-]+/g;
private readonly contextRegex = /@[\w-]+/g;
private readonly priorityRegex = /🔼|⏫|🔽|⏬️|🔺|\[#[A-C]\]/;
private readonly estimatedTimeRegex = /estimated:\s*(\d+)([hm])/i;
private config: TaskParserConfig;
constructor(config: Partial<TaskParserConfig> = {}) {
this.config = { ...DEFAULT_TASK_PARSER_CONFIG, ...config };
}
/**
* Parse a task from a text line
*/
parseTask(text: string, filePath: string, lineNum: number): Task | null {
const match = text.match(this.config.taskRegex);
if (!match) return null;
const [, prefix, status, content] = match;
const completed = status.toLowerCase() === "x";
// Basic task info
const task: Task = {
id: uuidv4(),
content: content.trim(),
filePath,
line: lineNum,
completed,
originalMarkdown: text,
tags: [],
children: [],
};
// Extract metadata
this.extractDates(task, content);
this.extractTags(task, content);
this.extractContext(task, content);
this.extractPriority(task, content);
this.extractEstimatedTime(task, content);
this.extractRecurrence(task, content);
return task;
}
/**
* Extract dates from task content
*/
private extractDates(task: Task, content: string): void {
// Start date
const startDateMatch = content.match(this.startDateRegex);
if (startDateMatch) {
task.startDate = new Date(startDateMatch[1]).getTime();
}
// Due date
const dueDateMatch = content.match(this.dueDateRegex);
if (dueDateMatch) {
task.dueDate = new Date(dueDateMatch[1]).getTime();
}
// Scheduled date
const scheduledDateMatch = content.match(this.scheduledDateRegex);
if (scheduledDateMatch) {
task.scheduledDate = new Date(scheduledDateMatch[1]).getTime();
}
// Completion date
const completedDateMatch = content.match(this.completedDateRegex);
if (completedDateMatch) {
task.completedDate = new Date(completedDateMatch[1]).getTime();
}
}
/**
* Extract tags from task content
*/
private extractTags(task: Task, content: string): void {
const tagMatches = content.match(this.tagRegex) || [];
task.tags = tagMatches.map((tag) => tag.trim());
// Check for project tags
const projectTag = task.tags.find((tag) =>
tag.startsWith(this.config.projectPrefix!)
);
if (projectTag) {
task.project = projectTag.substring(
this.config.projectPrefix!.length
);
}
}
/**
* Extract context from task content
*/
private extractContext(task: Task, content: string): void {
const contextMatches = content.match(this.contextRegex) || [];
if (contextMatches.length > 0) {
// Use the first context tag as the primary context
task.context = contextMatches[0]?.substring(1); // Remove the @ symbol
}
}
/**
* Extract priority from task content
*/
private extractPriority(task: Task, content: string): void {
const priorityMatch = content.match(this.priorityRegex);
if (priorityMatch && this.config.priorityMarkers) {
task.priority =
this.config.priorityMarkers[priorityMatch[0]] || undefined;
}
}
/**
* Extract estimated time from task content
*/
private extractEstimatedTime(task: Task, content: string): void {
const timeMatch = content.match(this.estimatedTimeRegex);
if (timeMatch) {
const value = parseInt(timeMatch[1]);
const unit = timeMatch[2].toLowerCase();
if (unit === "h") {
task.estimatedTime = value * 60; // Convert hours to minutes
} else {
task.estimatedTime = value; // Already in minutes
}
}
}
/**
* Extract recurrence information from task content
*/
private extractRecurrence(task: Task, content: string): void {
const recurrenceMatch = content.match(this.recurrenceRegex);
if (recurrenceMatch) {
task.recurrence = recurrenceMatch[1];
}
}
/**
* Generate markdown text from a task object
*/
generateMarkdown(task: Task): string {
let markdown = `- [${task.completed ? "x" : " "}] ${task.content}`;
// This is simplified - a full implementation would reconstruct the original
// markdown with all the metadata in the correct format
return markdown;
}
/**
* Parse all tasks from a file
*/
async parseTasksFromFile(
file: TFile,
fileContent: string,
metadata?: CachedMetadata
): Promise<Task[]> {
const lines = fileContent.split("\n");
const tasks: Task[] = [];
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
const task = this.parseTask(line, file.path, i);
if (task) {
tasks.push(task);
}
}
// Build parent-child relationships
this.buildTaskHierarchy(tasks);
return tasks;
}
/**
* Build parent-child relationships between tasks
*/
private buildTaskHierarchy(tasks: Task[]): void {
// This is a simple indent-based approach
// A more sophisticated implementation would use Obsidian's list item metadata
// Sort tasks by line number
tasks.sort((a, b) => a.line - b.line);
// Build parent-child relationships based on indentation
for (let i = 0; i < tasks.length; i++) {
const currentTask = tasks[i];
const currentIndent = this.getIndentLevel(
currentTask.originalMarkdown
);
// Look for potential parent tasks (must be before current task and have less indentation)
for (let j = i - 1; j >= 0; j--) {
const potentialParent = tasks[j];
const parentIndent = this.getIndentLevel(
potentialParent.originalMarkdown
);
if (parentIndent < currentIndent) {
// Found a parent
currentTask.parent = potentialParent.id;
potentialParent.children.push(currentTask.id);
break;
}
}
}
}
/**
* Get indentation level of a line
*/
private getIndentLevel(line: string): number {
const match = line.match(/^(\s*)/);
return match ? match[1].length : 0;
}
}

View file

@ -0,0 +1,696 @@
import { JsonLink, Link } from "expression/link";
import { getExtension, getFileTitle } from "utils/normalizers";
import { CachedMetadata, FileStats, FrontMatterCache } from "obsidian";
import { parse as parseYaml } from "yaml";
import BTree from "sorted-btree";
import {
InlineField,
JsonInlineField,
asInlineField,
extractFullLineField,
extractInlineFields,
jsonInlineField,
} from "./inline-field";
import {
JsonMarkdownBlock,
JsonMarkdownListBlock,
JsonMarkdownListItem,
JsonMarkdownPage,
JsonMarkdownSection,
JsonMarkdownTaskItem,
JsonMarkdownDatablock,
JsonMarkdownCodeblock,
JsonFrontmatterEntry,
} from "index/types/json/markdown";
import { JsonConversion } from "index/types/json/common";
import { YamlConversion } from "index/types/yaml";
import { mapObjectValues } from "utils/data";
/** Matches yaml datablocks, which show up as independent objects in the datacore index. */
const YAML_DATA_REGEX = /```yaml:data/i;
/** Matches the start of any codeblock fence. */
const CODEBLOCK_FENCE_REGEX = /^(?:```|~~~)(.*)$/im;
/** Matches list items (including inside text blocks). */
const LIST_ITEM_REGEX =
/^[\s>]*(\d+\.|\d+\)|\*|-|\+)\s*(\[.{0,1}\])?\s*(.*)$/mu;
/**
* Given the raw source and Obsidian metadata for a given markdown file,
* return full markdown file metadata.
*/
export function markdownImport(
path: string,
markdown: string,
metadata: CachedMetadata,
stats: FileStats
): JsonMarkdownPage {
const {
lines,
metadata: pageMetadata,
frontmatter,
sections,
} = markdownSourceImport(path, markdown, metadata);
const page = new PageData(
path,
stats,
lines.length,
pageMetadata,
sections,
frontmatter
);
return page.build();
}
/** Import markdown file metadata, producing all relevant markdown data. */
export function markdownSourceImport(
path: string,
markdown: string,
metadata: CachedMetadata
): {
lines: string[];
metadata: Metadata;
frontmatter?: Record<string, JsonFrontmatterEntry>;
sections: SectionData[];
} {
const frontmatter: Record<string, JsonFrontmatterEntry> | undefined =
metadata.frontmatter
? parseFrontmatterBlock(metadata.frontmatter)
: undefined;
const lines = markdown.split(/\r\n|\r|\n/);
const markdownMetadata = new Metadata();
const sectionArray: SectionData[] = [];
//////////////
// Sections //
//////////////
const metaheadings = metadata.headings ?? [];
metaheadings.sort((a, b) => a.position.start.line - b.position.start.line);
const sections = new BTree<number, SectionData>(undefined, (a, b) => a - b);
for (let index = 0; index < metaheadings.length; index++) {
const entry = metaheadings[index];
const start = entry.position.start.line;
const end =
index == metaheadings.length - 1
? lines.length
: metaheadings[index + 1].position.start.line;
const section = new SectionData(
start,
end,
entry.heading,
entry.level,
index + 1
);
sections.set(start, section);
}
// Add an implicit section for the "heading" section of the page if there is not an immediate header but there is
// some content in the file. If there are other sections, then go up to that, otherwise, go for the entire file.
if (sections.size == 0) {
if (!emptylines(lines, 0, lines.length)) {
const section = new SectionData(
0,
lines.length,
getFileTitle(path),
1,
0
);
sections.set(0, section);
}
} else {
// Find the start of the first section.
const first = sections.getPairOrNextHigher(0)?.[1]!!;
if (first.start > 0 && !emptylines(lines, 0, first.start)) {
const section = new SectionData(
0,
first.start,
getFileTitle(path),
1,
0
);
sections.set(0, section);
}
}
////////////
// Blocks //
////////////
// All blocks; we will assign tags and other metadata to blocks as we encounter them. At the end, only blocks that
// have actual metadata will be stored to save on memory pressure.
const blocks = new BTree<number, BlockData>(undefined, (a, b) => a - b);
let blockOrdinal = 1;
for (const block of metadata.sections || []) {
// Skip headings blocks, we handle them specially as sections.
if (block.type === "heading") continue;
const start = block.position.start.line;
const end = block.position.end.line + 1;
const startLine = lines[start]; // to use to check the codeblock type
if (block.type === "list") {
blocks.set(
start,
new ListBlockData(start, end, blockOrdinal++, block.id)
);
} else if (block.type == "code" && YAML_DATA_REGEX.test(startLine)) {
const yaml: string = lines
.slice(start + 1, end - 1)
.join("\n")
.replace(/\t/gm, " ");
const split: Record<string, JsonFrontmatterEntry> =
parseFrontmatterBlock(parseYaml(yaml));
blocks.set(
start,
new DatablockData(start, end, blockOrdinal++, split, block.id)
);
} else if (block.type === "code") {
// Check if the block is fenced.
const match = startLine.match(CODEBLOCK_FENCE_REGEX);
if (!match) {
// This is an indented-style codeblock.
blocks.set(
start,
new CodeblockData(
start,
end,
blockOrdinal++,
[],
"indent",
start,
end,
block.id
)
);
} else {
const languages =
match.length > 1 && match[1] ? match[1].split(",") : [];
blocks.set(
start,
new CodeblockData(
start,
end,
blockOrdinal++,
languages,
"fenced",
start + 1,
end - 1,
block.id
)
);
}
} else {
blocks.set(
start,
new BaseBlockData(
start,
end,
blockOrdinal++,
block.type,
block.id
)
);
}
}
// Add blocks to sections.
for (const block of blocks.values()) {
lookup(block.start, sections)?.block(block);
}
///////////
// Lists //
///////////
// All list items in lists. Start with a simple trivial pass.
const listItems = new BTree<number, ListItemData>(
undefined,
(a, b) => a - b
);
for (const list of metadata.listItems || []) {
const line = lines[list.position.start.line];
// TODO: Implement flag which skips indexing list items.
const match = line.match(LIST_ITEM_REGEX);
let symbol = undefined,
text = undefined;
if (match) {
symbol = match[1];
text = match[3];
}
const item = new ListItemData(
list.position.start.line,
list.position.end.line + 1,
list.parent,
symbol,
list.id,
list.task,
text
);
listItems.set(item.start, item);
}
// In the second list pass, actually construct the list heirarchy.
for (const item of listItems.values()) {
if (item.parentLine < 0) {
const listBlock = blocks.get(-item.parentLine);
if (!listBlock || !(listBlock.type === "list")) continue;
(listBlock as ListBlockData).items.push(item);
} else {
listItems.get(item.parentLine)?.elements.push(item);
}
}
//////////
// Tags //
//////////
// For each tag, assign it to the appropriate section and block that it is a part of.
for (const tagdef of metadata.tags ?? []) {
const tag = tagdef.tag.startsWith("#") ? tagdef.tag : "#" + tagdef.tag;
const line = tagdef.position.start.line;
markdownMetadata.tag(tag);
lookup(line, sections)?.metadata.tag(tag);
lookup(line, blocks)?.metadata.tag(tag);
lookup(line, listItems)?.metadata.tag(tag);
}
// Add frontmatter tags.
if (metadata.frontmatter) {
for (const rawtag of extractTags(metadata.frontmatter)) {
const tag = rawtag.startsWith("#") ? rawtag : "#" + rawtag;
markdownMetadata.tag(tag);
}
}
///////////
// Links //
///////////
for (let linkdef of metadata.links ?? []) {
const link = Link.infer(linkdef.link);
const line = linkdef.position.start.line;
markdownMetadata.link(link);
lookup(line, sections)?.metadata.link(link);
lookup(line, blocks)?.metadata.link(link);
lookup(line, listItems)?.metadata.link(link);
}
///////////////////////
// Frontmatter Links //
///////////////////////
// Frontmatter links are only assigned to the page.
for (const linkdef of metadata.frontmatterLinks ?? []) {
markdownMetadata.link(
Link.infer(linkdef.link, false, linkdef.displayText)
);
}
///////////////////
// Inline Fields //
///////////////////
for (const field of iterateInlineFields(lines)) {
const line = field.position.line;
markdownMetadata.inlineField(field);
lookup(line, sections)?.metadata.inlineField(field);
lookup(line, blocks)?.metadata.inlineField(field);
lookup(line, listItems)?.metadata.inlineField(field);
}
sectionArray.push(...sections.values());
return {
lines,
frontmatter,
metadata: markdownMetadata,
sections: sectionArray,
};
}
//////////////////
// Parsing Aids //
//////////////////
/** Check if the given line range is all empty. Start is inclusive, end exclusive. */
function emptylines(lines: string[], start: number, end: number): boolean {
for (let index = start; index < end; index++) {
if (lines[index].trim() !== "") return false;
}
return true;
}
/**
* Yields all inline fields found in the document by traversing line by line through the document. Performs some optimizations
* to skip extra-large lines, and can be disabled.
*/
export function* iterateInlineFields(
content: string[]
): Generator<InlineField> {
for (let lineno = 0; lineno < content.length; lineno++) {
const line = content[lineno];
// Fast-bailout for lines that are too long or do not contain '::'.
if (line.length > 32768 || !line.includes("::")) continue;
// TODO: Re-add support for those custom emoji fields on tasks and similar.
let inlineFields = extractInlineFields(line);
if (inlineFields.length > 0) {
for (let ifield of inlineFields)
yield asInlineField(ifield, lineno);
} else {
let fullLine = extractFullLineField(line);
if (fullLine) yield asInlineField(fullLine, lineno);
}
}
}
/** Top-level function which maps a YAML block - including frontmatter - into frontmatter entries. */
export function parseFrontmatterBlock(
block: Record<string, any>
): Record<string, JsonFrontmatterEntry> {
const result: Record<string, JsonFrontmatterEntry> = {};
for (const key of Object.keys(block)) {
const value = block[key];
result[key.toLowerCase()] = {
key: key,
value: JsonConversion.json(YamlConversion.literal(value)),
raw: value,
};
}
return result;
}
/** Finds an element which contains the given line. */
export function lookup<T extends { start: number; end: number }>(
line: number,
tree: BTree<number, T>
): T | undefined {
const target = tree.getPairOrNextLower(line)?.[1];
if (target && target.end >= line) return target;
return undefined;
}
/** Extract tags intelligently from frontmatter. Handles arrays, numbers, and strings. */
export function extractTags(metadata: FrontMatterCache): string[] {
let tagKeys = Object.keys(metadata).filter(
(t) => t.toLowerCase() == "tags" || t.toLowerCase() == "tag"
);
return tagKeys
.map((k) => splitFrontmatterTagOrAlias(metadata[k], /[,\s]+/))
.reduce((p, c) => p.concat(c), [])
.map((str) => (str.startsWith("#") ? str : "#" + str));
}
/** Split a frontmatter list into separate elements; handles actual lists, comma separated lists, and single elements. */
export function splitFrontmatterTagOrAlias(data: any, on: RegExp): string[] {
if (data == null || data == undefined) return [];
if (Array.isArray(data)) {
return data
.filter((s) => !!s)
.map((s) => splitFrontmatterTagOrAlias(s, on))
.reduce((p, c) => p.concat(c), []);
}
// Force to a string to handle numbers and so on.
return ("" + data)
.split(on)
.filter((t) => !!t)
.map((t) => t.trim())
.filter((t) => t.length > 0);
}
///////////////////////
// Builder Utilities //
///////////////////////
/** Convienent shared utility for tracking metadata - links, tags, and so on. */
export class Metadata {
public tags: Set<string> = new Set();
public links: Link[] = [];
public inlineFields: Record<string, InlineField> = {};
/** Add a tag to the metadata. */
public tag(tag: string) {
this.tags.add(tag);
}
/** Add a link to the metadata. */
public link(link: Link) {
if (this.links.find((v) => v.equals(link))) return;
this.links.push(link);
}
/** Add an inline field to the metadata. */
public inlineField(field: InlineField) {
const lower = field.key.toLowerCase();
if (
Object.keys(this.inlineFields).some(
(key) => key.toLowerCase() == lower
)
)
return;
this.inlineFields[lower] = field;
}
/** Return a list of unique added tags. */
public finishTags(): string[] {
return Array.from(this.tags);
}
/** Return a list of JSON-serialized links. */
public finishLinks(): JsonLink[] {
return this.links.map((link) => link.toObject());
}
/** Return a list of JSON-serialized inline fields. */
public finishInlineFields(): Record<string, JsonInlineField> {
return mapObjectValues(this.inlineFields, jsonInlineField);
}
}
/** Convienent utility for constructing page objects. */
export class PageData {
public constructor(
public path: string,
public stats: FileStats,
public length: number,
public metadata: Metadata,
public sections: SectionData[],
public frontmatter?: Record<string, JsonFrontmatterEntry>
) {}
public build(): JsonMarkdownPage {
return {
$path: this.path,
$ctime: this.stats.ctime,
$mtime: this.stats.mtime,
$size: this.stats.size,
$extension: getExtension(this.path),
$position: { start: 0, end: this.length },
$tags: this.metadata.finishTags(),
$links: this.metadata.finishLinks(),
$infields: this.metadata.finishInlineFields(),
$sections: this.sections.map((x) => x.build()),
$frontmatter: this.frontmatter,
};
}
}
/** Convienent utility for constructing markdown sections. */
export class SectionData {
public blocks: BlockData[] = [];
public metadata: Metadata = new Metadata();
public constructor(
public start: number,
public end: number,
public title: string,
public level: number,
public ordinal: number
) {}
public block(block: BlockData) {
this.blocks.push(block);
}
public build(): JsonMarkdownSection {
return {
$title: this.title,
$ordinal: this.ordinal,
$level: this.level,
$tags: this.metadata.finishTags(),
$infields: this.metadata.finishInlineFields(),
$links: this.metadata.finishLinks(),
$position: { start: this.start, end: this.end },
$blocks: this.blocks.map((block) => block.build()),
};
}
}
/** Constructs markdown list blocks specifically. */
export class ListBlockData {
public type: string = "list";
public metadata: Metadata = new Metadata();
public items: ListItemData[] = [];
public constructor(
public start: number,
public end: number,
public ordinal: number,
public blockId?: string
) {}
public build(): JsonMarkdownListBlock {
return {
$ordinal: this.ordinal,
$position: { start: this.start, end: this.end },
$infields: this.metadata.finishInlineFields(),
$tags: this.metadata.finishTags(),
$links: this.metadata.finishLinks(),
$type: "list",
$blockId: this.blockId,
$elements: this.items.map((item) => item.build()),
};
}
}
/** Constructs markdown codeblocks specifically. */
export class CodeblockData {
public type: string = "codeblock";
public metadata: Metadata = new Metadata();
public constructor(
public start: number,
public end: number,
public ordinal: number,
public languages: string[],
public style: "indent" | "fenced",
public contentStart: number,
public contentEnd: number,
public blockId?: string
) {}
public build(): JsonMarkdownCodeblock {
return {
$type: "codeblock",
$ordinal: this.ordinal,
$position: { start: this.start, end: this.end },
$infields: this.metadata.finishInlineFields(),
$tags: this.metadata.finishTags(),
$links: this.metadata.finishLinks(),
$blockId: this.blockId,
$languages: this.languages,
$style: this.style,
$contentPosition: {
start: this.contentStart,
end: this.contentEnd,
},
};
}
}
/** Constructs markdown datablocks specifically. */
export class DatablockData {
public type: string = "datablock";
public metadata: Metadata = new Metadata();
public constructor(
public start: number,
public end: number,
public ordinal: number,
public data: Record<string, JsonFrontmatterEntry>,
public blockId?: string
) {}
public build(): JsonMarkdownDatablock {
return {
$type: "datablock",
$ordinal: this.ordinal,
$position: { start: this.start, end: this.end },
$infields: this.metadata.finishInlineFields(),
$tags: this.metadata.finishTags(),
$links: this.metadata.finishLinks(),
$blockId: this.blockId,
$data: this.data,
};
}
}
/** Base block metadata used for non-specific blocks. */
export class BaseBlockData {
public metadata: Metadata = new Metadata();
public constructor(
public start: number,
public end: number,
public ordinal: number,
public type: string,
public blockId?: string
) {}
public build(): JsonMarkdownBlock {
return {
$type: this.type,
$ordinal: this.ordinal,
$position: { start: this.start, end: this.end },
$infields: this.metadata.finishInlineFields(),
$tags: this.metadata.finishTags(),
$links: this.metadata.finishLinks(),
$blockId: this.blockId,
};
}
}
export type BlockData =
| ListBlockData
| CodeblockData
| DatablockData
| BaseBlockData;
/** Utility for constructing markdown list items. */
export class ListItemData {
public metadata: Metadata = new Metadata();
public elements: ListItemData[] = [];
public constructor(
public start: number,
public end: number,
public parentLine: number,
public symbol?: string,
public blockId?: string,
public status?: string,
public text?: string
) {}
public build(): JsonMarkdownListItem {
return {
$parentLine: this.parentLine,
$position: { start: this.start, end: this.end },
$blockId: this.blockId,
$elements: this.elements.map((element) => element.build()),
$type: this.status ? "task" : "list",
$infields: this.metadata.finishInlineFields(),
$tags: this.metadata.finishTags(),
$links: this.metadata.finishLinks(),
$status: this.status,
$symbol: this.symbol,
$text: this.text,
} as JsonMarkdownTaskItem;
}
}

0
src/utils/readme.md Normal file
View file

View file

@ -0,0 +1,169 @@
/**
* Optimized task indexing system focused on task-related data only
*/
import { Component, TFile } from "obsidian";
/** Core task data structure */
export interface Task {
/** Unique identifier for the task */
id: string;
/** Task content text */
content: string;
/** File path where the task is located */
filePath: string;
/** Line number in the file */
line: number;
/** Whether the task is completed or not */
completed: boolean;
/** Original markdown text */
originalMarkdown: string;
/** Creation date (optional) */
createdDate?: number;
/** Start date for the task (Tasks plugin compatible) */
startDate?: number;
/** Scheduled date (Tasks plugin compatible) */
scheduledDate?: number;
/** Due date for the task */
dueDate?: number;
/** Date when the task was completed */
completedDate?: number;
/** Recurrence pattern (Tasks plugin compatible) */
recurrence?: string;
/** Tags associated with the task */
tags: string[];
/** Project associated with task (derived from frontmatter or special tags) */
project?: string;
/** Context for the task (e.g. @home, @work) */
context?: string;
/** Priority level (1-3, higher is more important) */
priority?: number;
/** Parent task ID for hierarchical tasks */
parent?: string;
/** Child task IDs */
children: string[];
/** Estimated time in minutes */
estimatedTime?: number;
/** Actual time spent in minutes */
actualTime?: number;
}
/** High-performance cache structure for tasks */
export interface TaskCache {
/** Main task store: taskId -> Task */
tasks: Map<string, Task>;
/** File index: filePath -> Set<taskIds> */
files: Map<string, Set<string>>;
/** Tag index: tag -> Set<taskIds> */
tags: Map<string, Set<string>>;
/** Project index: project -> Set<taskIds> */
projects: Map<string, Set<string>>;
/** Context index: context -> Set<taskIds> */
contexts: Map<string, Set<string>>;
/** Due date index: dueDate(YYYY-MM-DD) -> Set<taskIds> */
dueDate: Map<string, Set<string>>;
/** Start date index: startDate(YYYY-MM-DD) -> Set<taskIds> */
startDate: Map<string, Set<string>>;
/** Scheduled date index: scheduledDate(YYYY-MM-DD) -> Set<taskIds> */
scheduledDate: Map<string, Set<string>>;
/** Completion status index: boolean -> Set<taskIds> */
completed: Map<boolean, Set<string>>;
/** Priority index: priority -> Set<taskIds> */
priority: Map<number, Set<string>>;
}
/** Task filter interface for querying tasks */
export interface TaskFilter {
type:
| "tag"
| "project"
| "context"
| "dueDate"
| "startDate"
| "scheduledDate"
| "status"
| "priority"
| "recurrence";
operator:
| "="
| "!="
| "<"
| ">"
| "contains"
| "empty"
| "not-empty"
| "before"
| "after";
value: any;
conjunction?: "AND" | "OR";
}
/** Sort criteria for task lists */
export interface SortingCriteria {
field: keyof Task;
direction: "asc" | "desc";
}
/** Task parsing configuration */
export interface TaskParserConfig {
/** Regular expression to match task items */
taskRegex: RegExp;
/** Start date format for parsing */
startDateFormat?: string;
/** Due date format for parsing */
dueDateFormat?: string;
/** Scheduled date format for parsing */
scheduledDateFormat?: string;
/** Project tag prefix */
projectPrefix?: string;
/** Context tag prefix */
contextPrefix?: string;
/** Task priority markers */
priorityMarkers?: Record<string, number>;
}
/** Task indexer interface */
export interface TaskIndexer extends Component {
/** Initialize the task indexer */
initialize(): Promise<void>;
/** Get the current task cache */
getCache(): TaskCache;
/** Index a single file */
indexFile(file: TFile): Promise<void>;
/** Index all files in the vault */
indexAllFiles(): Promise<void>;
/** Update index for a modified file */
updateIndex(file: TFile): Promise<void>;
/** Query tasks based on filters and sorting criteria */
queryTasks(filters: TaskFilter[], sortBy: SortingCriteria[]): Task[];
/** Get task by ID */
getTaskById(id: string): Task | undefined;
/** Create a new task */
createTask(task: Partial<Task>): Promise<Task>;
/** Update an existing task */
updateTask(task: Task): Promise<void>;
/** Delete a task */
deleteTask(taskId: string): Promise<void>;
}

940
src/utils/types/markdown.ts Normal file
View file

@ -0,0 +1,940 @@
/**
* @module indexables
*/
import { Link, Literal, Literals } from "expression/literal";
import { getFileTitle } from "utils/normalizers";
import {
FILE_TYPE,
File,
Indexable,
LINKABLE_TYPE,
LINKBEARING_TYPE,
Linkable,
Linkbearing,
TAGGABLE_TYPE,
Taggable,
} from "index/types/indexable";
import { DateTime } from "luxon";
import {
Extractors,
FIELDBEARING_TYPE,
Field,
FieldExtractor,
Fieldbearing,
} from "../../expression/field";
import {
InlineField,
jsonInlineField,
valueInlineField,
} from "index/import/inline-field";
import {
LineSpan,
JsonMarkdownPage,
JsonMarkdownSection,
JsonMarkdownBlock,
JsonMarkdownListBlock,
JsonMarkdownListItem,
JsonMarkdownTaskItem,
JsonMarkdownDatablock as JsonMarkdownDatablock,
JsonMarkdownCodeblock,
JsonFrontmatterEntry,
} from "./json/markdown";
import { mapObjectValues } from "utils/data";
import { JsonConversion } from "./json/common";
/** A markdown file in the vault; the source of most metadata. */
export class MarkdownPage
implements File, Linkbearing, Taggable, Indexable, Fieldbearing
{
/** All of the types that a markdown file is. */
static TYPES = [
FILE_TYPE,
"markdown",
"page",
TAGGABLE_TYPE,
LINKABLE_TYPE,
LINKBEARING_TYPE,
FIELDBEARING_TYPE,
];
// Use static types for all markdown files.
$types: string[] = MarkdownPage.TYPES;
$typename: string = "Page";
// Markdown file IDs are always just the full path.
get $id() {
return this.$path;
}
// The file of a file is... it's file.
get $file() {
return this.$path;
}
/** Frontmatter values in the file, if present. Maps lower case frontmatter key -> entry. */
$frontmatter?: Record<string, FrontmatterEntry>;
/** Map of all distinct inline fields in the document. Maps lower case key name -> full metadata. */
$infields: Record<string, InlineField>;
/** The path this file exists at. */
$path: string;
/** Obsidian-provided date this page was created. */
$ctime: DateTime;
/** Obsidian-provided date this page was modified. */
$mtime: DateTime;
/** The extension; for markdown files, almost always '.md'. */
$extension: string;
/** Obsidian-provided size of this page in bytes. */
$size: number = 0;
/** The full extent of the file (start 0, end the number of lines in the file.) */
$position: LineSpan;
/** The exact tags in the file. */
$tags: string[];
/** All links in the file. */
$links: Link[];
/**
* All child markdown sections of this markdown file. The initial section before any content is special and is
* named with the title of the file.
*/
$sections: MarkdownSection[] = [];
/** Create a markdown file from the given raw values. */
static from(
raw: JsonMarkdownPage,
normalizer: LinkNormalizer = NOOP_NORMALIZER
): MarkdownPage {
const sections = raw.$sections.map((sect) =>
MarkdownSection.from(sect, raw.$path, normalizer)
);
return new MarkdownPage({
$path: raw.$path,
$frontmatter: raw.$frontmatter
? mapObjectValues(raw.$frontmatter, (fm) =>
normalizeLinks(valueFrontmatterEntry(fm), normalizer)
)
: undefined,
$infields: mapObjectValues(raw.$infields, (field) =>
normalizeLinks(valueInlineField(field), normalizer)
),
$ctime: DateTime.fromMillis(raw.$ctime),
$mtime: DateTime.fromMillis(raw.$mtime),
$extension: raw.$extension,
$size: raw.$size,
$position: raw.$position,
$tags: raw.$tags,
$links: raw.$links.map((link) => normalizer(Link.fromObject(link))),
$sections: sections,
});
}
private constructor(init: Partial<MarkdownPage>) {
Object.assign(this, init);
}
/** Return the number of lines in the document. */
get $lineCount() {
return this.$position.end;
}
/** The name of the file. */
get $name() {
return getFileTitle(this.$path);
}
/** A link to this file. */
get $link() {
return Link.file(this.$path);
}
/** All of the indexed fields in this object. */
get fields(): Field[] {
return MarkdownPage.FIELD_DEF(this);
}
/** Get the full field definition for the given field. */
public field(key: string): Field | undefined {
return MarkdownPage.FIELD_DEF(this, key)?.[0];
}
/** Get the value for the given field. */
public value(key: string): Literal | undefined {
return this.field(key)?.value;
}
/** Convert this page into it's partial representation for saving. */
public json(): JsonMarkdownPage {
return {
$path: this.$path,
$frontmatter: this.$frontmatter
? mapObjectValues(this.$frontmatter, jsonFrontmatterEntry)
: undefined,
$infields: mapObjectValues(this.$infields, jsonInlineField),
$ctime: this.$ctime.toMillis(),
$mtime: this.$mtime.toMillis(),
$extension: this.$extension,
$size: this.$size,
$position: this.$position,
$tags: this.$tags,
$links: this.$links.map((link) => link.toObject()),
$sections: this.$sections.map((sect) => sect.json()),
};
}
private static FIELD_DEF: FieldExtractor<MarkdownPage> = Extractors.merge(
Extractors.intrinsics(),
Extractors.frontmatter((f) => f.$frontmatter),
Extractors.inlineFields((f) => f.$infields)
);
}
export class MarkdownSection
implements Indexable, Taggable, Linkable, Linkbearing, Fieldbearing
{
/** All of the types that a markdown section is. */
static TYPES = [
"markdown",
"section",
TAGGABLE_TYPE,
LINKABLE_TYPE,
LINKBEARING_TYPE,
FIELDBEARING_TYPE,
];
/** Path of the file that this section is in. */
$types: string[] = MarkdownSection.TYPES;
$typename: string = "Section";
$id: string;
$file: string;
/** The index of this section in the file. */
$ordinal: number;
/** The title of the section; the root (implicit) section will have the title of the page. */
$title: string;
/** The indentation level of the section (1 - 6). */
$level: number;
/** The span of lines indicating the position of the section. */
$position: LineSpan;
/** All tags on the file. */
$tags: string[];
/** All links in the file. */
$links: Link[];
/** All of the markdown blocks in this section. */
$blocks: MarkdownBlock[];
/** Map of all distinct inline fields in the document, from key name -> metadata. */
$infields: Record<string, InlineField>;
/** Convert raw markdown section data to the appropriate class. */
static from(
raw: JsonMarkdownSection,
file: string,
normalizer: LinkNormalizer = NOOP_NORMALIZER
): MarkdownSection {
const blocks = raw.$blocks.map((block) =>
MarkdownBlock.from(block, file, normalizer)
);
return new MarkdownSection({
$file: file,
$id: MarkdownSection.readableId(file, raw.$title, raw.$ordinal),
$ordinal: raw.$ordinal,
$title: raw.$title,
$level: raw.$level,
$position: raw.$position,
$tags: raw.$tags,
$links: raw.$links.map((l) => normalizer(Link.fromObject(l))),
$blocks: blocks,
$infields: mapObjectValues(raw.$infields, (i) =>
normalizeLinks(valueInlineField(i), normalizer)
),
});
}
private constructor(init: Partial<MarkdownSection>) {
Object.assign(this, init);
}
/** Obtain the number of lines in the section. */
get $lineCount(): number {
return this.$position.end - this.$position.start;
}
/** Alias for title which allows searching over pages and sections by 'name'. */
get $name(): string {
return this.$title;
}
/** Return a link to this section. */
get $link(): Link {
return Link.header(this.$file, this.$title);
}
/** All of the indexed fields in this object. */
get fields(): Field[] {
return MarkdownSection.FIELD_DEF(this);
}
/** Fetch a specific field by key. */
public field(key: string): Field {
return MarkdownSection.FIELD_DEF(this, key)?.[0];
}
public value(key: string): Literal | undefined {
return this.field(key)?.value;
}
public json(): JsonMarkdownSection {
return {
$ordinal: this.$ordinal,
$title: this.$title,
$level: this.$level,
$position: this.$position,
$tags: this.$tags,
$links: this.$links.map((link) => link.toObject()),
$blocks: this.$blocks.map((block) => block.json()),
$infields: mapObjectValues(this.$infields, jsonInlineField),
};
}
private static FIELD_DEF: FieldExtractor<MarkdownSection> =
Extractors.merge(
Extractors.intrinsics(),
Extractors.inlineFields((f) => f.$infields)
);
/** Generate a readable ID for this section using the first 8 characters of the string and the ordinal. */
static readableId(file: string, title: string, ordinal: number): string {
const first8 = title
.substring(0, Math.min(title.length, 8))
.replace(/[^A-Za-z0-9-_]+/gi, "-");
return `${file}/section${ordinal}/${first8}`;
}
}
/** Base class for all markdown blocks. */
export class MarkdownBlock
implements Indexable, Linkbearing, Taggable, Fieldbearing
{
static TYPES = [
"markdown",
"block",
LINKBEARING_TYPE,
TAGGABLE_TYPE,
FIELDBEARING_TYPE,
];
$types: string[] = MarkdownBlock.TYPES;
$typename: string = "Block";
$id: string;
$file: string;
/** The index of this block in the file. */
$ordinal: number;
/** The position/extent of the block. */
$position: LineSpan;
/** All tags on the block. */
$tags: string[];
/** All links in the file. */
$links: Link[];
/** Map of all distinct inline fields in the document, from key name -> metadata. */
$infields: Record<string, InlineField>;
/** If present, the distinct block ID for this block. */
$blockId?: string;
/** The type of block - paragraph, list, and so on. */
$type: string;
static from(
object: JsonMarkdownBlock,
file: string,
normalizer: LinkNormalizer = NOOP_NORMALIZER
): MarkdownBlock {
if (object.$type === "list") {
return MarkdownListBlock.from(
object as JsonMarkdownListBlock,
file,
normalizer
);
} else if (object.$type === "datablock") {
return MarkdownDatablock.from(
object as JsonMarkdownDatablock,
file,
normalizer
);
} else if (object.$type === "codeblock") {
return MarkdownCodeblock.from(
object as JsonMarkdownCodeblock,
file,
normalizer
);
}
return new MarkdownBlock({
$file: file,
$id: MarkdownBlock.readableId(file, object.$ordinal),
$ordinal: object.$ordinal,
$position: object.$position,
$tags: object.$tags,
$links: object.$links.map((l) => normalizer(Link.fromObject(l))),
$infields: mapObjectValues(object.$infields, (i) =>
normalizeLinks(valueInlineField(i), normalizer)
),
$blockId: object.$blockId,
$type: object.$type,
});
}
protected constructor(init: Partial<MarkdownBlock>) {
Object.assign(this, init);
}
/** If this block has a block ID, the link to this block. */
get $link(): Link | undefined {
if (this.$blockId) return Link.block(this.$file, this.$blockId);
else return undefined;
}
/** All of the indexed fields in this object. */
get fields() {
return MarkdownBlock.FIELD_DEF(this);
}
/** Fetch a specific field by key. */
public field(key: string) {
return MarkdownBlock.FIELD_DEF(this, key)?.[0];
}
public value(key: string): Literal | undefined {
return this.field(key)?.value;
}
public json(): JsonMarkdownBlock {
return {
$ordinal: this.$ordinal,
$position: this.$position,
$tags: this.$tags,
$links: this.$links.map((l) => l.toObject()),
$infields: mapObjectValues(this.$infields, jsonInlineField),
$blockId: this.$blockId,
$type: this.$type,
};
}
static FIELD_DEF: FieldExtractor<MarkdownBlock> = Extractors.merge(
Extractors.intrinsics(),
Extractors.inlineFields((f) => f.$infields)
);
/** Generate a readable ID for this block using the ordinal of the block. */
static readableId(file: string, ordinal: number): string {
return `${file}/block${ordinal}`;
}
}
/** Special block for markdown lists (of either plain list entries or tasks). */
export class MarkdownListBlock
extends MarkdownBlock
implements Taggable, Linkbearing
{
static TYPES = [
"markdown",
"block",
"block-list",
TAGGABLE_TYPE,
LINKBEARING_TYPE,
];
$types: string[] = MarkdownListBlock.TYPES;
$typename: string = "List Block";
/** The list items inside of this block. */
$elements: MarkdownListItem[];
/** Create a list block from a serialized value. */
static from(
object: JsonMarkdownListBlock,
file: string,
normalizer: LinkNormalizer = NOOP_NORMALIZER
): MarkdownListBlock {
const elements = object.$elements.map((elem) =>
MarkdownListItem.from(elem, file, normalizer)
);
return new MarkdownListBlock({
// TODO: This is shared with other blocks, should probably be fixed.
$file: file,
$id: MarkdownBlock.readableId(file, object.$ordinal),
$ordinal: object.$ordinal,
$position: object.$position,
$tags: object.$tags,
$links: object.$links.map((l) => normalizer(Link.fromObject(l))),
$infields: mapObjectValues(object.$infields, (i) =>
normalizeLinks(valueInlineField(i), normalizer)
),
$blockId: object.$blockId,
$elements: elements,
$type: "list",
});
}
public json(): JsonMarkdownListBlock {
return Object.assign(super.json(), {
$elements: this.$elements.map((elem) => elem.json()),
}) as JsonMarkdownListBlock;
}
public constructor(init: Partial<MarkdownListBlock>) {
super(init);
}
}
/** A block containing markdown code. */
export class MarkdownCodeblock
extends MarkdownBlock
implements Indexable, Fieldbearing, Linkbearing
{
static TYPES = [
"markdown",
"block",
"codeblock",
TAGGABLE_TYPE,
LINKBEARING_TYPE,
FIELDBEARING_TYPE,
];
$types: string[] = MarkdownCodeblock.TYPES;
$languages: string[];
$contentPosition: { start: number; end: number };
$style: "fenced" | "indent";
public constructor(init: Partial<MarkdownCodeblock>) {
super(init);
}
static from(
object: JsonMarkdownCodeblock,
file: string,
normalizer: LinkNormalizer = NOOP_NORMALIZER
): MarkdownCodeblock {
return new MarkdownCodeblock({
$file: file,
$id: MarkdownCodeblock.readableId(file, object.$position.start),
$position: object.$position,
$ordinal: object.$ordinal,
$typename: "Codeblock",
$type: "codeblock",
$blockId: object.$blockId,
$languages: object.$languages,
$links: object.$links.map((link) =>
normalizer(Link.fromObject(link))
),
$tags: object.$tags,
$infields: mapObjectValues(object.$infields, valueInlineField),
$contentPosition: object.$contentPosition,
$style: object.$style,
});
}
/** All of the indexed fields in this object. */
get fields() {
return MarkdownCodeblock.SUB_FIELD_DEF(this);
}
/** Fetch a specific field by key. */
public field(key: string) {
return MarkdownCodeblock.SUB_FIELD_DEF(this, key)?.[0];
}
public value(key: string): Literal | undefined {
return this.field(key)?.value;
}
public json(): JsonMarkdownCodeblock {
return Object.assign(super.json(), {
$languages: this.$languages,
$contentPosition: this.$contentPosition,
$style: this.$style,
}) as JsonMarkdownCodeblock;
}
static readableId(file: string, line: number): string {
return `${file}/codeblock${line}`;
}
static SUB_FIELD_DEF: FieldExtractor<MarkdownCodeblock> =
Extractors.merge<MarkdownCodeblock>(MarkdownBlock.FIELD_DEF);
}
/** A data-annotated YAML codeblock. */
export class MarkdownDatablock
extends MarkdownBlock
implements Indexable, Fieldbearing, Linkbearing
{
static TYPES = [
"markdown",
"block",
"datablock",
TAGGABLE_TYPE,
LINKBEARING_TYPE,
FIELDBEARING_TYPE,
];
$types: string[] = MarkdownDatablock.TYPES;
$data: Record<string, FrontmatterEntry>;
public constructor(init: Partial<MarkdownDatablock>) {
super(init);
}
static from(
object: JsonMarkdownDatablock,
file: string,
normalizer: LinkNormalizer = NOOP_NORMALIZER
): MarkdownDatablock {
// Datablocks are based on what is essentially just frontmatter; we can apply
// the same normalization logic to them.
const normdata = normalizeLinks(
mapObjectValues(object.$data, valueFrontmatterEntry),
normalizer
);
const links = gatherLinks(normdata);
const tags = gatherTags(normdata);
return new MarkdownDatablock({
$file: file,
$id: MarkdownDatablock.readableId(file, object.$position.start),
$position: object.$position,
$infields: {},
$ordinal: object.$ordinal,
$data: normdata,
$links: links,
$typename: "Datablock",
$tags: tags,
$type: "datablock",
$blockId: object.$blockId,
});
}
/** All of the indexed fields in this object. */
get fields() {
return MarkdownDatablock.SUB_FIELD_DEF(this);
}
/** Fetch a specific field by key. */
public field(key: string) {
return MarkdownDatablock.SUB_FIELD_DEF(this, key)?.[0];
}
public value(key: string): Literal | undefined {
return this.field(key)?.value;
}
public json(): JsonMarkdownDatablock {
return Object.assign(super.json(), {
$data: mapObjectValues(this.$data, jsonFrontmatterEntry),
}) as JsonMarkdownDatablock;
}
static readableId(file: string, line: number): string {
return `${file}/datablock${line}`;
}
static SUB_FIELD_DEF: FieldExtractor<MarkdownDatablock> =
Extractors.merge<MarkdownDatablock>(
MarkdownBlock.FIELD_DEF,
Extractors.frontmatter((f) => f.$data)
);
}
/** A specific list item in a list. */
export class MarkdownListItem
implements Indexable, Linkbearing, Taggable, Fieldbearing
{
static TYPES = [
"markdown",
"list-item",
LINKBEARING_TYPE,
TAGGABLE_TYPE,
FIELDBEARING_TYPE,
];
$types: string[] = MarkdownListItem.TYPES;
$typename: string = "List Item";
$id: string;
$file: string;
/** The position of the list item in the file. */
$position: LineSpan;
/** Child elements of this list item. */
$elements: MarkdownListItem[];
/** The type of list item that this element is. */
$type: string;
/** Exact tags on this list item. */
$tags: string[];
/** Map of all distinct inline fields in the document, from key name -> metadata. */
$infields: Record<string, InlineField>;
/** All links in the file. */
$links: Link[];
/** The block ID of this list item if present. */
$blockId?: string;
/**
* The line number of the parent of this list item.
* If a positive number, then this list element is a child
* of the list element at that line.
*
* If a negative number, then this list element is a root element
* of a list starting at that line (negated). I.e., -7 means
* this is a root element of the list starting at line 7.
*/
$parentLine: number;
/** The marker used to start the list item (such as - or + or *). On a malformed task, may be undefined. */
$symbol?: string;
/** The text contents of the list item. */
$text?: string;
/** Create a list item from a serialized object. */
static from(
object: JsonMarkdownListItem,
file: string,
normalizer: LinkNormalizer = NOOP_NORMALIZER
): MarkdownListItem {
if (object.$type === "task")
return MarkdownTaskItem.from(
object as JsonMarkdownTaskItem,
file,
normalizer
);
const elements = object.$elements.map((elem) =>
MarkdownListItem.from(elem, file, normalizer)
);
return new MarkdownListItem({
$file: file,
$id: MarkdownListItem.readableId(file, object.$position.start),
$position: object.$position,
$elements: elements,
$type: object.$type,
$tags: object.$tags,
$infields: mapObjectValues(object.$infields, (i) =>
normalizeLinks(valueInlineField(i), normalizer)
),
$links: object.$links.map((l) => normalizer(Link.fromObject(l))),
$blockId: object.$blockId,
$parentLine: object.$parentLine,
$text: object.$text,
$symbol: object.$symbol,
});
}
protected constructor(init: Partial<MarkdownListItem>) {
Object.assign(this, init);
}
/** Get the line that this list item starts on. */
get $line(): number {
return this.$position.start;
}
/** The number of lines in this list item. */
get $lineCount(): number {
return this.$position.end - this.$position.start + 1;
}
/** Cleaned text that is garaunteed to be non-null and has indenation and inline fields removed. */
get $cleantext() {
if (!this.$text) return "";
return (
this.$text
// Eliminate [key:: value] annotations.
.replace(/(.*?)([\[\(][^:(\[]+::\s*.*?[\]\)]\s*)$/gm, "$1")
// Trim whitespace.
.trim()
);
}
/** All of the indexed fields in this object. */
get fields() {
return MarkdownListItem.FIELD_DEF(this);
}
/** Fetch a specific field by key. */
public field(key: string) {
return MarkdownListItem.FIELD_DEF(this, key)?.[0];
}
public value(key: string): Literal | undefined {
return this.field(key)?.value;
}
public json(): JsonMarkdownListItem {
return {
$position: this.$position,
$elements: this.$elements.map((elem) => elem.json()),
$type: this.$type,
$tags: this.$tags,
$infields: mapObjectValues(this.$infields, jsonInlineField),
$links: this.$links,
$blockId: this.$blockId,
$parentLine: this.$parentLine,
$symbol: this.$symbol,
$text: this.$text,
};
}
private static FIELD_DEF: FieldExtractor<MarkdownListItem> =
Extractors.merge(
Extractors.intrinsics(),
Extractors.inlineFields((f) => f.$infields)
);
/** Generate a readable ID for this item using the line number. */
static readableId(file: string, line: number): string {
return `${file}/list${line}`;
}
}
/** A specific task inside of a markdown list. */
export class MarkdownTaskItem
extends MarkdownListItem
implements Indexable, Linkbearing, Taggable, Fieldbearing
{
static TYPES = [
"markdown",
"list-item",
"task",
LINKBEARING_TYPE,
TAGGABLE_TYPE,
FIELDBEARING_TYPE,
];
$types: string[] = MarkdownTaskItem.TYPES;
$typename: string = "Task";
/** The text inside of the task item. */
$status: string;
public static from(
object: JsonMarkdownTaskItem,
file: string,
normalizer: LinkNormalizer
): MarkdownTaskItem {
const elements = object.$elements.map((elem) =>
MarkdownListItem.from(elem, file, normalizer)
);
return new MarkdownTaskItem({
$file: file,
$id: MarkdownListItem.readableId(file, object.$position.start),
$position: object.$position,
$elements: elements,
$type: object.$type,
$tags: object.$tags,
$infields: mapObjectValues(object.$infields, (i) =>
normalizeLinks(valueInlineField(i), normalizer)
),
$links: object.$links.map((l) => normalizer(Link.fromObject(l))),
$blockId: object.$blockId,
$parentLine: object.$parentLine,
$status: object.$status,
$symbol: object.$symbol,
$text: object.$text,
});
}
public constructor(init: Partial<MarkdownTaskItem>) {
super(init);
}
public json(): JsonMarkdownListItem {
return Object.assign(super.json(), {
$status: this.$status,
});
}
/** Determine if the given task is completed. */
public get $completed() {
return this.$status === "x" || this.$status === "X";
}
}
/** An entry in the frontmatter; includes the raw value, parsed value, and raw key (before lower-casing). */
export interface FrontmatterEntry {
/** The actual string in frontmatter with exact casing. */
key: string;
/** The parsed value of the frontmatter entry (date, duration, etc.). */
value: Literal;
/** The raw value of the frontmatter entry before parsing; generally a string or number. */
raw: string;
}
/** Convert a regular frontmatter entry into a JSON frontmatter entry.
* @hidden
*/
export function jsonFrontmatterEntry(
raw: FrontmatterEntry
): JsonFrontmatterEntry {
return {
key: raw.key,
value: JsonConversion.json(raw.value),
raw: raw.raw,
};
}
/** Convert a json frontmatter entry to a regular frontmatter entry.
* @hidden
*/
export function valueFrontmatterEntry(
raw: JsonFrontmatterEntry
): FrontmatterEntry {
return {
key: raw.key,
value: JsonConversion.value(raw.value),
raw: raw.raw,
};
}
/** Normalize links deeply in the object.
* @hidden
*/
export function normalizeLinks<T extends Literal>(
input: T,
normalizer: LinkNormalizer
): T {
return Literals.mapLeaves(input, (value) => {
if (Literals.isLink(value)) return normalizer(value);
else return value;
}) as T;
}
/** Recursively gather links from a literal object.
* @hidden
*/
export function gatherLinks(input: Literal): Link[] {
const result: Link[] = [];
Literals.mapLeaves(input, (value) => {
if (Literals.isLink(value)) result.push(value);
return null;
});
return result;
}
/** Gather tags from a datablock.
* @hidden
*/
export function gatherTags(data: Record<string, FrontmatterEntry>): string[] {
function recurse(input: any): string[] {
if (Literals.isString(input))
return [input.startsWith("#") ? input : "#" + input];
else if (Literals.isArray(input)) return input.flatMap(recurse);
else return [];
}
let tags: string[] = [];
if ("tag" in data) tags = tags.concat(recurse(data["tags"]));
if ("tags" in data) tags = tags.concat(recurse(data["tags"]));
return tags;
}
/** A link normalizer which takes in a raw link and produces a normalized link. */
export type LinkNormalizer = (link: Link) => Link;
export const NOOP_NORMALIZER: LinkNormalizer = (x) => x;

View file

@ -0,0 +1,337 @@
/**
* Web worker for background processing of task indexing
*/
import { FileStats } from "obsidian";
import { Task } from "../types/TaskIndex";
/** Command to parse task from a file */
export interface ParseTasksCommand {
type: "parseTasks";
filePath: string;
content: string;
stats: FileStats;
}
/** Command to batch index multiple files */
export interface BatchIndexCommand {
type: "batchIndex";
files: {
path: string;
content: string;
stats: FileStats;
}[];
}
/** Available commands that can be sent to the worker */
export type IndexerCommand = ParseTasksCommand | BatchIndexCommand;
/** Result of task parsing */
export interface TaskParseResult {
type: "parseResult";
filePath: string;
tasks: Task[];
stats: {
totalTasks: number;
completedTasks: number;
processingTimeMs: number;
};
}
/** Result of batch indexing */
export interface BatchIndexResult {
type: "batchResult";
results: {
filePath: string;
taskCount: number;
}[];
stats: {
totalFiles: number;
totalTasks: number;
processingTimeMs: number;
};
}
/** Result of a worker operation or error */
export type IndexerResult = TaskParseResult | BatchIndexResult | ErrorResult;
/** Error response */
export interface ErrorResult {
type: "error";
error: string;
filePath?: string;
}
/**
* Regular expressions for parsing task components
*/
const TASK_REGEX = /^([\s>]*- \[(.)\])\s*(.*)$/m;
const START_DATE_REGEX = /📅 (\d{4}-\d{2}-\d{2})/;
const COMPLETED_DATE_REGEX = /✅ (\d{4}-\d{2}-\d{2})/;
const DUE_DATE_REGEX = /⏳ (\d{4}-\d{2}-\d{2})/;
const SCHEDULED_DATE_REGEX = /⏰ (\d{4}-\d{2}-\d{2})/;
const RECURRENCE_REGEX = /🔁 (.*?)(?=\s|$)/;
const TAG_REGEX = /#[\w\/-]+/g;
const CONTEXT_REGEX = /@[\w-]+/g;
const PRIORITY_REGEX = /🔼|⏫|🔽|⏬️|🔺|\[#[A-C]\]/;
const PRIORITY_MAP: Record<string, number> = {
"⏫": 3, // High
"🔼": 2, // Medium
"🔽": 1, // Low
"⏬️": 1, // Lowest
"🔺": 5, // Highest
"[#A]": 4, // High (letter format)
"[#B]": 3, // Medium (letter format)
"[#C]": 2, // Low (letter format)
};
/**
* Parse tasks from file content
*/
function parseTasksFromContent(filePath: string, content: string): Task[] {
const lines = content.split(/\r?\n/);
const tasks: Task[] = [];
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
const taskMatch = line.match(TASK_REGEX);
if (taskMatch) {
const [, prefix, status, content] = taskMatch;
const completed = status.toLowerCase() === "x";
// Generate a deterministic ID based on file path and line number
// This helps with task tracking across worker calls
const id = `${filePath}-L${i}`;
// Basic task info
const task: Task = {
id,
content: content.trim(),
filePath,
line: i,
completed,
originalMarkdown: line,
tags: [],
children: [],
};
// Extract metadata
extractDates(task, content);
extractTags(task, content);
extractContext(task, content);
extractPriority(task, content);
tasks.push(task);
}
}
// Build parent-child relationships
buildTaskHierarchy(tasks);
return tasks;
}
/**
* Extract dates from task content
*/
function extractDates(task: Task, content: string): void {
// Start date
const startDateMatch = content.match(START_DATE_REGEX);
if (startDateMatch) {
task.startDate = new Date(startDateMatch[1]).getTime();
}
// Due date
const dueDateMatch = content.match(DUE_DATE_REGEX);
if (dueDateMatch) {
task.dueDate = new Date(dueDateMatch[1]).getTime();
}
// Scheduled date
const scheduledDateMatch = content.match(SCHEDULED_DATE_REGEX);
if (scheduledDateMatch) {
task.scheduledDate = new Date(scheduledDateMatch[1]).getTime();
}
// Completion date
const completedDateMatch = content.match(COMPLETED_DATE_REGEX);
if (completedDateMatch) {
task.completedDate = new Date(completedDateMatch[1]).getTime();
}
}
/**
* Extract tags from task content
*/
function extractTags(task: Task, content: string): void {
const tagMatches = content.match(TAG_REGEX) || [];
task.tags = tagMatches.map((tag) => tag.trim());
// Check for project tags
const projectTag = task.tags.find((tag) => tag.startsWith("#project/"));
if (projectTag) {
task.project = projectTag.substring("#project/".length);
}
}
/**
* Extract context from task content
*/
function extractContext(task: Task, content: string): void {
const contextMatches = content.match(CONTEXT_REGEX) || [];
if (contextMatches.length > 0) {
// Use the first context tag as the primary context
task.context = contextMatches[0]?.substring(1); // Remove the @ symbol
}
}
/**
* Extract priority from task content
*/
function extractPriority(task: Task, content: string): void {
const priorityMatch = content.match(PRIORITY_REGEX);
if (priorityMatch) {
task.priority = PRIORITY_MAP[priorityMatch[0]] || undefined;
}
}
/**
* Build parent-child relationships between tasks
*/
function buildTaskHierarchy(tasks: Task[]): void {
// Sort tasks by line number
tasks.sort((a, b) => a.line - b.line);
// Build parent-child relationships based on indentation
for (let i = 0; i < tasks.length; i++) {
const currentTask = tasks[i];
const currentIndent = getIndentLevel(currentTask.originalMarkdown);
// Look for potential parent tasks (must be before current task and have less indentation)
for (let j = i - 1; j >= 0; j--) {
const potentialParent = tasks[j];
const parentIndent = getIndentLevel(
potentialParent.originalMarkdown
);
if (parentIndent < currentIndent) {
// Found a parent
currentTask.parent = potentialParent.id;
potentialParent.children.push(currentTask.id);
break;
}
}
}
}
/**
* Get indentation level of a line
*/
function getIndentLevel(line: string): number {
const match = line.match(/^(\s*)/);
return match ? match[1].length : 0;
}
/**
* Process a single file
*/
function processFile(
filePath: string,
content: string,
stats: FileStats
): TaskParseResult {
const startTime = performance.now();
try {
const tasks = parseTasksFromContent(filePath, content);
const completedTasks = tasks.filter((t) => t.completed).length;
return {
type: "parseResult",
filePath,
tasks,
stats: {
totalTasks: tasks.length,
completedTasks,
processingTimeMs: Math.round(performance.now() - startTime),
},
};
} catch (error) {
console.error(`Error processing file ${filePath}:`, error);
throw error;
}
}
/**
* Process multiple files in batch
*/
function processBatch(
files: { path: string; content: string; stats: FileStats }[]
): BatchIndexResult {
const startTime = performance.now();
const results: { filePath: string; taskCount: number }[] = [];
let totalTasks = 0;
for (const file of files) {
try {
const parseResult = processFile(
file.path,
file.content,
file.stats
);
totalTasks += parseResult.stats.totalTasks;
results.push({
filePath: file.path,
taskCount: parseResult.stats.totalTasks,
});
} catch (error) {
console.error(
`Error in batch processing for file ${file.path}:`,
error
);
// Continue with other files even if one fails
}
}
return {
type: "batchResult",
results,
stats: {
totalFiles: files.length,
totalTasks,
processingTimeMs: Math.round(performance.now() - startTime),
},
};
}
/**
* Web worker message handler
*/
self.onmessage = async (event) => {
try {
const message = event.data as IndexerCommand;
if (message.type === "parseTasks") {
const result = processFile(
message.filePath,
message.content,
message.stats
);
self.postMessage(result);
} else if (message.type === "batchIndex") {
const result = processBatch(message.files);
self.postMessage(result);
} else {
self.postMessage({
type: "error",
error: `Unknown command type: ${(message as any).type}`,
} as ErrorResult);
}
} catch (error) {
self.postMessage({
type: "error",
error: error instanceof Error ? error.message : String(error),
} as ErrorResult);
}
};

View file

@ -0,0 +1,457 @@
/**
* Manager for task indexing web workers
*/
import { TFile, Vault } from "obsidian";
import { Task } from "../types/TaskIndex";
import {
BatchIndexCommand,
BatchIndexResult,
ErrorResult,
IndexerCommand,
IndexerResult,
ParseTasksCommand,
TaskParseResult,
} from "./TaskIndexWorker";
// Import worker
import TaskWorker from "./TaskIndexWorker";
/**
* Options for worker pool
*/
export interface WorkerPoolOptions {
/** Maximum number of workers to use */
maxWorkers: number;
/** Target CPU utilization (0.1 to 1.0) */
cpuUtilization: number;
/** Whether to enable debug logging */
debug?: boolean;
}
/**
* Default worker pool options
*/
export const DEFAULT_WORKER_OPTIONS: WorkerPoolOptions = {
maxWorkers: Math.max(1, (navigator.hardwareConcurrency || 2) - 1),
cpuUtilization: 0.75,
debug: false,
};
/**
* Task callback type
*/
export type TaskCallback<T> = (error: Error | null, result?: T) => void;
/**
* Task request interface
*/
interface TaskRequest {
/** Command to process */
command: IndexerCommand;
/** Callback to call with result */
callback: TaskCallback<IndexerResult>;
}
/**
* Worker data for pool
*/
interface WorkerData {
/** Worker instance */
worker: Worker;
/** Whether the worker is busy */
busy: boolean;
/** Last time this worker finished a task */
lastTaskFinished: number;
/** Number of tasks processed */
tasksProcessed: number;
/** Total processing time */
totalProcessingTime: number;
}
/**
* Worker pool for task processing
*/
export class TaskWorkerManager {
/** Worker pool */
private workers: WorkerData[] = [];
/** Queue of pending tasks */
private taskQueue: TaskRequest[] = [];
/** Whether the pool is currently active */
private active: boolean = true;
/** Options for the worker pool */
private options: WorkerPoolOptions;
/** Vault instance */
private vault: Vault;
/**
* Create a new worker pool
*/
constructor(vault: Vault, options: Partial<WorkerPoolOptions> = {}) {
this.options = { ...DEFAULT_WORKER_OPTIONS, ...options };
this.vault = vault;
// Initialize workers up to max
this.initializeWorkers();
}
/**
* Initialize workers in the pool
*/
private initializeWorkers(): void {
const workerCount = Math.min(
this.options.maxWorkers,
navigator.hardwareConcurrency || 2
);
for (let i = 0; i < workerCount; i++) {
try {
const worker = new Worker(
URL.createObjectURL(
new Blob([`importScripts('${TaskWorker}')`], {
type: "application/javascript",
})
)
);
// Setup worker message handler
worker.onmessage = (event) => {
this.handleWorkerMessage(worker, event.data);
};
worker.onerror = (event) => {
console.error("Worker error:", event);
};
this.workers.push({
worker,
busy: false,
lastTaskFinished: Date.now(),
tasksProcessed: 0,
totalProcessingTime: 0,
});
this.log(`Initialized worker #${i + 1}`);
} catch (error) {
console.error("Failed to initialize worker:", error);
}
}
this.log(
`Initialized ${this.workers.length} workers (requested ${workerCount})`
);
// Check if we have any workers
if (this.workers.length === 0) {
console.warn(
"No workers could be initialized, falling back to main thread processing"
);
}
}
/**
* Process a single file for tasks
*/
public processFile(file: TFile, callback: TaskCallback<Task[]>): void {
this.vault
.cachedRead(file)
.then((content) => {
// Create file stats (adapting to the worker's needs)
const stats = {
ctime: file.stat.ctime,
mtime: file.stat.mtime,
size: file.stat.size,
};
const command: ParseTasksCommand = {
type: "parseTasks",
filePath: file.path,
content,
stats,
};
this.queueTask(command, (error, result) => {
if (error) {
callback(error);
return;
}
if (!result || result.type === "error") {
callback(
new Error(
(result as ErrorResult)?.error ||
"Unknown error"
)
);
return;
}
if (result.type === "parseResult") {
const parseResult = result as TaskParseResult;
callback(null, parseResult.tasks);
} else {
callback(
new Error(`Unexpected result type: ${result.type}`)
);
}
});
})
.catch((error) => {
callback(error);
});
}
/**
* Process multiple files in a batch
*/
public processBatch(
files: TFile[],
callback: TaskCallback<Map<string, Task[]>>
): void {
// Read all files first
Promise.all(
files.map((file) =>
this.vault.cachedRead(file).then((content) => ({
file,
content,
}))
)
)
.then((fileContents) => {
const command: BatchIndexCommand = {
type: "batchIndex",
files: fileContents.map(({ file, content }) => ({
path: file.path,
content,
stats: {
ctime: file.stat.ctime,
mtime: file.stat.mtime,
size: file.stat.size,
},
})),
};
this.queueTask(command, (error, result) => {
if (error) {
callback(error);
return;
}
if (!result || result.type === "error") {
callback(
new Error(
(result as ErrorResult)?.error ||
"Unknown error"
)
);
return;
}
if (result.type === "batchResult") {
const batchResult = result as BatchIndexResult;
// This is just a summary, not the actual tasks
callback(
new Error("Batch result does not include tasks")
);
} else if (result.type === "parseResult") {
// Single file result, convert to map
const parseResult = result as TaskParseResult;
const resultMap = new Map<string, Task[]>();
resultMap.set(parseResult.filePath, parseResult.tasks);
callback(null, resultMap);
} else {
callback(
new Error(`Unexpected result type: ${result.type}`)
);
}
});
})
.catch((error) => {
callback(error);
});
}
/**
* Queue a task for processing by a worker
*/
private queueTask(
command: IndexerCommand,
callback: TaskCallback<IndexerResult>
): void {
this.taskQueue.push({
command,
callback,
});
this.processQueue();
}
/**
* Process the task queue
*/
private processQueue(): void {
if (!this.active || this.taskQueue.length === 0) {
return;
}
// Try to find an available worker
const availableWorker = this.getAvailableWorker();
if (!availableWorker) {
// No available workers, will retry when a worker becomes available
return;
}
// Get the next task
const task = this.taskQueue.shift();
if (!task) {
return;
}
// Mark the worker as busy
availableWorker.busy = true;
try {
// Send the task to the worker
availableWorker.worker.postMessage(task.command);
// Store the callback with the worker so we can call it when the worker responds
(availableWorker as any).currentTask = task;
(availableWorker as any).taskStartTime = Date.now();
this.log(`Sent task to worker: ${task.command.type}`);
} catch (error) {
// Failed to send task to worker
console.error("Failed to send task to worker:", error);
// Mark the worker as available again
availableWorker.busy = false;
// Call the callback with the error
task.callback(error as Error);
// Try to process the next task
this.processQueue();
}
}
/**
* Handle a message from a worker
*/
private handleWorkerMessage(worker: Worker, data: IndexerResult): void {
// Find the worker data
const workerData = this.workers.find((w) => w.worker === worker);
if (!workerData) {
console.error("Received message from unknown worker:", data);
return;
}
// Get the task that was being processed
const task = (workerData as any).currentTask as TaskRequest | undefined;
const taskStartTime = (workerData as any).taskStartTime as
| number
| undefined;
// Update worker stats
workerData.busy = false;
workerData.lastTaskFinished = Date.now();
workerData.tasksProcessed++;
if (taskStartTime) {
const processingTime = Date.now() - taskStartTime;
workerData.totalProcessingTime += processingTime;
// Apply throttling based on CPU utilization
const delay = Math.round(
(processingTime * (1 - this.options.cpuUtilization)) /
this.options.cpuUtilization
);
if (delay > 0) {
// Delay before this worker processes another task
setTimeout(() => {
// Process next task if there are any
this.processQueue();
}, delay);
this.log(
`Worker throttled for ${delay}ms (processed in ${processingTime}ms)`
);
return;
}
}
// Clear the current task
(workerData as any).currentTask = undefined;
(workerData as any).taskStartTime = undefined;
// Call the callback with the result
if (task) {
if (data.type === "error") {
task.callback(new Error((data as ErrorResult).error));
} else {
task.callback(null, data);
}
} else {
console.error(
"Received message from worker with no associated task:",
data
);
}
// Process next task if there are any
this.processQueue();
}
/**
* Get an available worker
*/
private getAvailableWorker(): WorkerData | undefined {
// First, look for a non-busy worker
for (const worker of this.workers) {
if (!worker.busy) {
return worker;
}
}
return undefined;
}
/**
* Shutdown the worker pool
*/
public shutdown(): void {
this.active = false;
// Terminate all workers
for (const worker of this.workers) {
worker.worker.terminate();
}
// Clear the workers array
this.workers = [];
// Clear the task queue and call all callbacks with an error
for (const task of this.taskQueue) {
task.callback(new Error("Worker pool shut down"));
}
this.taskQueue = [];
this.log("Worker pool shut down");
}
/**
* Get the number of pending tasks
*/
public getPendingTaskCount(): number {
return this.taskQueue.length;
}
/**
* Log a message if debugging is enabled
*/
private log(message: string): void {
if (this.options.debug) {
console.log(`[TaskWorkerManager] ${message}`);
}
}
}

View file

@ -0,0 +1,22 @@
/** A promise that can be resolved directly. */
export type Deferred<T> = Promise<T> & {
resolve: (value: T) => void;
reject: (error: any) => void;
};
/** Create a new deferred object, which is a resolvable promise. */
export function deferred<T>(): Deferred<T> {
let resolve: (value: T) => void;
let reject: (error: any) => void;
const promise = new Promise((res, rej) => {
resolve = res;
reject = rej;
});
const deferred = promise as any as Deferred<T>;
deferred.resolve = resolve!;
deferred.reject = reject!;
return deferred;
}

View file

@ -0,0 +1,234 @@
/** Controls and creates Dataview file importers, allowing for asynchronous loading and parsing of files. */
import ImportWorker from "index/web-worker/importer.worker";
import { Component, FileManager, MetadataCache, TFile, Vault } from "obsidian";
import { CanvasImport, MarkdownImport } from "index/web-worker/message";
import { Deferred, deferred } from "utils/deferred";
import { Queue } from "@datastructures-js/queue";
/** Settings for throttling import. */
export interface ImportThrottle {
/** The number of workers to use for imports. */
workers: number;
/** A number between 0.1 and 1 which indicates total cpu utilization target; 0.1 means spend 10% of time */
utilization: number;
}
/** Default throttle configuration. */
export const DEFAULT_THROTTLE: ImportThrottle = {
workers: 2,
utilization: 0.75,
};
/** Multi-threaded file parser which debounces rapid file requests automatically. */
export class FileImporter extends Component {
/* Background workers which do the actual file parsing. */
workers: Map<number, PoolWorker>;
/** The next worker ID to hand out. */
nextWorkerId: number;
/** If true, the importer is now inactive and will not process further files. */
shutdown: boolean;
/** List of files which have been queued for a reload. */
queue: Queue<[TFile, Deferred<any>]>;
/** Outstanding loads indexed by path. */
outstanding: Map<string, Promise<any>>;
/** Throttle settings. */
throttle: () => ImportThrottle;
public constructor(
public vault: Vault,
public fileManager: FileManager,
public metadataCache: MetadataCache,
throttle?: () => ImportThrottle
) {
super();
this.workers = new Map();
this.shutdown = false;
this.nextWorkerId = 0;
this.throttle = throttle ?? (() => DEFAULT_THROTTLE);
this.queue = new Queue();
this.outstanding = new Map();
}
/**
* Queue the given file for importing. Multiple import requests for the same file in a short time period will be de-bounced
* and all be resolved by a single actual file reload.
*/
public import<T>(file: TFile): Promise<T> {
// De-bounce repeated requests for the same file.
let existing = this.outstanding.get(file.path);
if (existing) return existing;
let promise = deferred<T>();
this.outstanding.set(file.path, promise);
this.queue.enqueue([file, promise]);
this.schedule();
return promise;
}
/** Reset any active throttles on the importer (such as if the utilization changes). */
public unthrottle() {
for (let worker of this.workers.values()) {
worker.availableAt = Date.now();
}
}
/** Poll from the queue and execute if there is an available worker. */
private async schedule() {
if (this.queue.size() == 0 || this.shutdown) return;
const worker = this.availableWorker();
if (!worker) return;
const [file, promise] = this.queue.dequeue()!;
worker.active = [file, promise, Date.now()];
try {
switch (file.extension) {
case "markdown":
case "md": {
const contents = await this.vault.cachedRead(file);
worker!.worker.postMessage({
type: "markdown",
path: file.path,
contents: contents,
stat: file.stat,
metadata: this.metadataCache.getFileCache(file),
} as MarkdownImport);
break;
}
case "canvas": {
const contents = await this.vault.cachedRead(file);
worker!.worker.postMessage({
type: "canvas",
path: file.path,
contents: contents,
stat: file.stat,
index: this.fileManager.linkUpdaters.canvas.canvas.index
.index[file.path],
} as CanvasImport);
break;
}
}
} catch (ex) {
console.log("Datacore: Background file reloading failed. " + ex);
// Message failed, release this worker.
worker.active = undefined;
}
}
/** Finish the parsing of a file, potentially queueing a new file. */
private finish(worker: PoolWorker, data: any) {
if (!worker.active) {
console.log(
"Datacore: Received a stale worker message. Ignoring.",
data
);
return;
}
const [file, promise, start] = worker.active!;
// Resolve promises to let users know this file has finished.
if ("$error" in data) promise.reject(data["$error"]);
else promise.resolve(data);
// Remove file from outstanding.
this.outstanding.delete(file.path);
// Remove this worker if we are over capacity.
// Otherwise, notify the queue this file is available for new work.
if (this.workers.size > this.throttle().workers) {
this.workers.delete(worker.id);
terminate(worker);
} else {
const now = Date.now();
const throttle = Math.max(0.1, this.throttle().utilization) - 1.0;
const delay = (now - start) * throttle;
worker.active = undefined;
if (delay <= 1e-10) {
worker.availableAt = now;
this.schedule();
} else {
worker.availableAt = now + delay;
// Note: I'm pretty sure this will garauntee that this executes AFTER delay milliseconds,
// so this should be fine; if it's not, we'll have to swap to an external timeout loop
// which infinitely reschedules itself to the next available execution time.
setTimeout(this.schedule.bind(this), delay);
}
}
}
/** Obtain an available worker, returning undefined if one does not exist. */
private availableWorker(): PoolWorker | undefined {
const now = Date.now();
for (let worker of this.workers.values()) {
if (!worker.active && worker.availableAt <= now) {
return worker;
}
}
// Make a new worker if we can.
if (this.workers.size < this.throttle().workers) {
let worker = this.newWorker();
this.workers.set(worker.id, worker);
return worker;
}
return undefined;
}
/** Create a new worker bound to this importer. */
private newWorker(): PoolWorker {
let worker: PoolWorker = {
id: this.nextWorkerId++,
availableAt: Date.now(),
worker: new ImportWorker(),
};
worker.worker.onmessage = (evt) => this.finish(worker, evt.data);
return worker;
}
/** Reject all outstanding promises and close all workers on close. */
public onunload(): void {
for (let worker of this.workers.values()) {
terminate(worker);
}
while (!this.queue.isEmpty()) {
const [_file, promise] = this.queue.pop()!;
promise.reject("Terminated");
}
this.shutdown = true;
}
}
/** A worker in the pool of executing workers. */
interface PoolWorker {
/** The id of this worker. */
id: number;
/** The raw underlying worker. */
worker: Worker;
/** UNIX time indicating the next time this worker is available for execution according to target utilization. */
availableAt: number;
/** The active promise this worker is working on, if any. */
active?: [TFile, Deferred<any>, number];
}
/** Terminate a pool worker. */
function terminate(worker: PoolWorker) {
worker.worker.terminate();
if (worker.active) worker.active[1].reject("Terminated");
worker.active = undefined;
}

View file

@ -0,0 +1,47 @@
import { canvasImport } from "index/import/canvas";
import { markdownImport } from "index/import/markdown";
import {
CanvasImportResult,
ImportCommand,
MarkdownImportResult,
} from "index/web-worker/message";
/** Web worker entry point for importing. */
onmessage = async (event) => {
try {
const message = event.data as ImportCommand;
if (message.type === "markdown") {
const markdown = markdownImport(
message.path,
message.contents,
message.metadata,
message.stat
);
postMessage({
type: "markdown",
result: markdown,
} as MarkdownImportResult);
} else if (message.type === "canvas") {
const canvas = canvasImport(
message.path,
message.contents,
message.index,
message.stat
);
postMessage({
type: "canvas",
result: canvas,
} as CanvasImportResult);
} else {
postMessage({ $error: "Unsupported import method." });
}
} catch (error) {
console.error(
`Datacore Indexer failed to index ${event.data.path}: ${error}`
);
postMessage({ $error: error.message });
}
};

View file

@ -0,0 +1,58 @@
import { CanvasMetadataIndex, JsonCanvas } from "index/types/json/canvas";
import { JsonMarkdownPage } from "index/types/json/markdown";
import { CachedMetadata, FileStats } from "obsidian";
/** A command to import a markdown file. */
export interface MarkdownImport {
type: "markdown";
/** The path we are importing. */
path: string;
/** The file contents to import. */
contents: string;
/** The stats for the file. */
stat: FileStats;
/** Metadata for the file. */
metadata: CachedMetadata;
}
/** A command to import a canvas file. */
export interface CanvasImport {
type: "canvas";
/** The path we are importing. */
path: string;
/** The raw JSON contents we are importing. */
contents: string;
/** The stats for the file. */
stat: FileStats;
/** the canvas's metadata cache */
index: CanvasMetadataIndex["any"];
}
/** Available import commands to be sent to an import web worker. */
export type ImportCommand = MarkdownImport | CanvasImport;
/** The result of importing a file of some variety. */
export interface MarkdownImportResult {
/** The type of import. */
type: "markdown";
/** The result of importing. */
result: JsonMarkdownPage;
}
export interface CanvasImportResult {
type: "canvas";
result: JsonCanvas;
}
export interface ImportFailure {
/** Failed to import. */
type: "error";
/** The error that the worker indicated on failure. */
$error: string;
}
export type ImportResult =
| MarkdownImportResult
| CanvasImportResult
| ImportFailure;

View file

View file

@ -39,5 +39,6 @@
"6.1.0": "0.15.2",
"6.2.0": "0.15.2",
"6.2.1": "0.15.2",
"6.2.2": "0.15.2"
"6.2.2": "0.15.2",
"7.0.0": "0.15.2"
}