mirror of
https://github.com/taskgenius/taskgenius-plugin.git
synced 2026-07-22 06:40:25 +00:00
feat: support full task index
This commit is contained in:
parent
2f656e7774
commit
895a46e29f
21 changed files with 4265 additions and 3 deletions
|
|
@ -68,6 +68,7 @@ interface Task {
|
|||
class TaskParser {
|
||||
// Regular expressions for Tasks plugin syntax
|
||||
private readonly startDateRegex = /📅 (\d{4}-\d{2}-\d{2})/;
|
||||
private readonly completedDateRegex = /✅ (\d{4}-\d{2}-\d{2})/;
|
||||
private readonly dueDateRegex = /⏳ (\d{4}-\d{2}-\d{2})/;
|
||||
private readonly scheduledDateRegex = /⏰ (\d{4}-\d{2}-\d{2})/;
|
||||
private readonly recurrenceRegex = /🔁 (.*?)(?=\s|$)/;
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ esbuild
|
|||
},
|
||||
minify: prod ? true : false,
|
||||
entryPoints: ["src/index.ts"],
|
||||
plugins: [inlineWorkerPlugin({ workerName: "Datacore Indexer" })],
|
||||
bundle: true,
|
||||
external: [
|
||||
"obsidian",
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"id": "obsidian-task-progress-bar",
|
||||
"name": "Task Genius",
|
||||
"version": "6.2.2",
|
||||
"version": "7.0.0",
|
||||
"minAppVersion": "0.15.2",
|
||||
"description": "Comprehensive task management that includes progress bars, task status cycling, and advanced task tracking features.",
|
||||
"author": "Boninall",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "obsidian-task-progress-bar",
|
||||
"version": "6.2.2",
|
||||
"version": "7.0.0",
|
||||
"description": "Comprehensive task management plugin for Obsidian with progress bars, task status cycling, and advanced task tracking features.",
|
||||
"main": "main.js",
|
||||
"scripts": {
|
||||
|
|
@ -42,11 +42,15 @@
|
|||
"codemirror": "^6.0.0",
|
||||
"cross-env": "^7.0.3",
|
||||
"esbuild": "0.13.12",
|
||||
"esbuild-plugin-inline-worker": "https://github.com/mitschabaude/esbuild-plugin-inline-worker",
|
||||
"monkey-around": "^3.0.0",
|
||||
"obsidian": "^1.8.7",
|
||||
"obsidian-typings": "^2.39.0",
|
||||
"regexp-match-indices": "^1.0.2",
|
||||
"tslib": "2.4.0",
|
||||
"typescript": "4.7.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"uuid": "^11.1.0"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,10 @@ settings:
|
|||
importers:
|
||||
|
||||
.:
|
||||
dependencies:
|
||||
uuid:
|
||||
specifier: ^11.1.0
|
||||
version: 11.1.0
|
||||
devDependencies:
|
||||
'@codemirror/autocomplete':
|
||||
specifier: ^6.0.0
|
||||
|
|
@ -59,6 +63,9 @@ importers:
|
|||
esbuild:
|
||||
specifier: 0.13.12
|
||||
version: 0.13.12
|
||||
esbuild-plugin-inline-worker:
|
||||
specifier: https://github.com/mitschabaude/esbuild-plugin-inline-worker
|
||||
version: https://codeload.github.com/mitschabaude/esbuild-plugin-inline-worker/tar.gz/d1aaffc721a62a3fe33f59f8f69b462c7dd05f45(esbuild@0.13.12)
|
||||
monkey-around:
|
||||
specifier: ^3.0.0
|
||||
version: 3.0.0
|
||||
|
|
@ -623,6 +630,9 @@ packages:
|
|||
colord@2.9.3:
|
||||
resolution: {integrity: sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw==}
|
||||
|
||||
commondir@1.0.1:
|
||||
resolution: {integrity: sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==}
|
||||
|
||||
concat-map@0.0.1:
|
||||
resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==}
|
||||
|
||||
|
|
@ -776,6 +786,12 @@ packages:
|
|||
cpu: [x64]
|
||||
os: [openbsd]
|
||||
|
||||
esbuild-plugin-inline-worker@https://codeload.github.com/mitschabaude/esbuild-plugin-inline-worker/tar.gz/d1aaffc721a62a3fe33f59f8f69b462c7dd05f45:
|
||||
resolution: {tarball: https://codeload.github.com/mitschabaude/esbuild-plugin-inline-worker/tar.gz/d1aaffc721a62a3fe33f59f8f69b462c7dd05f45}
|
||||
version: 0.1.1
|
||||
peerDependencies:
|
||||
esbuild: latest
|
||||
|
||||
esbuild-sunos-64@0.13.12:
|
||||
resolution: {integrity: sha512-jBsF+e0woK3miKI8ufGWKG3o3rY9DpHvCVRn5eburMIIE+2c+y3IZ1srsthKyKI6kkXLvV4Cf/E7w56kLipMXw==}
|
||||
cpu: [x64]
|
||||
|
|
@ -881,6 +897,14 @@ packages:
|
|||
resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==}
|
||||
engines: {node: '>=8'}
|
||||
|
||||
find-cache-dir@3.3.2:
|
||||
resolution: {integrity: sha512-wXZV5emFEjrridIgED11OoUKLxiYjAcqot/NJdAkOhlJ+vGzwhOAfcG5OX1jP+S0PcjEn8bdMJv+g2jwQ3Onig==}
|
||||
engines: {node: '>=8'}
|
||||
|
||||
find-up@4.1.0:
|
||||
resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==}
|
||||
engines: {node: '>=8'}
|
||||
|
||||
find-up@5.0.0:
|
||||
resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==}
|
||||
engines: {node: '>=10'}
|
||||
|
|
@ -1048,6 +1072,10 @@ packages:
|
|||
resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==}
|
||||
engines: {node: '>= 0.8.0'}
|
||||
|
||||
locate-path@5.0.0:
|
||||
resolution: {integrity: sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==}
|
||||
engines: {node: '>=8'}
|
||||
|
||||
locate-path@6.0.0:
|
||||
resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==}
|
||||
engines: {node: '>=10'}
|
||||
|
|
@ -1059,6 +1087,10 @@ packages:
|
|||
resolution: {integrity: sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==}
|
||||
engines: {node: '>=8'}
|
||||
|
||||
make-dir@3.1.0:
|
||||
resolution: {integrity: sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==}
|
||||
engines: {node: '>=8'}
|
||||
|
||||
matcher@3.0.0:
|
||||
resolution: {integrity: sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==}
|
||||
engines: {node: '>=10'}
|
||||
|
|
@ -1149,14 +1181,26 @@ packages:
|
|||
resolution: {integrity: sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg==}
|
||||
engines: {node: '>=8'}
|
||||
|
||||
p-limit@2.3.0:
|
||||
resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==}
|
||||
engines: {node: '>=6'}
|
||||
|
||||
p-limit@3.1.0:
|
||||
resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==}
|
||||
engines: {node: '>=10'}
|
||||
|
||||
p-locate@4.1.0:
|
||||
resolution: {integrity: sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==}
|
||||
engines: {node: '>=8'}
|
||||
|
||||
p-locate@5.0.0:
|
||||
resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==}
|
||||
engines: {node: '>=10'}
|
||||
|
||||
p-try@2.2.0:
|
||||
resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==}
|
||||
engines: {node: '>=6'}
|
||||
|
||||
parent-module@1.0.1:
|
||||
resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==}
|
||||
engines: {node: '>=6'}
|
||||
|
|
@ -1187,6 +1231,10 @@ packages:
|
|||
pixi.js@7.2.4:
|
||||
resolution: {integrity: sha512-nBH60meoLnHxoMFz17HoMxXS4uJpG5jwIdL+Gx2S11TzWgP3iKF+/WLOTrkSdyuQoQSdIBxVqpnYii0Wiox15A==}
|
||||
|
||||
pkg-dir@4.2.0:
|
||||
resolution: {integrity: sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==}
|
||||
engines: {node: '>=8'}
|
||||
|
||||
prelude-ls@1.2.1:
|
||||
resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==}
|
||||
engines: {node: '>= 0.8.0'}
|
||||
|
|
@ -1368,6 +1416,10 @@ packages:
|
|||
resolution: {integrity: sha512-oCwdVC7mTuWiPyjLUz/COz5TLk6wgp0RCsN+wHZ2Ekneac9w8uuV0njcbbie2ME+Vs+d6duwmYuR3HgQXs1fOg==}
|
||||
engines: {node: '>= 0.4'}
|
||||
|
||||
uuid@11.1.0:
|
||||
resolution: {integrity: sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==}
|
||||
hasBin: true
|
||||
|
||||
w3c-keyname@2.2.8:
|
||||
resolution: {integrity: sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==}
|
||||
|
||||
|
|
@ -2041,6 +2093,8 @@ snapshots:
|
|||
|
||||
colord@2.9.3: {}
|
||||
|
||||
commondir@1.0.1: {}
|
||||
|
||||
concat-map@0.0.1: {}
|
||||
|
||||
crelt@1.0.6: {}
|
||||
|
|
@ -2164,6 +2218,11 @@ snapshots:
|
|||
esbuild-openbsd-64@0.13.12:
|
||||
optional: true
|
||||
|
||||
esbuild-plugin-inline-worker@https://codeload.github.com/mitschabaude/esbuild-plugin-inline-worker/tar.gz/d1aaffc721a62a3fe33f59f8f69b462c7dd05f45(esbuild@0.13.12):
|
||||
dependencies:
|
||||
esbuild: 0.13.12
|
||||
find-cache-dir: 3.3.2
|
||||
|
||||
esbuild-sunos-64@0.13.12:
|
||||
optional: true
|
||||
|
||||
|
|
@ -2315,6 +2374,17 @@ snapshots:
|
|||
dependencies:
|
||||
to-regex-range: 5.0.1
|
||||
|
||||
find-cache-dir@3.3.2:
|
||||
dependencies:
|
||||
commondir: 1.0.1
|
||||
make-dir: 3.1.0
|
||||
pkg-dir: 4.2.0
|
||||
|
||||
find-up@4.1.0:
|
||||
dependencies:
|
||||
locate-path: 5.0.0
|
||||
path-exists: 4.0.0
|
||||
|
||||
find-up@5.0.0:
|
||||
dependencies:
|
||||
locate-path: 6.0.0
|
||||
|
|
@ -2506,6 +2576,10 @@ snapshots:
|
|||
prelude-ls: 1.2.1
|
||||
type-check: 0.4.0
|
||||
|
||||
locate-path@5.0.0:
|
||||
dependencies:
|
||||
p-locate: 4.1.0
|
||||
|
||||
locate-path@6.0.0:
|
||||
dependencies:
|
||||
p-locate: 5.0.0
|
||||
|
|
@ -2514,6 +2588,10 @@ snapshots:
|
|||
|
||||
lowercase-keys@2.0.0: {}
|
||||
|
||||
make-dir@3.1.0:
|
||||
dependencies:
|
||||
semver: 6.3.1
|
||||
|
||||
matcher@3.0.0:
|
||||
dependencies:
|
||||
escape-string-regexp: 4.0.0
|
||||
|
|
@ -2593,14 +2671,24 @@ snapshots:
|
|||
|
||||
p-cancelable@2.1.1: {}
|
||||
|
||||
p-limit@2.3.0:
|
||||
dependencies:
|
||||
p-try: 2.2.0
|
||||
|
||||
p-limit@3.1.0:
|
||||
dependencies:
|
||||
yocto-queue: 0.1.0
|
||||
|
||||
p-locate@4.1.0:
|
||||
dependencies:
|
||||
p-limit: 2.3.0
|
||||
|
||||
p-locate@5.0.0:
|
||||
dependencies:
|
||||
p-limit: 3.1.0
|
||||
|
||||
p-try@2.2.0: {}
|
||||
|
||||
parent-module@1.0.1:
|
||||
dependencies:
|
||||
callsites: 3.1.0
|
||||
|
|
@ -2652,6 +2740,10 @@ snapshots:
|
|||
transitivePeerDependencies:
|
||||
- '@pixi/utils'
|
||||
|
||||
pkg-dir@4.2.0:
|
||||
dependencies:
|
||||
find-up: 4.1.0
|
||||
|
||||
prelude-ls@1.2.1: {}
|
||||
|
||||
progress@2.0.3: {}
|
||||
|
|
@ -2817,6 +2909,8 @@ snapshots:
|
|||
punycode: 1.4.1
|
||||
qs: 6.14.0
|
||||
|
||||
uuid@11.1.0: {}
|
||||
|
||||
w3c-keyname@2.2.8: {}
|
||||
|
||||
which@2.0.2:
|
||||
|
|
|
|||
0
src/pages/TaskView.ts
Normal file
0
src/pages/TaskView.ts
Normal file
40
src/utils/TODO.md
Normal file
40
src/utils/TODO.md
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
# Refactoring TODO List
|
||||
|
||||
## Task Index Refactoring
|
||||
|
||||
### High Priority Tasks
|
||||
- [ ] Remove non-task related indexing from importer.ts
|
||||
- [ ] Optimize task data indexing for high performance
|
||||
- [ ] Refactor worker implementation to focus only on task-related data
|
||||
- [ ] Implement incremental updates for task data
|
||||
- [ ] Redesign data structures for task metadata based on task-view.md
|
||||
|
||||
### Data Structure Optimization
|
||||
- [ ] Simplify task cache structure to include only task-relevant fields
|
||||
- [ ] Design efficient indexing for task filtering and querying
|
||||
- [ ] Implement optimized data structures for tag and date indexing
|
||||
- [ ] Reduce memory footprint of task index
|
||||
|
||||
### Worker Optimization
|
||||
- [ ] Refactor web worker implementation for better performance
|
||||
- [ ] Implement batch processing of task updates
|
||||
- [ ] Add support for priority-based task processing
|
||||
- [ ] Improve throttling mechanism for background processing
|
||||
|
||||
### Future Improvements
|
||||
- [ ] Design a metadata import strategy compatible with task-view.md
|
||||
- [ ] Implement LRU cache for frequently accessed tasks
|
||||
- [ ] Add support for efficient task dependency tracking
|
||||
- [ ] Create API for external plugin integration
|
||||
|
||||
## Architecture Improvements
|
||||
- [ ] Separate task parsing from task indexing logic
|
||||
- [ ] Create dedicated service for task data management
|
||||
- [ ] Implement event-based system for task updates
|
||||
- [ ] Add telemetry for performance monitoring (opt-in)
|
||||
|
||||
## Technical Debt
|
||||
- [ ] Fix type errors in current implementation
|
||||
- [ ] Add comprehensive unit tests for task indexing
|
||||
- [ ] Document all public APIs and interfaces
|
||||
- [ ] Create migration path for future index structure changes
|
||||
904
src/utils/import/TaskIndexer.ts
Normal file
904
src/utils/import/TaskIndexer.ts
Normal file
|
|
@ -0,0 +1,904 @@
|
|||
/**
|
||||
* High-performance task indexer implementation
|
||||
*/
|
||||
|
||||
import {
|
||||
App,
|
||||
Component,
|
||||
FileStats,
|
||||
MetadataCache,
|
||||
TFile,
|
||||
Vault,
|
||||
} from "obsidian";
|
||||
import {
|
||||
SortingCriteria,
|
||||
Task,
|
||||
TaskCache,
|
||||
TaskFilter,
|
||||
TaskIndexer as TaskIndexerInterface,
|
||||
TaskParserConfig,
|
||||
} from "../types/TaskIndex";
|
||||
import { TaskParser } from "./TaskParser";
|
||||
|
||||
/**
|
||||
* Utility to format a date for index keys (YYYY-MM-DD)
|
||||
*/
|
||||
function formatDateForIndex(date: number): string {
|
||||
const d = new Date(date);
|
||||
return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(
|
||||
2,
|
||||
"0"
|
||||
)}-${String(d.getDate()).padStart(2, "0")}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Implementation of the task indexer that focuses only on task-related data
|
||||
*/
|
||||
export class TaskIndexer extends Component implements TaskIndexerInterface {
|
||||
private taskCache: TaskCache;
|
||||
private parser: TaskParser;
|
||||
private lastIndexTime: Map<string, number> = new Map();
|
||||
|
||||
// Queue for throttling file indexing
|
||||
private indexQueue: TFile[] = [];
|
||||
private isProcessingQueue = false;
|
||||
|
||||
constructor(
|
||||
private app: App,
|
||||
private vault: Vault,
|
||||
private metadataCache: MetadataCache,
|
||||
config?: Partial<TaskParserConfig>
|
||||
) {
|
||||
super();
|
||||
this.taskCache = this.initEmptyCache();
|
||||
this.parser = new TaskParser(config);
|
||||
|
||||
// Setup file change listeners for incremental updates
|
||||
this.setupEventListeners();
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize an empty task cache
|
||||
*/
|
||||
private initEmptyCache(): TaskCache {
|
||||
return {
|
||||
tasks: new Map<string, Task>(),
|
||||
files: new Map<string, Set<string>>(),
|
||||
tags: new Map<string, Set<string>>(),
|
||||
projects: new Map<string, Set<string>>(),
|
||||
contexts: new Map<string, Set<string>>(),
|
||||
dueDate: new Map<string, Set<string>>(),
|
||||
startDate: new Map<string, Set<string>>(),
|
||||
scheduledDate: new Map<string, Set<string>>(),
|
||||
completed: new Map<boolean, Set<string>>(),
|
||||
priority: new Map<number, Set<string>>(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup file change event listeners
|
||||
*/
|
||||
private setupEventListeners(): void {
|
||||
// Watch for file modifications
|
||||
this.registerEvent(
|
||||
this.vault.on("modify", (file) => {
|
||||
if (file instanceof TFile && file.extension === "md") {
|
||||
this.queueFileForIndexing(file);
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
// Watch for file deletions
|
||||
this.registerEvent(
|
||||
this.vault.on("delete", (file) => {
|
||||
if (file instanceof TFile && file.extension === "md") {
|
||||
this.removeFileFromIndex(file);
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
// Watch for new files
|
||||
this.registerEvent(
|
||||
this.vault.on("create", (file) => {
|
||||
if (file instanceof TFile && file.extension === "md") {
|
||||
this.queueFileForIndexing(file);
|
||||
}
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Queue a file for indexing with throttling
|
||||
*/
|
||||
private queueFileForIndexing(file: TFile): void {
|
||||
if (!this.indexQueue.some((f) => f.path === file.path)) {
|
||||
this.indexQueue.push(file);
|
||||
}
|
||||
|
||||
if (!this.isProcessingQueue) {
|
||||
this.processIndexQueue();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process the file index queue with throttling
|
||||
*/
|
||||
private async processIndexQueue(): Promise<void> {
|
||||
if (this.indexQueue.length === 0) {
|
||||
this.isProcessingQueue = false;
|
||||
return;
|
||||
}
|
||||
|
||||
this.isProcessingQueue = true;
|
||||
const file = this.indexQueue.shift();
|
||||
|
||||
if (file) {
|
||||
await this.indexFile(file);
|
||||
|
||||
// Process next file after a small delay
|
||||
setTimeout(() => this.processIndexQueue(), 50);
|
||||
} else {
|
||||
this.isProcessingQueue = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the task indexer
|
||||
*/
|
||||
public async initialize(): Promise<void> {
|
||||
// Start with an empty cache
|
||||
this.taskCache = this.initEmptyCache();
|
||||
|
||||
// Get all markdown files
|
||||
const files = this.vault.getMarkdownFiles();
|
||||
|
||||
// Index the files in batches to avoid UI freezing
|
||||
const batchSize = 20;
|
||||
for (let i = 0; i < files.length; i += batchSize) {
|
||||
const batch = files.slice(i, i + batchSize);
|
||||
|
||||
await Promise.all(batch.map((file) => this.indexFile(file)));
|
||||
|
||||
// Yield to main thread after each batch
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
|
||||
console.log(
|
||||
`Task indexing complete: ${this.taskCache.tasks.size} tasks found`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current task cache
|
||||
*/
|
||||
public getCache(): TaskCache {
|
||||
return this.taskCache;
|
||||
}
|
||||
|
||||
/**
|
||||
* Index all files in the vault
|
||||
*/
|
||||
public async indexAllFiles(): Promise<void> {
|
||||
await this.initialize();
|
||||
}
|
||||
|
||||
/**
|
||||
* Index a single file
|
||||
*/
|
||||
public async indexFile(file: TFile): Promise<void> {
|
||||
try {
|
||||
// Skip if file has not been modified since last indexing
|
||||
const fileStats = await this.vault.adapter.stat(file.path);
|
||||
const lastMtime = this.lastIndexTime.get(file.path);
|
||||
|
||||
if (lastMtime && fileStats?.mtime && fileStats.mtime <= lastMtime) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Read file content
|
||||
const fileContent = await this.vault.cachedRead(file);
|
||||
const metadata = this.metadataCache.getFileCache(file);
|
||||
|
||||
// Parse tasks
|
||||
const tasks = await this.parser.parseTasksFromFile(
|
||||
file,
|
||||
fileContent,
|
||||
metadata ?? undefined
|
||||
);
|
||||
|
||||
// Remove existing tasks for this file first
|
||||
this.removeFileFromIndex(file);
|
||||
|
||||
// Update cache with new tasks
|
||||
const fileTaskIds = new Set<string>();
|
||||
|
||||
for (const task of tasks) {
|
||||
// Store task in main task map
|
||||
this.taskCache.tasks.set(task.id, task);
|
||||
fileTaskIds.add(task.id);
|
||||
|
||||
// Update all indexes
|
||||
this.updateIndexMaps(task);
|
||||
}
|
||||
|
||||
// Update file index
|
||||
this.taskCache.files.set(file.path, fileTaskIds);
|
||||
this.lastIndexTime.set(file.path, Date.now());
|
||||
} catch (error) {
|
||||
console.error(`Error indexing file ${file.path}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update index for a modified file - just an alias for indexFile
|
||||
*/
|
||||
public async updateIndex(file: TFile): Promise<void> {
|
||||
await this.indexFile(file);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a file from the index
|
||||
*/
|
||||
private removeFileFromIndex(file: TFile): void {
|
||||
const taskIds = this.taskCache.files.get(file.path);
|
||||
if (!taskIds) return;
|
||||
|
||||
// Remove each task from all indexes
|
||||
for (const taskId of taskIds) {
|
||||
const task = this.taskCache.tasks.get(taskId);
|
||||
if (task) {
|
||||
this.removeTaskFromIndexes(task);
|
||||
}
|
||||
|
||||
// Remove from main task map
|
||||
this.taskCache.tasks.delete(taskId);
|
||||
}
|
||||
|
||||
// Remove from file index
|
||||
this.taskCache.files.delete(file.path);
|
||||
this.lastIndexTime.delete(file.path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Update all index maps for a task
|
||||
*/
|
||||
private updateIndexMaps(task: Task): void {
|
||||
// Update completed status index
|
||||
let completedTasks =
|
||||
this.taskCache.completed.get(task.completed) || new Set();
|
||||
completedTasks.add(task.id);
|
||||
this.taskCache.completed.set(task.completed, completedTasks);
|
||||
|
||||
// Update tag index
|
||||
for (const tag of task.tags) {
|
||||
let tagTasks = this.taskCache.tags.get(tag) || new Set();
|
||||
tagTasks.add(task.id);
|
||||
this.taskCache.tags.set(tag, tagTasks);
|
||||
}
|
||||
|
||||
// Update project index
|
||||
if (task.project) {
|
||||
let projectTasks =
|
||||
this.taskCache.projects.get(task.project) || new Set();
|
||||
projectTasks.add(task.id);
|
||||
this.taskCache.projects.set(task.project, projectTasks);
|
||||
}
|
||||
|
||||
// Update context index
|
||||
if (task.context) {
|
||||
let contextTasks =
|
||||
this.taskCache.contexts.get(task.context) || new Set();
|
||||
contextTasks.add(task.id);
|
||||
this.taskCache.contexts.set(task.context, contextTasks);
|
||||
}
|
||||
|
||||
// Update date indexes
|
||||
if (task.dueDate) {
|
||||
const dateStr = formatDateForIndex(task.dueDate);
|
||||
let dueTasks = this.taskCache.dueDate.get(dateStr) || new Set();
|
||||
dueTasks.add(task.id);
|
||||
this.taskCache.dueDate.set(dateStr, dueTasks);
|
||||
}
|
||||
|
||||
if (task.startDate) {
|
||||
const dateStr = formatDateForIndex(task.startDate);
|
||||
let startTasks = this.taskCache.startDate.get(dateStr) || new Set();
|
||||
startTasks.add(task.id);
|
||||
this.taskCache.startDate.set(dateStr, startTasks);
|
||||
}
|
||||
|
||||
if (task.scheduledDate) {
|
||||
const dateStr = formatDateForIndex(task.scheduledDate);
|
||||
let scheduledTasks =
|
||||
this.taskCache.scheduledDate.get(dateStr) || new Set();
|
||||
scheduledTasks.add(task.id);
|
||||
this.taskCache.scheduledDate.set(dateStr, scheduledTasks);
|
||||
}
|
||||
|
||||
// Update priority index
|
||||
if (task.priority !== undefined) {
|
||||
let priorityTasks =
|
||||
this.taskCache.priority.get(task.priority) || new Set();
|
||||
priorityTasks.add(task.id);
|
||||
this.taskCache.priority.set(task.priority, priorityTasks);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a task from all indexes
|
||||
*/
|
||||
private removeTaskFromIndexes(task: Task): void {
|
||||
// Remove from completed index
|
||||
const completedTasks = this.taskCache.completed.get(task.completed);
|
||||
if (completedTasks) {
|
||||
completedTasks.delete(task.id);
|
||||
if (completedTasks.size === 0) {
|
||||
this.taskCache.completed.delete(task.completed);
|
||||
}
|
||||
}
|
||||
|
||||
// Remove from tag index
|
||||
for (const tag of task.tags) {
|
||||
const tagTasks = this.taskCache.tags.get(tag);
|
||||
if (tagTasks) {
|
||||
tagTasks.delete(task.id);
|
||||
if (tagTasks.size === 0) {
|
||||
this.taskCache.tags.delete(tag);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remove from project index
|
||||
if (task.project) {
|
||||
const projectTasks = this.taskCache.projects.get(task.project);
|
||||
if (projectTasks) {
|
||||
projectTasks.delete(task.id);
|
||||
if (projectTasks.size === 0) {
|
||||
this.taskCache.projects.delete(task.project);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remove from context index
|
||||
if (task.context) {
|
||||
const contextTasks = this.taskCache.contexts.get(task.context);
|
||||
if (contextTasks) {
|
||||
contextTasks.delete(task.id);
|
||||
if (contextTasks.size === 0) {
|
||||
this.taskCache.contexts.delete(task.context);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remove from date indexes
|
||||
if (task.dueDate) {
|
||||
const dateStr = formatDateForIndex(task.dueDate);
|
||||
const dueTasks = this.taskCache.dueDate.get(dateStr);
|
||||
if (dueTasks) {
|
||||
dueTasks.delete(task.id);
|
||||
if (dueTasks.size === 0) {
|
||||
this.taskCache.dueDate.delete(dateStr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (task.startDate) {
|
||||
const dateStr = formatDateForIndex(task.startDate);
|
||||
const startTasks = this.taskCache.startDate.get(dateStr);
|
||||
if (startTasks) {
|
||||
startTasks.delete(task.id);
|
||||
if (startTasks.size === 0) {
|
||||
this.taskCache.startDate.delete(dateStr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (task.scheduledDate) {
|
||||
const dateStr = formatDateForIndex(task.scheduledDate);
|
||||
const scheduledTasks = this.taskCache.scheduledDate.get(dateStr);
|
||||
if (scheduledTasks) {
|
||||
scheduledTasks.delete(task.id);
|
||||
if (scheduledTasks.size === 0) {
|
||||
this.taskCache.scheduledDate.delete(dateStr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remove from priority index
|
||||
if (task.priority !== undefined) {
|
||||
const priorityTasks = this.taskCache.priority.get(task.priority);
|
||||
if (priorityTasks) {
|
||||
priorityTasks.delete(task.id);
|
||||
if (priorityTasks.size === 0) {
|
||||
this.taskCache.priority.delete(task.priority);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Query tasks based on filters and sorting criteria
|
||||
*/
|
||||
public queryTasks(
|
||||
filters: TaskFilter[],
|
||||
sortBy: SortingCriteria[] = []
|
||||
): Task[] {
|
||||
if (filters.length === 0 && this.taskCache.tasks.size < 1000) {
|
||||
// If no filters and small task count, just return all tasks
|
||||
const allTasks = Array.from(this.taskCache.tasks.values());
|
||||
return this.applySorting(allTasks, sortBy);
|
||||
}
|
||||
|
||||
// Start with a null set to indicate we haven't applied any filters yet
|
||||
let resultTaskIds: Set<string> | null = null;
|
||||
|
||||
// Apply each filter
|
||||
for (const filter of filters) {
|
||||
const filteredIds = this.applyFilter(filter);
|
||||
|
||||
if (resultTaskIds === null) {
|
||||
// First filter
|
||||
resultTaskIds = filteredIds;
|
||||
} else if (filter.conjunction === "OR") {
|
||||
// Union sets (OR)
|
||||
filteredIds.forEach((id) => resultTaskIds!.add(id));
|
||||
} else {
|
||||
// Intersection (AND is default)
|
||||
resultTaskIds = new Set(
|
||||
[...resultTaskIds].filter((id) => filteredIds.has(id))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// If we have no filters, include all tasks
|
||||
if (resultTaskIds === null) {
|
||||
resultTaskIds = new Set(this.taskCache.tasks.keys());
|
||||
}
|
||||
|
||||
// Convert to task array
|
||||
const tasks = Array.from(resultTaskIds)
|
||||
.map((id) => this.taskCache.tasks.get(id)!)
|
||||
.filter((task) => task !== undefined);
|
||||
|
||||
// Apply sorting
|
||||
return this.applySorting(tasks, sortBy);
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply a filter to the task cache
|
||||
*/
|
||||
private applyFilter(filter: TaskFilter): Set<string> {
|
||||
switch (filter.type) {
|
||||
case "tag":
|
||||
return this.filterByTag(filter);
|
||||
case "project":
|
||||
return this.filterByProject(filter);
|
||||
case "context":
|
||||
return this.filterByContext(filter);
|
||||
case "status":
|
||||
return this.filterByStatus(filter);
|
||||
case "priority":
|
||||
return this.filterByPriority(filter);
|
||||
case "dueDate":
|
||||
return this.filterByDueDate(filter);
|
||||
case "startDate":
|
||||
return this.filterByStartDate(filter);
|
||||
case "scheduledDate":
|
||||
return this.filterByScheduledDate(filter);
|
||||
default:
|
||||
console.warn(`Unsupported filter type: ${filter.type}`);
|
||||
return new Set();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter tasks by tag
|
||||
*/
|
||||
private filterByTag(filter: TaskFilter): Set<string> {
|
||||
if (filter.operator === "contains") {
|
||||
return this.taskCache.tags.get(filter.value as string) || new Set();
|
||||
} else if (filter.operator === "!=") {
|
||||
// Get all task IDs
|
||||
const allTaskIds = new Set(this.taskCache.tasks.keys());
|
||||
// Get tasks with the specified tag
|
||||
const tagTaskIds =
|
||||
this.taskCache.tags.get(filter.value as string) || new Set();
|
||||
// Return tasks that don't have the tag
|
||||
return new Set([...allTaskIds].filter((id) => !tagTaskIds.has(id)));
|
||||
}
|
||||
|
||||
return new Set();
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter tasks by project
|
||||
*/
|
||||
private filterByProject(filter: TaskFilter): Set<string> {
|
||||
if (filter.operator === "=") {
|
||||
return (
|
||||
this.taskCache.projects.get(filter.value as string) || new Set()
|
||||
);
|
||||
} else if (filter.operator === "!=") {
|
||||
// Get all task IDs
|
||||
const allTaskIds = new Set(this.taskCache.tasks.keys());
|
||||
// Get tasks with the specified project
|
||||
const projectTaskIds =
|
||||
this.taskCache.projects.get(filter.value as string) ||
|
||||
new Set();
|
||||
// Return tasks that don't have the project
|
||||
return new Set(
|
||||
[...allTaskIds].filter((id) => !projectTaskIds.has(id))
|
||||
);
|
||||
} else if (filter.operator === "empty") {
|
||||
// Get all task IDs
|
||||
const allTaskIds = new Set(this.taskCache.tasks.keys());
|
||||
// Get all tasks with any project
|
||||
const tasksWithProject = new Set<string>();
|
||||
for (const projectTasks of this.taskCache.projects.values()) {
|
||||
for (const taskId of projectTasks) {
|
||||
tasksWithProject.add(taskId);
|
||||
}
|
||||
}
|
||||
// Return tasks without a project
|
||||
return new Set(
|
||||
[...allTaskIds].filter((id) => !tasksWithProject.has(id))
|
||||
);
|
||||
}
|
||||
|
||||
return new Set();
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter tasks by context
|
||||
*/
|
||||
private filterByContext(filter: TaskFilter): Set<string> {
|
||||
if (filter.operator === "=") {
|
||||
return (
|
||||
this.taskCache.contexts.get(filter.value as string) || new Set()
|
||||
);
|
||||
} else if (filter.operator === "!=") {
|
||||
// Get all task IDs
|
||||
const allTaskIds = new Set(this.taskCache.tasks.keys());
|
||||
// Get tasks with the specified context
|
||||
const contextTaskIds =
|
||||
this.taskCache.contexts.get(filter.value as string) ||
|
||||
new Set();
|
||||
// Return tasks that don't have the context
|
||||
return new Set(
|
||||
[...allTaskIds].filter((id) => !contextTaskIds.has(id))
|
||||
);
|
||||
} else if (filter.operator === "empty") {
|
||||
// Get all task IDs
|
||||
const allTaskIds = new Set(this.taskCache.tasks.keys());
|
||||
// Get all tasks with any context
|
||||
const tasksWithContext = new Set<string>();
|
||||
for (const contextTasks of this.taskCache.contexts.values()) {
|
||||
for (const taskId of contextTasks) {
|
||||
tasksWithContext.add(taskId);
|
||||
}
|
||||
}
|
||||
// Return tasks without a context
|
||||
return new Set(
|
||||
[...allTaskIds].filter((id) => !tasksWithContext.has(id))
|
||||
);
|
||||
}
|
||||
|
||||
return new Set();
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter tasks by status (completed or not)
|
||||
*/
|
||||
private filterByStatus(filter: TaskFilter): Set<string> {
|
||||
if (filter.operator === "=") {
|
||||
return (
|
||||
this.taskCache.completed.get(filter.value as boolean) ||
|
||||
new Set()
|
||||
);
|
||||
}
|
||||
|
||||
return new Set();
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter tasks by priority
|
||||
*/
|
||||
private filterByPriority(filter: TaskFilter): Set<string> {
|
||||
if (filter.operator === "=") {
|
||||
return (
|
||||
this.taskCache.priority.get(filter.value as number) || new Set()
|
||||
);
|
||||
} else if (filter.operator === ">") {
|
||||
// Get tasks with priority higher than the specified value
|
||||
const result = new Set<string>();
|
||||
for (const [
|
||||
priority,
|
||||
taskIds,
|
||||
] of this.taskCache.priority.entries()) {
|
||||
if (priority > (filter.value as number)) {
|
||||
for (const taskId of taskIds) {
|
||||
result.add(taskId);
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
} else if (filter.operator === "<") {
|
||||
// Get tasks with priority lower than the specified value
|
||||
const result = new Set<string>();
|
||||
for (const [
|
||||
priority,
|
||||
taskIds,
|
||||
] of this.taskCache.priority.entries()) {
|
||||
if (priority < (filter.value as number)) {
|
||||
for (const taskId of taskIds) {
|
||||
result.add(taskId);
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
return new Set();
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter tasks by due date
|
||||
*/
|
||||
private filterByDueDate(filter: TaskFilter): Set<string> {
|
||||
if (filter.operator === "=") {
|
||||
// Exact match on date string (YYYY-MM-DD)
|
||||
return (
|
||||
this.taskCache.dueDate.get(filter.value as string) || new Set()
|
||||
);
|
||||
} else if (
|
||||
filter.operator === "before" ||
|
||||
filter.operator === "after"
|
||||
) {
|
||||
// Convert value to Date if it's a string
|
||||
let compareDate: Date;
|
||||
if (typeof filter.value === "string") {
|
||||
compareDate = new Date(filter.value);
|
||||
} else {
|
||||
compareDate = new Date(filter.value as number);
|
||||
}
|
||||
|
||||
// Get all tasks with due dates
|
||||
const result = new Set<string>();
|
||||
for (const [dateStr, taskIds] of this.taskCache.dueDate.entries()) {
|
||||
const date = new Date(dateStr);
|
||||
|
||||
if (
|
||||
(filter.operator === "before" && date < compareDate) ||
|
||||
(filter.operator === "after" && date > compareDate)
|
||||
) {
|
||||
for (const taskId of taskIds) {
|
||||
result.add(taskId);
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
} else if (filter.operator === "empty") {
|
||||
// Get all task IDs
|
||||
const allTaskIds = new Set(this.taskCache.tasks.keys());
|
||||
// Get all tasks with any due date
|
||||
const tasksWithDueDate = new Set<string>();
|
||||
for (const dueTasks of this.taskCache.dueDate.values()) {
|
||||
for (const taskId of dueTasks) {
|
||||
tasksWithDueDate.add(taskId);
|
||||
}
|
||||
}
|
||||
// Return tasks without a due date
|
||||
return new Set(
|
||||
[...allTaskIds].filter((id) => !tasksWithDueDate.has(id))
|
||||
);
|
||||
}
|
||||
|
||||
return new Set();
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter tasks by start date
|
||||
*/
|
||||
private filterByStartDate(filter: TaskFilter): Set<string> {
|
||||
// Similar implementation to filterByDueDate
|
||||
if (filter.operator === "=") {
|
||||
return (
|
||||
this.taskCache.startDate.get(filter.value as string) ||
|
||||
new Set()
|
||||
);
|
||||
} else if (
|
||||
filter.operator === "before" ||
|
||||
filter.operator === "after"
|
||||
) {
|
||||
let compareDate: Date;
|
||||
if (typeof filter.value === "string") {
|
||||
compareDate = new Date(filter.value);
|
||||
} else {
|
||||
compareDate = new Date(filter.value as number);
|
||||
}
|
||||
|
||||
const result = new Set<string>();
|
||||
for (const [
|
||||
dateStr,
|
||||
taskIds,
|
||||
] of this.taskCache.startDate.entries()) {
|
||||
const date = new Date(dateStr);
|
||||
|
||||
if (
|
||||
(filter.operator === "before" && date < compareDate) ||
|
||||
(filter.operator === "after" && date > compareDate)
|
||||
) {
|
||||
for (const taskId of taskIds) {
|
||||
result.add(taskId);
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
} else if (filter.operator === "empty") {
|
||||
const allTaskIds = new Set(this.taskCache.tasks.keys());
|
||||
const tasksWithStartDate = new Set<string>();
|
||||
for (const startTasks of this.taskCache.startDate.values()) {
|
||||
for (const taskId of startTasks) {
|
||||
tasksWithStartDate.add(taskId);
|
||||
}
|
||||
}
|
||||
return new Set(
|
||||
[...allTaskIds].filter((id) => !tasksWithStartDate.has(id))
|
||||
);
|
||||
}
|
||||
|
||||
return new Set();
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter tasks by scheduled date
|
||||
*/
|
||||
private filterByScheduledDate(filter: TaskFilter): Set<string> {
|
||||
// Similar implementation to filterByDueDate
|
||||
if (filter.operator === "=") {
|
||||
return (
|
||||
this.taskCache.scheduledDate.get(filter.value as string) ||
|
||||
new Set()
|
||||
);
|
||||
} else if (
|
||||
filter.operator === "before" ||
|
||||
filter.operator === "after"
|
||||
) {
|
||||
let compareDate: Date;
|
||||
if (typeof filter.value === "string") {
|
||||
compareDate = new Date(filter.value);
|
||||
} else {
|
||||
compareDate = new Date(filter.value as number);
|
||||
}
|
||||
|
||||
const result = new Set<string>();
|
||||
for (const [
|
||||
dateStr,
|
||||
taskIds,
|
||||
] of this.taskCache.scheduledDate.entries()) {
|
||||
const date = new Date(dateStr);
|
||||
|
||||
if (
|
||||
(filter.operator === "before" && date < compareDate) ||
|
||||
(filter.operator === "after" && date > compareDate)
|
||||
) {
|
||||
for (const taskId of taskIds) {
|
||||
result.add(taskId);
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
} else if (filter.operator === "empty") {
|
||||
const allTaskIds = new Set(this.taskCache.tasks.keys());
|
||||
const tasksWithScheduledDate = new Set<string>();
|
||||
for (const scheduledTasks of this.taskCache.scheduledDate.values()) {
|
||||
for (const taskId of scheduledTasks) {
|
||||
tasksWithScheduledDate.add(taskId);
|
||||
}
|
||||
}
|
||||
return new Set(
|
||||
[...allTaskIds].filter((id) => !tasksWithScheduledDate.has(id))
|
||||
);
|
||||
}
|
||||
|
||||
return new Set();
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply sorting to tasks
|
||||
*/
|
||||
private applySorting(tasks: Task[], sortBy: SortingCriteria[]): Task[] {
|
||||
if (sortBy.length === 0) {
|
||||
// Default sorting: priority desc, due date asc
|
||||
return [...tasks].sort((a, b) => {
|
||||
// First by priority (high to low)
|
||||
const priorityA = a.priority || 0;
|
||||
const priorityB = b.priority || 0;
|
||||
if (priorityA !== priorityB) {
|
||||
return priorityB - priorityA;
|
||||
}
|
||||
|
||||
// Then by due date (earliest first)
|
||||
const dueDateA = a.dueDate || Number.MAX_SAFE_INTEGER;
|
||||
const dueDateB = b.dueDate || Number.MAX_SAFE_INTEGER;
|
||||
return dueDateA - dueDateB;
|
||||
});
|
||||
}
|
||||
|
||||
return [...tasks].sort((a, b) => {
|
||||
for (const { field, direction } of sortBy) {
|
||||
const valueA = a[field];
|
||||
const valueB = b[field];
|
||||
|
||||
// Handle undefined values
|
||||
if (valueA === undefined && valueB === undefined) {
|
||||
continue;
|
||||
} else if (valueA === undefined) {
|
||||
return direction === "asc" ? 1 : -1;
|
||||
} else if (valueB === undefined) {
|
||||
return direction === "asc" ? -1 : 1;
|
||||
}
|
||||
|
||||
// Compare values
|
||||
if (valueA !== valueB) {
|
||||
const multiplier = direction === "asc" ? 1 : -1;
|
||||
|
||||
if (
|
||||
typeof valueA === "string" &&
|
||||
typeof valueB === "string"
|
||||
) {
|
||||
return valueA.localeCompare(valueB) * multiplier;
|
||||
} else if (
|
||||
typeof valueA === "number" &&
|
||||
typeof valueB === "number"
|
||||
) {
|
||||
return (valueA - valueB) * multiplier;
|
||||
} else if (
|
||||
valueA instanceof Date &&
|
||||
valueB instanceof Date
|
||||
) {
|
||||
return (
|
||||
(valueA.getTime() - valueB.getTime()) * multiplier
|
||||
);
|
||||
} else {
|
||||
// Convert to string and compare as fallback
|
||||
return (
|
||||
String(valueA).localeCompare(String(valueB)) *
|
||||
multiplier
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get task by ID
|
||||
*/
|
||||
public getTaskById(id: string): Task | undefined {
|
||||
return this.taskCache.tasks.get(id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new task
|
||||
*/
|
||||
public async createTask(taskData: Partial<Task>): Promise<Task> {
|
||||
throw new Error("Not implemented");
|
||||
}
|
||||
|
||||
/**
|
||||
* Update an existing task
|
||||
*/
|
||||
public async updateTask(task: Task): Promise<void> {
|
||||
throw new Error("Not implemented");
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a task
|
||||
*/
|
||||
public async deleteTask(taskId: string): Promise<void> {
|
||||
throw new Error("Not implemented");
|
||||
}
|
||||
}
|
||||
257
src/utils/import/TaskParser.ts
Normal file
257
src/utils/import/TaskParser.ts
Normal file
|
|
@ -0,0 +1,257 @@
|
|||
/**
|
||||
* Optimized task parser focused on task data only
|
||||
*/
|
||||
|
||||
import { CachedMetadata, FileStats, TFile } from "obsidian";
|
||||
import { Task, TaskParserConfig } from "../types/TaskIndex";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
|
||||
/**
|
||||
* Default configuration for the task parser
|
||||
*/
|
||||
export const DEFAULT_TASK_PARSER_CONFIG: TaskParserConfig = {
|
||||
taskRegex: /^([\s>]*- \[(.)\])\s*(.*)$/m,
|
||||
startDateFormat: "📅 YYYY-MM-DD",
|
||||
dueDateFormat: "⏳ YYYY-MM-DD",
|
||||
scheduledDateFormat: "⏰ YYYY-MM-DD",
|
||||
projectPrefix: "#project/",
|
||||
contextPrefix: "@",
|
||||
priorityMarkers: {
|
||||
"🔺": 5, // Highest
|
||||
"⏫": 4, // High
|
||||
"🔼": 3, // Medium
|
||||
"🔽": 2, // Low
|
||||
"⏬️": 1, // Lowest
|
||||
"[#A]": 4, // High (letter format)
|
||||
"[#B]": 3, // Medium (letter format)
|
||||
"[#C]": 2, // Low (letter format)
|
||||
},
|
||||
};
|
||||
|
||||
export class TaskParser {
|
||||
// Regular expressions for parsing task components
|
||||
private readonly startDateRegex = /📅 (\d{4}-\d{2}-\d{2})/;
|
||||
private readonly completedDateRegex = /✅ (\d{4}-\d{2}-\d{2})/;
|
||||
private readonly dueDateRegex = /⏳ (\d{4}-\d{2}-\d{2})/;
|
||||
private readonly scheduledDateRegex = /⏰ (\d{4}-\d{2}-\d{2})/;
|
||||
private readonly recurrenceRegex = /🔁 (.*?)(?=\s|$)/;
|
||||
private readonly tagRegex = /#[\w\/-]+/g;
|
||||
private readonly contextRegex = /@[\w-]+/g;
|
||||
private readonly priorityRegex = /🔼|⏫|🔽|⏬️|🔺|\[#[A-C]\]/;
|
||||
private readonly estimatedTimeRegex = /estimated:\s*(\d+)([hm])/i;
|
||||
|
||||
private config: TaskParserConfig;
|
||||
|
||||
constructor(config: Partial<TaskParserConfig> = {}) {
|
||||
this.config = { ...DEFAULT_TASK_PARSER_CONFIG, ...config };
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a task from a text line
|
||||
*/
|
||||
parseTask(text: string, filePath: string, lineNum: number): Task | null {
|
||||
const match = text.match(this.config.taskRegex);
|
||||
if (!match) return null;
|
||||
|
||||
const [, prefix, status, content] = match;
|
||||
const completed = status.toLowerCase() === "x";
|
||||
|
||||
// Basic task info
|
||||
const task: Task = {
|
||||
id: uuidv4(),
|
||||
content: content.trim(),
|
||||
filePath,
|
||||
line: lineNum,
|
||||
completed,
|
||||
originalMarkdown: text,
|
||||
tags: [],
|
||||
children: [],
|
||||
};
|
||||
|
||||
// Extract metadata
|
||||
this.extractDates(task, content);
|
||||
this.extractTags(task, content);
|
||||
this.extractContext(task, content);
|
||||
this.extractPriority(task, content);
|
||||
this.extractEstimatedTime(task, content);
|
||||
this.extractRecurrence(task, content);
|
||||
|
||||
return task;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract dates from task content
|
||||
*/
|
||||
private extractDates(task: Task, content: string): void {
|
||||
// Start date
|
||||
const startDateMatch = content.match(this.startDateRegex);
|
||||
if (startDateMatch) {
|
||||
task.startDate = new Date(startDateMatch[1]).getTime();
|
||||
}
|
||||
|
||||
// Due date
|
||||
const dueDateMatch = content.match(this.dueDateRegex);
|
||||
if (dueDateMatch) {
|
||||
task.dueDate = new Date(dueDateMatch[1]).getTime();
|
||||
}
|
||||
|
||||
// Scheduled date
|
||||
const scheduledDateMatch = content.match(this.scheduledDateRegex);
|
||||
if (scheduledDateMatch) {
|
||||
task.scheduledDate = new Date(scheduledDateMatch[1]).getTime();
|
||||
}
|
||||
|
||||
// Completion date
|
||||
const completedDateMatch = content.match(this.completedDateRegex);
|
||||
if (completedDateMatch) {
|
||||
task.completedDate = new Date(completedDateMatch[1]).getTime();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract tags from task content
|
||||
*/
|
||||
private extractTags(task: Task, content: string): void {
|
||||
const tagMatches = content.match(this.tagRegex) || [];
|
||||
task.tags = tagMatches.map((tag) => tag.trim());
|
||||
|
||||
// Check for project tags
|
||||
const projectTag = task.tags.find((tag) =>
|
||||
tag.startsWith(this.config.projectPrefix!)
|
||||
);
|
||||
if (projectTag) {
|
||||
task.project = projectTag.substring(
|
||||
this.config.projectPrefix!.length
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract context from task content
|
||||
*/
|
||||
private extractContext(task: Task, content: string): void {
|
||||
const contextMatches = content.match(this.contextRegex) || [];
|
||||
if (contextMatches.length > 0) {
|
||||
// Use the first context tag as the primary context
|
||||
task.context = contextMatches[0]?.substring(1); // Remove the @ symbol
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract priority from task content
|
||||
*/
|
||||
private extractPriority(task: Task, content: string): void {
|
||||
const priorityMatch = content.match(this.priorityRegex);
|
||||
if (priorityMatch && this.config.priorityMarkers) {
|
||||
task.priority =
|
||||
this.config.priorityMarkers[priorityMatch[0]] || undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract estimated time from task content
|
||||
*/
|
||||
private extractEstimatedTime(task: Task, content: string): void {
|
||||
const timeMatch = content.match(this.estimatedTimeRegex);
|
||||
if (timeMatch) {
|
||||
const value = parseInt(timeMatch[1]);
|
||||
const unit = timeMatch[2].toLowerCase();
|
||||
|
||||
if (unit === "h") {
|
||||
task.estimatedTime = value * 60; // Convert hours to minutes
|
||||
} else {
|
||||
task.estimatedTime = value; // Already in minutes
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract recurrence information from task content
|
||||
*/
|
||||
private extractRecurrence(task: Task, content: string): void {
|
||||
const recurrenceMatch = content.match(this.recurrenceRegex);
|
||||
if (recurrenceMatch) {
|
||||
task.recurrence = recurrenceMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate markdown text from a task object
|
||||
*/
|
||||
generateMarkdown(task: Task): string {
|
||||
let markdown = `- [${task.completed ? "x" : " "}] ${task.content}`;
|
||||
|
||||
// This is simplified - a full implementation would reconstruct the original
|
||||
// markdown with all the metadata in the correct format
|
||||
|
||||
return markdown;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse all tasks from a file
|
||||
*/
|
||||
async parseTasksFromFile(
|
||||
file: TFile,
|
||||
fileContent: string,
|
||||
metadata?: CachedMetadata
|
||||
): Promise<Task[]> {
|
||||
const lines = fileContent.split("\n");
|
||||
const tasks: Task[] = [];
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
const task = this.parseTask(line, file.path, i);
|
||||
|
||||
if (task) {
|
||||
tasks.push(task);
|
||||
}
|
||||
}
|
||||
|
||||
// Build parent-child relationships
|
||||
this.buildTaskHierarchy(tasks);
|
||||
|
||||
return tasks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build parent-child relationships between tasks
|
||||
*/
|
||||
private buildTaskHierarchy(tasks: Task[]): void {
|
||||
// This is a simple indent-based approach
|
||||
// A more sophisticated implementation would use Obsidian's list item metadata
|
||||
|
||||
// Sort tasks by line number
|
||||
tasks.sort((a, b) => a.line - b.line);
|
||||
|
||||
// Build parent-child relationships based on indentation
|
||||
for (let i = 0; i < tasks.length; i++) {
|
||||
const currentTask = tasks[i];
|
||||
const currentIndent = this.getIndentLevel(
|
||||
currentTask.originalMarkdown
|
||||
);
|
||||
|
||||
// Look for potential parent tasks (must be before current task and have less indentation)
|
||||
for (let j = i - 1; j >= 0; j--) {
|
||||
const potentialParent = tasks[j];
|
||||
const parentIndent = this.getIndentLevel(
|
||||
potentialParent.originalMarkdown
|
||||
);
|
||||
|
||||
if (parentIndent < currentIndent) {
|
||||
// Found a parent
|
||||
currentTask.parent = potentialParent.id;
|
||||
potentialParent.children.push(currentTask.id);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get indentation level of a line
|
||||
*/
|
||||
private getIndentLevel(line: string): number {
|
||||
const match = line.match(/^(\s*)/);
|
||||
return match ? match[1].length : 0;
|
||||
}
|
||||
}
|
||||
696
src/utils/import/markdown.ts
Normal file
696
src/utils/import/markdown.ts
Normal file
|
|
@ -0,0 +1,696 @@
|
|||
import { JsonLink, Link } from "expression/link";
|
||||
import { getExtension, getFileTitle } from "utils/normalizers";
|
||||
import { CachedMetadata, FileStats, FrontMatterCache } from "obsidian";
|
||||
import { parse as parseYaml } from "yaml";
|
||||
import BTree from "sorted-btree";
|
||||
import {
|
||||
InlineField,
|
||||
JsonInlineField,
|
||||
asInlineField,
|
||||
extractFullLineField,
|
||||
extractInlineFields,
|
||||
jsonInlineField,
|
||||
} from "./inline-field";
|
||||
import {
|
||||
JsonMarkdownBlock,
|
||||
JsonMarkdownListBlock,
|
||||
JsonMarkdownListItem,
|
||||
JsonMarkdownPage,
|
||||
JsonMarkdownSection,
|
||||
JsonMarkdownTaskItem,
|
||||
JsonMarkdownDatablock,
|
||||
JsonMarkdownCodeblock,
|
||||
JsonFrontmatterEntry,
|
||||
} from "index/types/json/markdown";
|
||||
import { JsonConversion } from "index/types/json/common";
|
||||
import { YamlConversion } from "index/types/yaml";
|
||||
import { mapObjectValues } from "utils/data";
|
||||
|
||||
/** Matches yaml datablocks, which show up as independent objects in the datacore index. */
|
||||
const YAML_DATA_REGEX = /```yaml:data/i;
|
||||
/** Matches the start of any codeblock fence. */
|
||||
const CODEBLOCK_FENCE_REGEX = /^(?:```|~~~)(.*)$/im;
|
||||
/** Matches list items (including inside text blocks). */
|
||||
const LIST_ITEM_REGEX =
|
||||
/^[\s>]*(\d+\.|\d+\)|\*|-|\+)\s*(\[.{0,1}\])?\s*(.*)$/mu;
|
||||
|
||||
/**
|
||||
* Given the raw source and Obsidian metadata for a given markdown file,
|
||||
* return full markdown file metadata.
|
||||
*/
|
||||
export function markdownImport(
|
||||
path: string,
|
||||
markdown: string,
|
||||
metadata: CachedMetadata,
|
||||
stats: FileStats
|
||||
): JsonMarkdownPage {
|
||||
const {
|
||||
lines,
|
||||
metadata: pageMetadata,
|
||||
frontmatter,
|
||||
sections,
|
||||
} = markdownSourceImport(path, markdown, metadata);
|
||||
const page = new PageData(
|
||||
path,
|
||||
stats,
|
||||
lines.length,
|
||||
pageMetadata,
|
||||
sections,
|
||||
frontmatter
|
||||
);
|
||||
|
||||
return page.build();
|
||||
}
|
||||
|
||||
/** Import markdown file metadata, producing all relevant markdown data. */
|
||||
export function markdownSourceImport(
|
||||
path: string,
|
||||
markdown: string,
|
||||
metadata: CachedMetadata
|
||||
): {
|
||||
lines: string[];
|
||||
metadata: Metadata;
|
||||
frontmatter?: Record<string, JsonFrontmatterEntry>;
|
||||
sections: SectionData[];
|
||||
} {
|
||||
const frontmatter: Record<string, JsonFrontmatterEntry> | undefined =
|
||||
metadata.frontmatter
|
||||
? parseFrontmatterBlock(metadata.frontmatter)
|
||||
: undefined;
|
||||
const lines = markdown.split(/\r\n|\r|\n/);
|
||||
const markdownMetadata = new Metadata();
|
||||
const sectionArray: SectionData[] = [];
|
||||
//////////////
|
||||
// Sections //
|
||||
//////////////
|
||||
|
||||
const metaheadings = metadata.headings ?? [];
|
||||
metaheadings.sort((a, b) => a.position.start.line - b.position.start.line);
|
||||
|
||||
const sections = new BTree<number, SectionData>(undefined, (a, b) => a - b);
|
||||
for (let index = 0; index < metaheadings.length; index++) {
|
||||
const entry = metaheadings[index];
|
||||
const start = entry.position.start.line;
|
||||
const end =
|
||||
index == metaheadings.length - 1
|
||||
? lines.length
|
||||
: metaheadings[index + 1].position.start.line;
|
||||
|
||||
const section = new SectionData(
|
||||
start,
|
||||
end,
|
||||
entry.heading,
|
||||
entry.level,
|
||||
index + 1
|
||||
);
|
||||
sections.set(start, section);
|
||||
}
|
||||
|
||||
// Add an implicit section for the "heading" section of the page if there is not an immediate header but there is
|
||||
// some content in the file. If there are other sections, then go up to that, otherwise, go for the entire file.
|
||||
if (sections.size == 0) {
|
||||
if (!emptylines(lines, 0, lines.length)) {
|
||||
const section = new SectionData(
|
||||
0,
|
||||
lines.length,
|
||||
getFileTitle(path),
|
||||
1,
|
||||
0
|
||||
);
|
||||
sections.set(0, section);
|
||||
}
|
||||
} else {
|
||||
// Find the start of the first section.
|
||||
const first = sections.getPairOrNextHigher(0)?.[1]!!;
|
||||
|
||||
if (first.start > 0 && !emptylines(lines, 0, first.start)) {
|
||||
const section = new SectionData(
|
||||
0,
|
||||
first.start,
|
||||
getFileTitle(path),
|
||||
1,
|
||||
0
|
||||
);
|
||||
|
||||
sections.set(0, section);
|
||||
}
|
||||
}
|
||||
|
||||
////////////
|
||||
// Blocks //
|
||||
////////////
|
||||
|
||||
// All blocks; we will assign tags and other metadata to blocks as we encounter them. At the end, only blocks that
|
||||
// have actual metadata will be stored to save on memory pressure.
|
||||
const blocks = new BTree<number, BlockData>(undefined, (a, b) => a - b);
|
||||
let blockOrdinal = 1;
|
||||
for (const block of metadata.sections || []) {
|
||||
// Skip headings blocks, we handle them specially as sections.
|
||||
if (block.type === "heading") continue;
|
||||
|
||||
const start = block.position.start.line;
|
||||
const end = block.position.end.line + 1;
|
||||
const startLine = lines[start]; // to use to check the codeblock type
|
||||
|
||||
if (block.type === "list") {
|
||||
blocks.set(
|
||||
start,
|
||||
new ListBlockData(start, end, blockOrdinal++, block.id)
|
||||
);
|
||||
} else if (block.type == "code" && YAML_DATA_REGEX.test(startLine)) {
|
||||
const yaml: string = lines
|
||||
.slice(start + 1, end - 1)
|
||||
.join("\n")
|
||||
.replace(/\t/gm, " ");
|
||||
const split: Record<string, JsonFrontmatterEntry> =
|
||||
parseFrontmatterBlock(parseYaml(yaml));
|
||||
|
||||
blocks.set(
|
||||
start,
|
||||
new DatablockData(start, end, blockOrdinal++, split, block.id)
|
||||
);
|
||||
} else if (block.type === "code") {
|
||||
// Check if the block is fenced.
|
||||
const match = startLine.match(CODEBLOCK_FENCE_REGEX);
|
||||
if (!match) {
|
||||
// This is an indented-style codeblock.
|
||||
blocks.set(
|
||||
start,
|
||||
new CodeblockData(
|
||||
start,
|
||||
end,
|
||||
blockOrdinal++,
|
||||
[],
|
||||
"indent",
|
||||
start,
|
||||
end,
|
||||
block.id
|
||||
)
|
||||
);
|
||||
} else {
|
||||
const languages =
|
||||
match.length > 1 && match[1] ? match[1].split(",") : [];
|
||||
blocks.set(
|
||||
start,
|
||||
new CodeblockData(
|
||||
start,
|
||||
end,
|
||||
blockOrdinal++,
|
||||
languages,
|
||||
"fenced",
|
||||
start + 1,
|
||||
end - 1,
|
||||
block.id
|
||||
)
|
||||
);
|
||||
}
|
||||
} else {
|
||||
blocks.set(
|
||||
start,
|
||||
new BaseBlockData(
|
||||
start,
|
||||
end,
|
||||
blockOrdinal++,
|
||||
block.type,
|
||||
block.id
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Add blocks to sections.
|
||||
for (const block of blocks.values()) {
|
||||
lookup(block.start, sections)?.block(block);
|
||||
}
|
||||
|
||||
///////////
|
||||
// Lists //
|
||||
///////////
|
||||
|
||||
// All list items in lists. Start with a simple trivial pass.
|
||||
const listItems = new BTree<number, ListItemData>(
|
||||
undefined,
|
||||
(a, b) => a - b
|
||||
);
|
||||
for (const list of metadata.listItems || []) {
|
||||
const line = lines[list.position.start.line];
|
||||
|
||||
// TODO: Implement flag which skips indexing list items.
|
||||
const match = line.match(LIST_ITEM_REGEX);
|
||||
let symbol = undefined,
|
||||
text = undefined;
|
||||
if (match) {
|
||||
symbol = match[1];
|
||||
text = match[3];
|
||||
}
|
||||
|
||||
const item = new ListItemData(
|
||||
list.position.start.line,
|
||||
list.position.end.line + 1,
|
||||
list.parent,
|
||||
symbol,
|
||||
list.id,
|
||||
list.task,
|
||||
text
|
||||
);
|
||||
|
||||
listItems.set(item.start, item);
|
||||
}
|
||||
|
||||
// In the second list pass, actually construct the list heirarchy.
|
||||
for (const item of listItems.values()) {
|
||||
if (item.parentLine < 0) {
|
||||
const listBlock = blocks.get(-item.parentLine);
|
||||
if (!listBlock || !(listBlock.type === "list")) continue;
|
||||
|
||||
(listBlock as ListBlockData).items.push(item);
|
||||
} else {
|
||||
listItems.get(item.parentLine)?.elements.push(item);
|
||||
}
|
||||
}
|
||||
|
||||
//////////
|
||||
// Tags //
|
||||
//////////
|
||||
|
||||
// For each tag, assign it to the appropriate section and block that it is a part of.
|
||||
for (const tagdef of metadata.tags ?? []) {
|
||||
const tag = tagdef.tag.startsWith("#") ? tagdef.tag : "#" + tagdef.tag;
|
||||
const line = tagdef.position.start.line;
|
||||
markdownMetadata.tag(tag);
|
||||
|
||||
lookup(line, sections)?.metadata.tag(tag);
|
||||
lookup(line, blocks)?.metadata.tag(tag);
|
||||
lookup(line, listItems)?.metadata.tag(tag);
|
||||
}
|
||||
|
||||
// Add frontmatter tags.
|
||||
if (metadata.frontmatter) {
|
||||
for (const rawtag of extractTags(metadata.frontmatter)) {
|
||||
const tag = rawtag.startsWith("#") ? rawtag : "#" + rawtag;
|
||||
markdownMetadata.tag(tag);
|
||||
}
|
||||
}
|
||||
|
||||
///////////
|
||||
// Links //
|
||||
///////////
|
||||
|
||||
for (let linkdef of metadata.links ?? []) {
|
||||
const link = Link.infer(linkdef.link);
|
||||
const line = linkdef.position.start.line;
|
||||
markdownMetadata.link(link);
|
||||
|
||||
lookup(line, sections)?.metadata.link(link);
|
||||
lookup(line, blocks)?.metadata.link(link);
|
||||
lookup(line, listItems)?.metadata.link(link);
|
||||
}
|
||||
|
||||
///////////////////////
|
||||
// Frontmatter Links //
|
||||
///////////////////////
|
||||
|
||||
// Frontmatter links are only assigned to the page.
|
||||
for (const linkdef of metadata.frontmatterLinks ?? []) {
|
||||
markdownMetadata.link(
|
||||
Link.infer(linkdef.link, false, linkdef.displayText)
|
||||
);
|
||||
}
|
||||
|
||||
///////////////////
|
||||
// Inline Fields //
|
||||
///////////////////
|
||||
|
||||
for (const field of iterateInlineFields(lines)) {
|
||||
const line = field.position.line;
|
||||
markdownMetadata.inlineField(field);
|
||||
|
||||
lookup(line, sections)?.metadata.inlineField(field);
|
||||
lookup(line, blocks)?.metadata.inlineField(field);
|
||||
lookup(line, listItems)?.metadata.inlineField(field);
|
||||
}
|
||||
sectionArray.push(...sections.values());
|
||||
return {
|
||||
lines,
|
||||
frontmatter,
|
||||
metadata: markdownMetadata,
|
||||
sections: sectionArray,
|
||||
};
|
||||
}
|
||||
//////////////////
|
||||
// Parsing Aids //
|
||||
//////////////////
|
||||
|
||||
/** Check if the given line range is all empty. Start is inclusive, end exclusive. */
|
||||
function emptylines(lines: string[], start: number, end: number): boolean {
|
||||
for (let index = start; index < end; index++) {
|
||||
if (lines[index].trim() !== "") return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Yields all inline fields found in the document by traversing line by line through the document. Performs some optimizations
|
||||
* to skip extra-large lines, and can be disabled.
|
||||
*/
|
||||
export function* iterateInlineFields(
|
||||
content: string[]
|
||||
): Generator<InlineField> {
|
||||
for (let lineno = 0; lineno < content.length; lineno++) {
|
||||
const line = content[lineno];
|
||||
|
||||
// Fast-bailout for lines that are too long or do not contain '::'.
|
||||
if (line.length > 32768 || !line.includes("::")) continue;
|
||||
|
||||
// TODO: Re-add support for those custom emoji fields on tasks and similar.
|
||||
let inlineFields = extractInlineFields(line);
|
||||
if (inlineFields.length > 0) {
|
||||
for (let ifield of inlineFields)
|
||||
yield asInlineField(ifield, lineno);
|
||||
} else {
|
||||
let fullLine = extractFullLineField(line);
|
||||
if (fullLine) yield asInlineField(fullLine, lineno);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Top-level function which maps a YAML block - including frontmatter - into frontmatter entries. */
|
||||
export function parseFrontmatterBlock(
|
||||
block: Record<string, any>
|
||||
): Record<string, JsonFrontmatterEntry> {
|
||||
const result: Record<string, JsonFrontmatterEntry> = {};
|
||||
for (const key of Object.keys(block)) {
|
||||
const value = block[key];
|
||||
|
||||
result[key.toLowerCase()] = {
|
||||
key: key,
|
||||
value: JsonConversion.json(YamlConversion.literal(value)),
|
||||
raw: value,
|
||||
};
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Finds an element which contains the given line. */
|
||||
export function lookup<T extends { start: number; end: number }>(
|
||||
line: number,
|
||||
tree: BTree<number, T>
|
||||
): T | undefined {
|
||||
const target = tree.getPairOrNextLower(line)?.[1];
|
||||
if (target && target.end >= line) return target;
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Extract tags intelligently from frontmatter. Handles arrays, numbers, and strings. */
|
||||
export function extractTags(metadata: FrontMatterCache): string[] {
|
||||
let tagKeys = Object.keys(metadata).filter(
|
||||
(t) => t.toLowerCase() == "tags" || t.toLowerCase() == "tag"
|
||||
);
|
||||
|
||||
return tagKeys
|
||||
.map((k) => splitFrontmatterTagOrAlias(metadata[k], /[,\s]+/))
|
||||
.reduce((p, c) => p.concat(c), [])
|
||||
.map((str) => (str.startsWith("#") ? str : "#" + str));
|
||||
}
|
||||
|
||||
/** Split a frontmatter list into separate elements; handles actual lists, comma separated lists, and single elements. */
|
||||
export function splitFrontmatterTagOrAlias(data: any, on: RegExp): string[] {
|
||||
if (data == null || data == undefined) return [];
|
||||
if (Array.isArray(data)) {
|
||||
return data
|
||||
.filter((s) => !!s)
|
||||
.map((s) => splitFrontmatterTagOrAlias(s, on))
|
||||
.reduce((p, c) => p.concat(c), []);
|
||||
}
|
||||
|
||||
// Force to a string to handle numbers and so on.
|
||||
return ("" + data)
|
||||
.split(on)
|
||||
.filter((t) => !!t)
|
||||
.map((t) => t.trim())
|
||||
.filter((t) => t.length > 0);
|
||||
}
|
||||
|
||||
///////////////////////
|
||||
// Builder Utilities //
|
||||
///////////////////////
|
||||
|
||||
/** Convienent shared utility for tracking metadata - links, tags, and so on. */
|
||||
export class Metadata {
|
||||
public tags: Set<string> = new Set();
|
||||
public links: Link[] = [];
|
||||
public inlineFields: Record<string, InlineField> = {};
|
||||
|
||||
/** Add a tag to the metadata. */
|
||||
public tag(tag: string) {
|
||||
this.tags.add(tag);
|
||||
}
|
||||
|
||||
/** Add a link to the metadata. */
|
||||
public link(link: Link) {
|
||||
if (this.links.find((v) => v.equals(link))) return;
|
||||
this.links.push(link);
|
||||
}
|
||||
|
||||
/** Add an inline field to the metadata. */
|
||||
public inlineField(field: InlineField) {
|
||||
const lower = field.key.toLowerCase();
|
||||
if (
|
||||
Object.keys(this.inlineFields).some(
|
||||
(key) => key.toLowerCase() == lower
|
||||
)
|
||||
)
|
||||
return;
|
||||
|
||||
this.inlineFields[lower] = field;
|
||||
}
|
||||
|
||||
/** Return a list of unique added tags. */
|
||||
public finishTags(): string[] {
|
||||
return Array.from(this.tags);
|
||||
}
|
||||
|
||||
/** Return a list of JSON-serialized links. */
|
||||
public finishLinks(): JsonLink[] {
|
||||
return this.links.map((link) => link.toObject());
|
||||
}
|
||||
|
||||
/** Return a list of JSON-serialized inline fields. */
|
||||
public finishInlineFields(): Record<string, JsonInlineField> {
|
||||
return mapObjectValues(this.inlineFields, jsonInlineField);
|
||||
}
|
||||
}
|
||||
|
||||
/** Convienent utility for constructing page objects. */
|
||||
export class PageData {
|
||||
public constructor(
|
||||
public path: string,
|
||||
public stats: FileStats,
|
||||
public length: number,
|
||||
public metadata: Metadata,
|
||||
public sections: SectionData[],
|
||||
public frontmatter?: Record<string, JsonFrontmatterEntry>
|
||||
) {}
|
||||
|
||||
public build(): JsonMarkdownPage {
|
||||
return {
|
||||
$path: this.path,
|
||||
$ctime: this.stats.ctime,
|
||||
$mtime: this.stats.mtime,
|
||||
$size: this.stats.size,
|
||||
$extension: getExtension(this.path),
|
||||
$position: { start: 0, end: this.length },
|
||||
$tags: this.metadata.finishTags(),
|
||||
$links: this.metadata.finishLinks(),
|
||||
$infields: this.metadata.finishInlineFields(),
|
||||
$sections: this.sections.map((x) => x.build()),
|
||||
$frontmatter: this.frontmatter,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/** Convienent utility for constructing markdown sections. */
|
||||
export class SectionData {
|
||||
public blocks: BlockData[] = [];
|
||||
public metadata: Metadata = new Metadata();
|
||||
|
||||
public constructor(
|
||||
public start: number,
|
||||
public end: number,
|
||||
public title: string,
|
||||
public level: number,
|
||||
public ordinal: number
|
||||
) {}
|
||||
|
||||
public block(block: BlockData) {
|
||||
this.blocks.push(block);
|
||||
}
|
||||
|
||||
public build(): JsonMarkdownSection {
|
||||
return {
|
||||
$title: this.title,
|
||||
$ordinal: this.ordinal,
|
||||
$level: this.level,
|
||||
$tags: this.metadata.finishTags(),
|
||||
$infields: this.metadata.finishInlineFields(),
|
||||
$links: this.metadata.finishLinks(),
|
||||
$position: { start: this.start, end: this.end },
|
||||
$blocks: this.blocks.map((block) => block.build()),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/** Constructs markdown list blocks specifically. */
|
||||
export class ListBlockData {
|
||||
public type: string = "list";
|
||||
public metadata: Metadata = new Metadata();
|
||||
public items: ListItemData[] = [];
|
||||
|
||||
public constructor(
|
||||
public start: number,
|
||||
public end: number,
|
||||
public ordinal: number,
|
||||
public blockId?: string
|
||||
) {}
|
||||
|
||||
public build(): JsonMarkdownListBlock {
|
||||
return {
|
||||
$ordinal: this.ordinal,
|
||||
$position: { start: this.start, end: this.end },
|
||||
$infields: this.metadata.finishInlineFields(),
|
||||
$tags: this.metadata.finishTags(),
|
||||
$links: this.metadata.finishLinks(),
|
||||
$type: "list",
|
||||
$blockId: this.blockId,
|
||||
$elements: this.items.map((item) => item.build()),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/** Constructs markdown codeblocks specifically. */
|
||||
export class CodeblockData {
|
||||
public type: string = "codeblock";
|
||||
public metadata: Metadata = new Metadata();
|
||||
|
||||
public constructor(
|
||||
public start: number,
|
||||
public end: number,
|
||||
public ordinal: number,
|
||||
public languages: string[],
|
||||
public style: "indent" | "fenced",
|
||||
public contentStart: number,
|
||||
public contentEnd: number,
|
||||
public blockId?: string
|
||||
) {}
|
||||
|
||||
public build(): JsonMarkdownCodeblock {
|
||||
return {
|
||||
$type: "codeblock",
|
||||
$ordinal: this.ordinal,
|
||||
$position: { start: this.start, end: this.end },
|
||||
$infields: this.metadata.finishInlineFields(),
|
||||
$tags: this.metadata.finishTags(),
|
||||
$links: this.metadata.finishLinks(),
|
||||
$blockId: this.blockId,
|
||||
$languages: this.languages,
|
||||
$style: this.style,
|
||||
$contentPosition: {
|
||||
start: this.contentStart,
|
||||
end: this.contentEnd,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/** Constructs markdown datablocks specifically. */
|
||||
export class DatablockData {
|
||||
public type: string = "datablock";
|
||||
public metadata: Metadata = new Metadata();
|
||||
|
||||
public constructor(
|
||||
public start: number,
|
||||
public end: number,
|
||||
public ordinal: number,
|
||||
public data: Record<string, JsonFrontmatterEntry>,
|
||||
public blockId?: string
|
||||
) {}
|
||||
|
||||
public build(): JsonMarkdownDatablock {
|
||||
return {
|
||||
$type: "datablock",
|
||||
$ordinal: this.ordinal,
|
||||
$position: { start: this.start, end: this.end },
|
||||
$infields: this.metadata.finishInlineFields(),
|
||||
$tags: this.metadata.finishTags(),
|
||||
$links: this.metadata.finishLinks(),
|
||||
$blockId: this.blockId,
|
||||
$data: this.data,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/** Base block metadata used for non-specific blocks. */
|
||||
export class BaseBlockData {
|
||||
public metadata: Metadata = new Metadata();
|
||||
|
||||
public constructor(
|
||||
public start: number,
|
||||
public end: number,
|
||||
public ordinal: number,
|
||||
public type: string,
|
||||
public blockId?: string
|
||||
) {}
|
||||
|
||||
public build(): JsonMarkdownBlock {
|
||||
return {
|
||||
$type: this.type,
|
||||
$ordinal: this.ordinal,
|
||||
$position: { start: this.start, end: this.end },
|
||||
$infields: this.metadata.finishInlineFields(),
|
||||
$tags: this.metadata.finishTags(),
|
||||
$links: this.metadata.finishLinks(),
|
||||
$blockId: this.blockId,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export type BlockData =
|
||||
| ListBlockData
|
||||
| CodeblockData
|
||||
| DatablockData
|
||||
| BaseBlockData;
|
||||
|
||||
/** Utility for constructing markdown list items. */
|
||||
export class ListItemData {
|
||||
public metadata: Metadata = new Metadata();
|
||||
public elements: ListItemData[] = [];
|
||||
|
||||
public constructor(
|
||||
public start: number,
|
||||
public end: number,
|
||||
public parentLine: number,
|
||||
public symbol?: string,
|
||||
public blockId?: string,
|
||||
public status?: string,
|
||||
public text?: string
|
||||
) {}
|
||||
|
||||
public build(): JsonMarkdownListItem {
|
||||
return {
|
||||
$parentLine: this.parentLine,
|
||||
$position: { start: this.start, end: this.end },
|
||||
$blockId: this.blockId,
|
||||
$elements: this.elements.map((element) => element.build()),
|
||||
$type: this.status ? "task" : "list",
|
||||
$infields: this.metadata.finishInlineFields(),
|
||||
$tags: this.metadata.finishTags(),
|
||||
$links: this.metadata.finishLinks(),
|
||||
$status: this.status,
|
||||
$symbol: this.symbol,
|
||||
$text: this.text,
|
||||
} as JsonMarkdownTaskItem;
|
||||
}
|
||||
}
|
||||
0
src/utils/readme.md
Normal file
0
src/utils/readme.md
Normal file
169
src/utils/types/TaskIndex.ts
Normal file
169
src/utils/types/TaskIndex.ts
Normal file
|
|
@ -0,0 +1,169 @@
|
|||
/**
|
||||
* Optimized task indexing system focused on task-related data only
|
||||
*/
|
||||
|
||||
import { Component, TFile } from "obsidian";
|
||||
|
||||
/** Core task data structure */
|
||||
export interface Task {
|
||||
/** Unique identifier for the task */
|
||||
id: string;
|
||||
/** Task content text */
|
||||
content: string;
|
||||
/** File path where the task is located */
|
||||
filePath: string;
|
||||
/** Line number in the file */
|
||||
line: number;
|
||||
/** Whether the task is completed or not */
|
||||
completed: boolean;
|
||||
/** Original markdown text */
|
||||
originalMarkdown: string;
|
||||
|
||||
/** Creation date (optional) */
|
||||
createdDate?: number;
|
||||
/** Start date for the task (Tasks plugin compatible) */
|
||||
startDate?: number;
|
||||
/** Scheduled date (Tasks plugin compatible) */
|
||||
scheduledDate?: number;
|
||||
/** Due date for the task */
|
||||
dueDate?: number;
|
||||
/** Date when the task was completed */
|
||||
completedDate?: number;
|
||||
/** Recurrence pattern (Tasks plugin compatible) */
|
||||
recurrence?: string;
|
||||
|
||||
/** Tags associated with the task */
|
||||
tags: string[];
|
||||
/** Project associated with task (derived from frontmatter or special tags) */
|
||||
project?: string;
|
||||
/** Context for the task (e.g. @home, @work) */
|
||||
context?: string;
|
||||
/** Priority level (1-3, higher is more important) */
|
||||
priority?: number;
|
||||
|
||||
/** Parent task ID for hierarchical tasks */
|
||||
parent?: string;
|
||||
/** Child task IDs */
|
||||
children: string[];
|
||||
|
||||
/** Estimated time in minutes */
|
||||
estimatedTime?: number;
|
||||
/** Actual time spent in minutes */
|
||||
actualTime?: number;
|
||||
}
|
||||
|
||||
/** High-performance cache structure for tasks */
|
||||
export interface TaskCache {
|
||||
/** Main task store: taskId -> Task */
|
||||
tasks: Map<string, Task>;
|
||||
|
||||
/** File index: filePath -> Set<taskIds> */
|
||||
files: Map<string, Set<string>>;
|
||||
|
||||
/** Tag index: tag -> Set<taskIds> */
|
||||
tags: Map<string, Set<string>>;
|
||||
|
||||
/** Project index: project -> Set<taskIds> */
|
||||
projects: Map<string, Set<string>>;
|
||||
|
||||
/** Context index: context -> Set<taskIds> */
|
||||
contexts: Map<string, Set<string>>;
|
||||
|
||||
/** Due date index: dueDate(YYYY-MM-DD) -> Set<taskIds> */
|
||||
dueDate: Map<string, Set<string>>;
|
||||
|
||||
/** Start date index: startDate(YYYY-MM-DD) -> Set<taskIds> */
|
||||
startDate: Map<string, Set<string>>;
|
||||
|
||||
/** Scheduled date index: scheduledDate(YYYY-MM-DD) -> Set<taskIds> */
|
||||
scheduledDate: Map<string, Set<string>>;
|
||||
|
||||
/** Completion status index: boolean -> Set<taskIds> */
|
||||
completed: Map<boolean, Set<string>>;
|
||||
|
||||
/** Priority index: priority -> Set<taskIds> */
|
||||
priority: Map<number, Set<string>>;
|
||||
}
|
||||
|
||||
/** Task filter interface for querying tasks */
|
||||
export interface TaskFilter {
|
||||
type:
|
||||
| "tag"
|
||||
| "project"
|
||||
| "context"
|
||||
| "dueDate"
|
||||
| "startDate"
|
||||
| "scheduledDate"
|
||||
| "status"
|
||||
| "priority"
|
||||
| "recurrence";
|
||||
operator:
|
||||
| "="
|
||||
| "!="
|
||||
| "<"
|
||||
| ">"
|
||||
| "contains"
|
||||
| "empty"
|
||||
| "not-empty"
|
||||
| "before"
|
||||
| "after";
|
||||
value: any;
|
||||
conjunction?: "AND" | "OR";
|
||||
}
|
||||
|
||||
/** Sort criteria for task lists */
|
||||
export interface SortingCriteria {
|
||||
field: keyof Task;
|
||||
direction: "asc" | "desc";
|
||||
}
|
||||
|
||||
/** Task parsing configuration */
|
||||
export interface TaskParserConfig {
|
||||
/** Regular expression to match task items */
|
||||
taskRegex: RegExp;
|
||||
/** Start date format for parsing */
|
||||
startDateFormat?: string;
|
||||
/** Due date format for parsing */
|
||||
dueDateFormat?: string;
|
||||
/** Scheduled date format for parsing */
|
||||
scheduledDateFormat?: string;
|
||||
/** Project tag prefix */
|
||||
projectPrefix?: string;
|
||||
/** Context tag prefix */
|
||||
contextPrefix?: string;
|
||||
/** Task priority markers */
|
||||
priorityMarkers?: Record<string, number>;
|
||||
}
|
||||
|
||||
/** Task indexer interface */
|
||||
export interface TaskIndexer extends Component {
|
||||
/** Initialize the task indexer */
|
||||
initialize(): Promise<void>;
|
||||
|
||||
/** Get the current task cache */
|
||||
getCache(): TaskCache;
|
||||
|
||||
/** Index a single file */
|
||||
indexFile(file: TFile): Promise<void>;
|
||||
|
||||
/** Index all files in the vault */
|
||||
indexAllFiles(): Promise<void>;
|
||||
|
||||
/** Update index for a modified file */
|
||||
updateIndex(file: TFile): Promise<void>;
|
||||
|
||||
/** Query tasks based on filters and sorting criteria */
|
||||
queryTasks(filters: TaskFilter[], sortBy: SortingCriteria[]): Task[];
|
||||
|
||||
/** Get task by ID */
|
||||
getTaskById(id: string): Task | undefined;
|
||||
|
||||
/** Create a new task */
|
||||
createTask(task: Partial<Task>): Promise<Task>;
|
||||
|
||||
/** Update an existing task */
|
||||
updateTask(task: Task): Promise<void>;
|
||||
|
||||
/** Delete a task */
|
||||
deleteTask(taskId: string): Promise<void>;
|
||||
}
|
||||
940
src/utils/types/markdown.ts
Normal file
940
src/utils/types/markdown.ts
Normal file
|
|
@ -0,0 +1,940 @@
|
|||
/**
|
||||
* @module indexables
|
||||
*/
|
||||
import { Link, Literal, Literals } from "expression/literal";
|
||||
import { getFileTitle } from "utils/normalizers";
|
||||
import {
|
||||
FILE_TYPE,
|
||||
File,
|
||||
Indexable,
|
||||
LINKABLE_TYPE,
|
||||
LINKBEARING_TYPE,
|
||||
Linkable,
|
||||
Linkbearing,
|
||||
TAGGABLE_TYPE,
|
||||
Taggable,
|
||||
} from "index/types/indexable";
|
||||
import { DateTime } from "luxon";
|
||||
import {
|
||||
Extractors,
|
||||
FIELDBEARING_TYPE,
|
||||
Field,
|
||||
FieldExtractor,
|
||||
Fieldbearing,
|
||||
} from "../../expression/field";
|
||||
import {
|
||||
InlineField,
|
||||
jsonInlineField,
|
||||
valueInlineField,
|
||||
} from "index/import/inline-field";
|
||||
import {
|
||||
LineSpan,
|
||||
JsonMarkdownPage,
|
||||
JsonMarkdownSection,
|
||||
JsonMarkdownBlock,
|
||||
JsonMarkdownListBlock,
|
||||
JsonMarkdownListItem,
|
||||
JsonMarkdownTaskItem,
|
||||
JsonMarkdownDatablock as JsonMarkdownDatablock,
|
||||
JsonMarkdownCodeblock,
|
||||
JsonFrontmatterEntry,
|
||||
} from "./json/markdown";
|
||||
import { mapObjectValues } from "utils/data";
|
||||
import { JsonConversion } from "./json/common";
|
||||
|
||||
/** A markdown file in the vault; the source of most metadata. */
|
||||
export class MarkdownPage
|
||||
implements File, Linkbearing, Taggable, Indexable, Fieldbearing
|
||||
{
|
||||
/** All of the types that a markdown file is. */
|
||||
static TYPES = [
|
||||
FILE_TYPE,
|
||||
"markdown",
|
||||
"page",
|
||||
TAGGABLE_TYPE,
|
||||
LINKABLE_TYPE,
|
||||
LINKBEARING_TYPE,
|
||||
FIELDBEARING_TYPE,
|
||||
];
|
||||
|
||||
// Use static types for all markdown files.
|
||||
$types: string[] = MarkdownPage.TYPES;
|
||||
$typename: string = "Page";
|
||||
|
||||
// Markdown file IDs are always just the full path.
|
||||
get $id() {
|
||||
return this.$path;
|
||||
}
|
||||
// The file of a file is... it's file.
|
||||
get $file() {
|
||||
return this.$path;
|
||||
}
|
||||
|
||||
/** Frontmatter values in the file, if present. Maps lower case frontmatter key -> entry. */
|
||||
$frontmatter?: Record<string, FrontmatterEntry>;
|
||||
/** Map of all distinct inline fields in the document. Maps lower case key name -> full metadata. */
|
||||
$infields: Record<string, InlineField>;
|
||||
|
||||
/** The path this file exists at. */
|
||||
$path: string;
|
||||
/** Obsidian-provided date this page was created. */
|
||||
$ctime: DateTime;
|
||||
/** Obsidian-provided date this page was modified. */
|
||||
$mtime: DateTime;
|
||||
/** The extension; for markdown files, almost always '.md'. */
|
||||
$extension: string;
|
||||
/** Obsidian-provided size of this page in bytes. */
|
||||
$size: number = 0;
|
||||
/** The full extent of the file (start 0, end the number of lines in the file.) */
|
||||
$position: LineSpan;
|
||||
/** The exact tags in the file. */
|
||||
$tags: string[];
|
||||
/** All links in the file. */
|
||||
$links: Link[];
|
||||
/**
|
||||
* All child markdown sections of this markdown file. The initial section before any content is special and is
|
||||
* named with the title of the file.
|
||||
*/
|
||||
$sections: MarkdownSection[] = [];
|
||||
|
||||
/** Create a markdown file from the given raw values. */
|
||||
static from(
|
||||
raw: JsonMarkdownPage,
|
||||
normalizer: LinkNormalizer = NOOP_NORMALIZER
|
||||
): MarkdownPage {
|
||||
const sections = raw.$sections.map((sect) =>
|
||||
MarkdownSection.from(sect, raw.$path, normalizer)
|
||||
);
|
||||
|
||||
return new MarkdownPage({
|
||||
$path: raw.$path,
|
||||
$frontmatter: raw.$frontmatter
|
||||
? mapObjectValues(raw.$frontmatter, (fm) =>
|
||||
normalizeLinks(valueFrontmatterEntry(fm), normalizer)
|
||||
)
|
||||
: undefined,
|
||||
$infields: mapObjectValues(raw.$infields, (field) =>
|
||||
normalizeLinks(valueInlineField(field), normalizer)
|
||||
),
|
||||
$ctime: DateTime.fromMillis(raw.$ctime),
|
||||
$mtime: DateTime.fromMillis(raw.$mtime),
|
||||
$extension: raw.$extension,
|
||||
$size: raw.$size,
|
||||
$position: raw.$position,
|
||||
$tags: raw.$tags,
|
||||
$links: raw.$links.map((link) => normalizer(Link.fromObject(link))),
|
||||
$sections: sections,
|
||||
});
|
||||
}
|
||||
|
||||
private constructor(init: Partial<MarkdownPage>) {
|
||||
Object.assign(this, init);
|
||||
}
|
||||
|
||||
/** Return the number of lines in the document. */
|
||||
get $lineCount() {
|
||||
return this.$position.end;
|
||||
}
|
||||
|
||||
/** The name of the file. */
|
||||
get $name() {
|
||||
return getFileTitle(this.$path);
|
||||
}
|
||||
|
||||
/** A link to this file. */
|
||||
get $link() {
|
||||
return Link.file(this.$path);
|
||||
}
|
||||
|
||||
/** All of the indexed fields in this object. */
|
||||
get fields(): Field[] {
|
||||
return MarkdownPage.FIELD_DEF(this);
|
||||
}
|
||||
|
||||
/** Get the full field definition for the given field. */
|
||||
public field(key: string): Field | undefined {
|
||||
return MarkdownPage.FIELD_DEF(this, key)?.[0];
|
||||
}
|
||||
|
||||
/** Get the value for the given field. */
|
||||
public value(key: string): Literal | undefined {
|
||||
return this.field(key)?.value;
|
||||
}
|
||||
|
||||
/** Convert this page into it's partial representation for saving. */
|
||||
public json(): JsonMarkdownPage {
|
||||
return {
|
||||
$path: this.$path,
|
||||
$frontmatter: this.$frontmatter
|
||||
? mapObjectValues(this.$frontmatter, jsonFrontmatterEntry)
|
||||
: undefined,
|
||||
$infields: mapObjectValues(this.$infields, jsonInlineField),
|
||||
$ctime: this.$ctime.toMillis(),
|
||||
$mtime: this.$mtime.toMillis(),
|
||||
$extension: this.$extension,
|
||||
$size: this.$size,
|
||||
$position: this.$position,
|
||||
$tags: this.$tags,
|
||||
$links: this.$links.map((link) => link.toObject()),
|
||||
$sections: this.$sections.map((sect) => sect.json()),
|
||||
};
|
||||
}
|
||||
|
||||
private static FIELD_DEF: FieldExtractor<MarkdownPage> = Extractors.merge(
|
||||
Extractors.intrinsics(),
|
||||
Extractors.frontmatter((f) => f.$frontmatter),
|
||||
Extractors.inlineFields((f) => f.$infields)
|
||||
);
|
||||
}
|
||||
|
||||
export class MarkdownSection
|
||||
implements Indexable, Taggable, Linkable, Linkbearing, Fieldbearing
|
||||
{
|
||||
/** All of the types that a markdown section is. */
|
||||
static TYPES = [
|
||||
"markdown",
|
||||
"section",
|
||||
TAGGABLE_TYPE,
|
||||
LINKABLE_TYPE,
|
||||
LINKBEARING_TYPE,
|
||||
FIELDBEARING_TYPE,
|
||||
];
|
||||
|
||||
/** Path of the file that this section is in. */
|
||||
$types: string[] = MarkdownSection.TYPES;
|
||||
$typename: string = "Section";
|
||||
$id: string;
|
||||
$file: string;
|
||||
|
||||
/** The index of this section in the file. */
|
||||
$ordinal: number;
|
||||
/** The title of the section; the root (implicit) section will have the title of the page. */
|
||||
$title: string;
|
||||
/** The indentation level of the section (1 - 6). */
|
||||
$level: number;
|
||||
/** The span of lines indicating the position of the section. */
|
||||
$position: LineSpan;
|
||||
/** All tags on the file. */
|
||||
$tags: string[];
|
||||
/** All links in the file. */
|
||||
$links: Link[];
|
||||
/** All of the markdown blocks in this section. */
|
||||
$blocks: MarkdownBlock[];
|
||||
/** Map of all distinct inline fields in the document, from key name -> metadata. */
|
||||
$infields: Record<string, InlineField>;
|
||||
|
||||
/** Convert raw markdown section data to the appropriate class. */
|
||||
static from(
|
||||
raw: JsonMarkdownSection,
|
||||
file: string,
|
||||
normalizer: LinkNormalizer = NOOP_NORMALIZER
|
||||
): MarkdownSection {
|
||||
const blocks = raw.$blocks.map((block) =>
|
||||
MarkdownBlock.from(block, file, normalizer)
|
||||
);
|
||||
return new MarkdownSection({
|
||||
$file: file,
|
||||
$id: MarkdownSection.readableId(file, raw.$title, raw.$ordinal),
|
||||
$ordinal: raw.$ordinal,
|
||||
$title: raw.$title,
|
||||
$level: raw.$level,
|
||||
$position: raw.$position,
|
||||
$tags: raw.$tags,
|
||||
$links: raw.$links.map((l) => normalizer(Link.fromObject(l))),
|
||||
$blocks: blocks,
|
||||
$infields: mapObjectValues(raw.$infields, (i) =>
|
||||
normalizeLinks(valueInlineField(i), normalizer)
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
private constructor(init: Partial<MarkdownSection>) {
|
||||
Object.assign(this, init);
|
||||
}
|
||||
|
||||
/** Obtain the number of lines in the section. */
|
||||
get $lineCount(): number {
|
||||
return this.$position.end - this.$position.start;
|
||||
}
|
||||
|
||||
/** Alias for title which allows searching over pages and sections by 'name'. */
|
||||
get $name(): string {
|
||||
return this.$title;
|
||||
}
|
||||
|
||||
/** Return a link to this section. */
|
||||
get $link(): Link {
|
||||
return Link.header(this.$file, this.$title);
|
||||
}
|
||||
|
||||
/** All of the indexed fields in this object. */
|
||||
get fields(): Field[] {
|
||||
return MarkdownSection.FIELD_DEF(this);
|
||||
}
|
||||
|
||||
/** Fetch a specific field by key. */
|
||||
public field(key: string): Field {
|
||||
return MarkdownSection.FIELD_DEF(this, key)?.[0];
|
||||
}
|
||||
|
||||
public value(key: string): Literal | undefined {
|
||||
return this.field(key)?.value;
|
||||
}
|
||||
|
||||
public json(): JsonMarkdownSection {
|
||||
return {
|
||||
$ordinal: this.$ordinal,
|
||||
$title: this.$title,
|
||||
$level: this.$level,
|
||||
$position: this.$position,
|
||||
$tags: this.$tags,
|
||||
$links: this.$links.map((link) => link.toObject()),
|
||||
$blocks: this.$blocks.map((block) => block.json()),
|
||||
$infields: mapObjectValues(this.$infields, jsonInlineField),
|
||||
};
|
||||
}
|
||||
|
||||
private static FIELD_DEF: FieldExtractor<MarkdownSection> =
|
||||
Extractors.merge(
|
||||
Extractors.intrinsics(),
|
||||
Extractors.inlineFields((f) => f.$infields)
|
||||
);
|
||||
|
||||
/** Generate a readable ID for this section using the first 8 characters of the string and the ordinal. */
|
||||
static readableId(file: string, title: string, ordinal: number): string {
|
||||
const first8 = title
|
||||
.substring(0, Math.min(title.length, 8))
|
||||
.replace(/[^A-Za-z0-9-_]+/gi, "-");
|
||||
|
||||
return `${file}/section${ordinal}/${first8}`;
|
||||
}
|
||||
}
|
||||
|
||||
/** Base class for all markdown blocks. */
|
||||
export class MarkdownBlock
|
||||
implements Indexable, Linkbearing, Taggable, Fieldbearing
|
||||
{
|
||||
static TYPES = [
|
||||
"markdown",
|
||||
"block",
|
||||
LINKBEARING_TYPE,
|
||||
TAGGABLE_TYPE,
|
||||
FIELDBEARING_TYPE,
|
||||
];
|
||||
|
||||
$types: string[] = MarkdownBlock.TYPES;
|
||||
$typename: string = "Block";
|
||||
$id: string;
|
||||
$file: string;
|
||||
|
||||
/** The index of this block in the file. */
|
||||
$ordinal: number;
|
||||
/** The position/extent of the block. */
|
||||
$position: LineSpan;
|
||||
/** All tags on the block. */
|
||||
$tags: string[];
|
||||
/** All links in the file. */
|
||||
$links: Link[];
|
||||
/** Map of all distinct inline fields in the document, from key name -> metadata. */
|
||||
$infields: Record<string, InlineField>;
|
||||
/** If present, the distinct block ID for this block. */
|
||||
$blockId?: string;
|
||||
/** The type of block - paragraph, list, and so on. */
|
||||
$type: string;
|
||||
|
||||
static from(
|
||||
object: JsonMarkdownBlock,
|
||||
file: string,
|
||||
normalizer: LinkNormalizer = NOOP_NORMALIZER
|
||||
): MarkdownBlock {
|
||||
if (object.$type === "list") {
|
||||
return MarkdownListBlock.from(
|
||||
object as JsonMarkdownListBlock,
|
||||
file,
|
||||
normalizer
|
||||
);
|
||||
} else if (object.$type === "datablock") {
|
||||
return MarkdownDatablock.from(
|
||||
object as JsonMarkdownDatablock,
|
||||
file,
|
||||
normalizer
|
||||
);
|
||||
} else if (object.$type === "codeblock") {
|
||||
return MarkdownCodeblock.from(
|
||||
object as JsonMarkdownCodeblock,
|
||||
file,
|
||||
normalizer
|
||||
);
|
||||
}
|
||||
|
||||
return new MarkdownBlock({
|
||||
$file: file,
|
||||
$id: MarkdownBlock.readableId(file, object.$ordinal),
|
||||
$ordinal: object.$ordinal,
|
||||
$position: object.$position,
|
||||
$tags: object.$tags,
|
||||
$links: object.$links.map((l) => normalizer(Link.fromObject(l))),
|
||||
$infields: mapObjectValues(object.$infields, (i) =>
|
||||
normalizeLinks(valueInlineField(i), normalizer)
|
||||
),
|
||||
$blockId: object.$blockId,
|
||||
$type: object.$type,
|
||||
});
|
||||
}
|
||||
|
||||
protected constructor(init: Partial<MarkdownBlock>) {
|
||||
Object.assign(this, init);
|
||||
}
|
||||
|
||||
/** If this block has a block ID, the link to this block. */
|
||||
get $link(): Link | undefined {
|
||||
if (this.$blockId) return Link.block(this.$file, this.$blockId);
|
||||
else return undefined;
|
||||
}
|
||||
|
||||
/** All of the indexed fields in this object. */
|
||||
get fields() {
|
||||
return MarkdownBlock.FIELD_DEF(this);
|
||||
}
|
||||
|
||||
/** Fetch a specific field by key. */
|
||||
public field(key: string) {
|
||||
return MarkdownBlock.FIELD_DEF(this, key)?.[0];
|
||||
}
|
||||
|
||||
public value(key: string): Literal | undefined {
|
||||
return this.field(key)?.value;
|
||||
}
|
||||
|
||||
public json(): JsonMarkdownBlock {
|
||||
return {
|
||||
$ordinal: this.$ordinal,
|
||||
$position: this.$position,
|
||||
$tags: this.$tags,
|
||||
$links: this.$links.map((l) => l.toObject()),
|
||||
$infields: mapObjectValues(this.$infields, jsonInlineField),
|
||||
$blockId: this.$blockId,
|
||||
$type: this.$type,
|
||||
};
|
||||
}
|
||||
|
||||
static FIELD_DEF: FieldExtractor<MarkdownBlock> = Extractors.merge(
|
||||
Extractors.intrinsics(),
|
||||
Extractors.inlineFields((f) => f.$infields)
|
||||
);
|
||||
|
||||
/** Generate a readable ID for this block using the ordinal of the block. */
|
||||
static readableId(file: string, ordinal: number): string {
|
||||
return `${file}/block${ordinal}`;
|
||||
}
|
||||
}
|
||||
|
||||
/** Special block for markdown lists (of either plain list entries or tasks). */
|
||||
export class MarkdownListBlock
|
||||
extends MarkdownBlock
|
||||
implements Taggable, Linkbearing
|
||||
{
|
||||
static TYPES = [
|
||||
"markdown",
|
||||
"block",
|
||||
"block-list",
|
||||
TAGGABLE_TYPE,
|
||||
LINKBEARING_TYPE,
|
||||
];
|
||||
|
||||
$types: string[] = MarkdownListBlock.TYPES;
|
||||
$typename: string = "List Block";
|
||||
|
||||
/** The list items inside of this block. */
|
||||
$elements: MarkdownListItem[];
|
||||
|
||||
/** Create a list block from a serialized value. */
|
||||
static from(
|
||||
object: JsonMarkdownListBlock,
|
||||
file: string,
|
||||
normalizer: LinkNormalizer = NOOP_NORMALIZER
|
||||
): MarkdownListBlock {
|
||||
const elements = object.$elements.map((elem) =>
|
||||
MarkdownListItem.from(elem, file, normalizer)
|
||||
);
|
||||
return new MarkdownListBlock({
|
||||
// TODO: This is shared with other blocks, should probably be fixed.
|
||||
$file: file,
|
||||
$id: MarkdownBlock.readableId(file, object.$ordinal),
|
||||
$ordinal: object.$ordinal,
|
||||
$position: object.$position,
|
||||
$tags: object.$tags,
|
||||
$links: object.$links.map((l) => normalizer(Link.fromObject(l))),
|
||||
$infields: mapObjectValues(object.$infields, (i) =>
|
||||
normalizeLinks(valueInlineField(i), normalizer)
|
||||
),
|
||||
$blockId: object.$blockId,
|
||||
$elements: elements,
|
||||
$type: "list",
|
||||
});
|
||||
}
|
||||
|
||||
public json(): JsonMarkdownListBlock {
|
||||
return Object.assign(super.json(), {
|
||||
$elements: this.$elements.map((elem) => elem.json()),
|
||||
}) as JsonMarkdownListBlock;
|
||||
}
|
||||
|
||||
public constructor(init: Partial<MarkdownListBlock>) {
|
||||
super(init);
|
||||
}
|
||||
}
|
||||
|
||||
/** A block containing markdown code. */
|
||||
export class MarkdownCodeblock
|
||||
extends MarkdownBlock
|
||||
implements Indexable, Fieldbearing, Linkbearing
|
||||
{
|
||||
static TYPES = [
|
||||
"markdown",
|
||||
"block",
|
||||
"codeblock",
|
||||
TAGGABLE_TYPE,
|
||||
LINKBEARING_TYPE,
|
||||
FIELDBEARING_TYPE,
|
||||
];
|
||||
|
||||
$types: string[] = MarkdownCodeblock.TYPES;
|
||||
$languages: string[];
|
||||
$contentPosition: { start: number; end: number };
|
||||
$style: "fenced" | "indent";
|
||||
|
||||
public constructor(init: Partial<MarkdownCodeblock>) {
|
||||
super(init);
|
||||
}
|
||||
|
||||
static from(
|
||||
object: JsonMarkdownCodeblock,
|
||||
file: string,
|
||||
normalizer: LinkNormalizer = NOOP_NORMALIZER
|
||||
): MarkdownCodeblock {
|
||||
return new MarkdownCodeblock({
|
||||
$file: file,
|
||||
$id: MarkdownCodeblock.readableId(file, object.$position.start),
|
||||
$position: object.$position,
|
||||
$ordinal: object.$ordinal,
|
||||
$typename: "Codeblock",
|
||||
$type: "codeblock",
|
||||
$blockId: object.$blockId,
|
||||
$languages: object.$languages,
|
||||
$links: object.$links.map((link) =>
|
||||
normalizer(Link.fromObject(link))
|
||||
),
|
||||
$tags: object.$tags,
|
||||
$infields: mapObjectValues(object.$infields, valueInlineField),
|
||||
$contentPosition: object.$contentPosition,
|
||||
$style: object.$style,
|
||||
});
|
||||
}
|
||||
|
||||
/** All of the indexed fields in this object. */
|
||||
get fields() {
|
||||
return MarkdownCodeblock.SUB_FIELD_DEF(this);
|
||||
}
|
||||
|
||||
/** Fetch a specific field by key. */
|
||||
public field(key: string) {
|
||||
return MarkdownCodeblock.SUB_FIELD_DEF(this, key)?.[0];
|
||||
}
|
||||
|
||||
public value(key: string): Literal | undefined {
|
||||
return this.field(key)?.value;
|
||||
}
|
||||
|
||||
public json(): JsonMarkdownCodeblock {
|
||||
return Object.assign(super.json(), {
|
||||
$languages: this.$languages,
|
||||
$contentPosition: this.$contentPosition,
|
||||
$style: this.$style,
|
||||
}) as JsonMarkdownCodeblock;
|
||||
}
|
||||
|
||||
static readableId(file: string, line: number): string {
|
||||
return `${file}/codeblock${line}`;
|
||||
}
|
||||
|
||||
static SUB_FIELD_DEF: FieldExtractor<MarkdownCodeblock> =
|
||||
Extractors.merge<MarkdownCodeblock>(MarkdownBlock.FIELD_DEF);
|
||||
}
|
||||
|
||||
/** A data-annotated YAML codeblock. */
|
||||
export class MarkdownDatablock
|
||||
extends MarkdownBlock
|
||||
implements Indexable, Fieldbearing, Linkbearing
|
||||
{
|
||||
static TYPES = [
|
||||
"markdown",
|
||||
"block",
|
||||
"datablock",
|
||||
TAGGABLE_TYPE,
|
||||
LINKBEARING_TYPE,
|
||||
FIELDBEARING_TYPE,
|
||||
];
|
||||
|
||||
$types: string[] = MarkdownDatablock.TYPES;
|
||||
$data: Record<string, FrontmatterEntry>;
|
||||
|
||||
public constructor(init: Partial<MarkdownDatablock>) {
|
||||
super(init);
|
||||
}
|
||||
|
||||
static from(
|
||||
object: JsonMarkdownDatablock,
|
||||
file: string,
|
||||
normalizer: LinkNormalizer = NOOP_NORMALIZER
|
||||
): MarkdownDatablock {
|
||||
// Datablocks are based on what is essentially just frontmatter; we can apply
|
||||
// the same normalization logic to them.
|
||||
const normdata = normalizeLinks(
|
||||
mapObjectValues(object.$data, valueFrontmatterEntry),
|
||||
normalizer
|
||||
);
|
||||
const links = gatherLinks(normdata);
|
||||
const tags = gatherTags(normdata);
|
||||
|
||||
return new MarkdownDatablock({
|
||||
$file: file,
|
||||
$id: MarkdownDatablock.readableId(file, object.$position.start),
|
||||
$position: object.$position,
|
||||
$infields: {},
|
||||
$ordinal: object.$ordinal,
|
||||
$data: normdata,
|
||||
$links: links,
|
||||
$typename: "Datablock",
|
||||
$tags: tags,
|
||||
$type: "datablock",
|
||||
$blockId: object.$blockId,
|
||||
});
|
||||
}
|
||||
|
||||
/** All of the indexed fields in this object. */
|
||||
get fields() {
|
||||
return MarkdownDatablock.SUB_FIELD_DEF(this);
|
||||
}
|
||||
|
||||
/** Fetch a specific field by key. */
|
||||
public field(key: string) {
|
||||
return MarkdownDatablock.SUB_FIELD_DEF(this, key)?.[0];
|
||||
}
|
||||
|
||||
public value(key: string): Literal | undefined {
|
||||
return this.field(key)?.value;
|
||||
}
|
||||
|
||||
public json(): JsonMarkdownDatablock {
|
||||
return Object.assign(super.json(), {
|
||||
$data: mapObjectValues(this.$data, jsonFrontmatterEntry),
|
||||
}) as JsonMarkdownDatablock;
|
||||
}
|
||||
|
||||
static readableId(file: string, line: number): string {
|
||||
return `${file}/datablock${line}`;
|
||||
}
|
||||
|
||||
static SUB_FIELD_DEF: FieldExtractor<MarkdownDatablock> =
|
||||
Extractors.merge<MarkdownDatablock>(
|
||||
MarkdownBlock.FIELD_DEF,
|
||||
Extractors.frontmatter((f) => f.$data)
|
||||
);
|
||||
}
|
||||
|
||||
/** A specific list item in a list. */
|
||||
export class MarkdownListItem
|
||||
implements Indexable, Linkbearing, Taggable, Fieldbearing
|
||||
{
|
||||
static TYPES = [
|
||||
"markdown",
|
||||
"list-item",
|
||||
LINKBEARING_TYPE,
|
||||
TAGGABLE_TYPE,
|
||||
FIELDBEARING_TYPE,
|
||||
];
|
||||
|
||||
$types: string[] = MarkdownListItem.TYPES;
|
||||
$typename: string = "List Item";
|
||||
$id: string;
|
||||
$file: string;
|
||||
|
||||
/** The position of the list item in the file. */
|
||||
$position: LineSpan;
|
||||
/** Child elements of this list item. */
|
||||
$elements: MarkdownListItem[];
|
||||
/** The type of list item that this element is. */
|
||||
$type: string;
|
||||
/** Exact tags on this list item. */
|
||||
$tags: string[];
|
||||
/** Map of all distinct inline fields in the document, from key name -> metadata. */
|
||||
$infields: Record<string, InlineField>;
|
||||
/** All links in the file. */
|
||||
$links: Link[];
|
||||
/** The block ID of this list item if present. */
|
||||
$blockId?: string;
|
||||
/**
|
||||
* The line number of the parent of this list item.
|
||||
* If a positive number, then this list element is a child
|
||||
* of the list element at that line.
|
||||
*
|
||||
* If a negative number, then this list element is a root element
|
||||
* of a list starting at that line (negated). I.e., -7 means
|
||||
* this is a root element of the list starting at line 7.
|
||||
*/
|
||||
$parentLine: number;
|
||||
/** The marker used to start the list item (such as - or + or *). On a malformed task, may be undefined. */
|
||||
$symbol?: string;
|
||||
/** The text contents of the list item. */
|
||||
$text?: string;
|
||||
|
||||
/** Create a list item from a serialized object. */
|
||||
static from(
|
||||
object: JsonMarkdownListItem,
|
||||
file: string,
|
||||
normalizer: LinkNormalizer = NOOP_NORMALIZER
|
||||
): MarkdownListItem {
|
||||
if (object.$type === "task")
|
||||
return MarkdownTaskItem.from(
|
||||
object as JsonMarkdownTaskItem,
|
||||
file,
|
||||
normalizer
|
||||
);
|
||||
|
||||
const elements = object.$elements.map((elem) =>
|
||||
MarkdownListItem.from(elem, file, normalizer)
|
||||
);
|
||||
return new MarkdownListItem({
|
||||
$file: file,
|
||||
$id: MarkdownListItem.readableId(file, object.$position.start),
|
||||
$position: object.$position,
|
||||
$elements: elements,
|
||||
$type: object.$type,
|
||||
$tags: object.$tags,
|
||||
$infields: mapObjectValues(object.$infields, (i) =>
|
||||
normalizeLinks(valueInlineField(i), normalizer)
|
||||
),
|
||||
$links: object.$links.map((l) => normalizer(Link.fromObject(l))),
|
||||
$blockId: object.$blockId,
|
||||
$parentLine: object.$parentLine,
|
||||
$text: object.$text,
|
||||
$symbol: object.$symbol,
|
||||
});
|
||||
}
|
||||
|
||||
protected constructor(init: Partial<MarkdownListItem>) {
|
||||
Object.assign(this, init);
|
||||
}
|
||||
|
||||
/** Get the line that this list item starts on. */
|
||||
get $line(): number {
|
||||
return this.$position.start;
|
||||
}
|
||||
|
||||
/** The number of lines in this list item. */
|
||||
get $lineCount(): number {
|
||||
return this.$position.end - this.$position.start + 1;
|
||||
}
|
||||
|
||||
/** Cleaned text that is garaunteed to be non-null and has indenation and inline fields removed. */
|
||||
get $cleantext() {
|
||||
if (!this.$text) return "";
|
||||
|
||||
return (
|
||||
this.$text
|
||||
// Eliminate [key:: value] annotations.
|
||||
.replace(/(.*?)([\[\(][^:(\[]+::\s*.*?[\]\)]\s*)$/gm, "$1")
|
||||
// Trim whitespace.
|
||||
.trim()
|
||||
);
|
||||
}
|
||||
|
||||
/** All of the indexed fields in this object. */
|
||||
get fields() {
|
||||
return MarkdownListItem.FIELD_DEF(this);
|
||||
}
|
||||
|
||||
/** Fetch a specific field by key. */
|
||||
public field(key: string) {
|
||||
return MarkdownListItem.FIELD_DEF(this, key)?.[0];
|
||||
}
|
||||
|
||||
public value(key: string): Literal | undefined {
|
||||
return this.field(key)?.value;
|
||||
}
|
||||
|
||||
public json(): JsonMarkdownListItem {
|
||||
return {
|
||||
$position: this.$position,
|
||||
$elements: this.$elements.map((elem) => elem.json()),
|
||||
$type: this.$type,
|
||||
$tags: this.$tags,
|
||||
$infields: mapObjectValues(this.$infields, jsonInlineField),
|
||||
$links: this.$links,
|
||||
$blockId: this.$blockId,
|
||||
$parentLine: this.$parentLine,
|
||||
$symbol: this.$symbol,
|
||||
$text: this.$text,
|
||||
};
|
||||
}
|
||||
|
||||
private static FIELD_DEF: FieldExtractor<MarkdownListItem> =
|
||||
Extractors.merge(
|
||||
Extractors.intrinsics(),
|
||||
Extractors.inlineFields((f) => f.$infields)
|
||||
);
|
||||
|
||||
/** Generate a readable ID for this item using the line number. */
|
||||
static readableId(file: string, line: number): string {
|
||||
return `${file}/list${line}`;
|
||||
}
|
||||
}
|
||||
|
||||
/** A specific task inside of a markdown list. */
|
||||
export class MarkdownTaskItem
|
||||
extends MarkdownListItem
|
||||
implements Indexable, Linkbearing, Taggable, Fieldbearing
|
||||
{
|
||||
static TYPES = [
|
||||
"markdown",
|
||||
"list-item",
|
||||
"task",
|
||||
LINKBEARING_TYPE,
|
||||
TAGGABLE_TYPE,
|
||||
FIELDBEARING_TYPE,
|
||||
];
|
||||
|
||||
$types: string[] = MarkdownTaskItem.TYPES;
|
||||
$typename: string = "Task";
|
||||
|
||||
/** The text inside of the task item. */
|
||||
$status: string;
|
||||
|
||||
public static from(
|
||||
object: JsonMarkdownTaskItem,
|
||||
file: string,
|
||||
normalizer: LinkNormalizer
|
||||
): MarkdownTaskItem {
|
||||
const elements = object.$elements.map((elem) =>
|
||||
MarkdownListItem.from(elem, file, normalizer)
|
||||
);
|
||||
return new MarkdownTaskItem({
|
||||
$file: file,
|
||||
$id: MarkdownListItem.readableId(file, object.$position.start),
|
||||
$position: object.$position,
|
||||
$elements: elements,
|
||||
$type: object.$type,
|
||||
$tags: object.$tags,
|
||||
$infields: mapObjectValues(object.$infields, (i) =>
|
||||
normalizeLinks(valueInlineField(i), normalizer)
|
||||
),
|
||||
$links: object.$links.map((l) => normalizer(Link.fromObject(l))),
|
||||
$blockId: object.$blockId,
|
||||
$parentLine: object.$parentLine,
|
||||
$status: object.$status,
|
||||
$symbol: object.$symbol,
|
||||
$text: object.$text,
|
||||
});
|
||||
}
|
||||
|
||||
public constructor(init: Partial<MarkdownTaskItem>) {
|
||||
super(init);
|
||||
}
|
||||
|
||||
public json(): JsonMarkdownListItem {
|
||||
return Object.assign(super.json(), {
|
||||
$status: this.$status,
|
||||
});
|
||||
}
|
||||
|
||||
/** Determine if the given task is completed. */
|
||||
public get $completed() {
|
||||
return this.$status === "x" || this.$status === "X";
|
||||
}
|
||||
}
|
||||
|
||||
/** An entry in the frontmatter; includes the raw value, parsed value, and raw key (before lower-casing). */
|
||||
export interface FrontmatterEntry {
|
||||
/** The actual string in frontmatter with exact casing. */
|
||||
key: string;
|
||||
/** The parsed value of the frontmatter entry (date, duration, etc.). */
|
||||
value: Literal;
|
||||
/** The raw value of the frontmatter entry before parsing; generally a string or number. */
|
||||
raw: string;
|
||||
}
|
||||
|
||||
/** Convert a regular frontmatter entry into a JSON frontmatter entry.
|
||||
* @hidden
|
||||
*/
|
||||
export function jsonFrontmatterEntry(
|
||||
raw: FrontmatterEntry
|
||||
): JsonFrontmatterEntry {
|
||||
return {
|
||||
key: raw.key,
|
||||
value: JsonConversion.json(raw.value),
|
||||
raw: raw.raw,
|
||||
};
|
||||
}
|
||||
|
||||
/** Convert a json frontmatter entry to a regular frontmatter entry.
|
||||
* @hidden
|
||||
*/
|
||||
export function valueFrontmatterEntry(
|
||||
raw: JsonFrontmatterEntry
|
||||
): FrontmatterEntry {
|
||||
return {
|
||||
key: raw.key,
|
||||
value: JsonConversion.value(raw.value),
|
||||
raw: raw.raw,
|
||||
};
|
||||
}
|
||||
|
||||
/** Normalize links deeply in the object.
|
||||
* @hidden
|
||||
*/
|
||||
export function normalizeLinks<T extends Literal>(
|
||||
input: T,
|
||||
normalizer: LinkNormalizer
|
||||
): T {
|
||||
return Literals.mapLeaves(input, (value) => {
|
||||
if (Literals.isLink(value)) return normalizer(value);
|
||||
else return value;
|
||||
}) as T;
|
||||
}
|
||||
|
||||
/** Recursively gather links from a literal object.
|
||||
* @hidden
|
||||
*/
|
||||
export function gatherLinks(input: Literal): Link[] {
|
||||
const result: Link[] = [];
|
||||
|
||||
Literals.mapLeaves(input, (value) => {
|
||||
if (Literals.isLink(value)) result.push(value);
|
||||
return null;
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Gather tags from a datablock.
|
||||
* @hidden
|
||||
*/
|
||||
export function gatherTags(data: Record<string, FrontmatterEntry>): string[] {
|
||||
function recurse(input: any): string[] {
|
||||
if (Literals.isString(input))
|
||||
return [input.startsWith("#") ? input : "#" + input];
|
||||
else if (Literals.isArray(input)) return input.flatMap(recurse);
|
||||
else return [];
|
||||
}
|
||||
|
||||
let tags: string[] = [];
|
||||
if ("tag" in data) tags = tags.concat(recurse(data["tags"]));
|
||||
if ("tags" in data) tags = tags.concat(recurse(data["tags"]));
|
||||
|
||||
return tags;
|
||||
}
|
||||
|
||||
/** A link normalizer which takes in a raw link and produces a normalized link. */
|
||||
export type LinkNormalizer = (link: Link) => Link;
|
||||
export const NOOP_NORMALIZER: LinkNormalizer = (x) => x;
|
||||
337
src/utils/workers/TaskIndexWorker.ts
Normal file
337
src/utils/workers/TaskIndexWorker.ts
Normal file
|
|
@ -0,0 +1,337 @@
|
|||
/**
|
||||
* Web worker for background processing of task indexing
|
||||
*/
|
||||
|
||||
import { FileStats } from "obsidian";
|
||||
import { Task } from "../types/TaskIndex";
|
||||
|
||||
/** Command to parse task from a file */
|
||||
export interface ParseTasksCommand {
|
||||
type: "parseTasks";
|
||||
filePath: string;
|
||||
content: string;
|
||||
stats: FileStats;
|
||||
}
|
||||
|
||||
/** Command to batch index multiple files */
|
||||
export interface BatchIndexCommand {
|
||||
type: "batchIndex";
|
||||
files: {
|
||||
path: string;
|
||||
content: string;
|
||||
stats: FileStats;
|
||||
}[];
|
||||
}
|
||||
|
||||
/** Available commands that can be sent to the worker */
|
||||
export type IndexerCommand = ParseTasksCommand | BatchIndexCommand;
|
||||
|
||||
/** Result of task parsing */
|
||||
export interface TaskParseResult {
|
||||
type: "parseResult";
|
||||
filePath: string;
|
||||
tasks: Task[];
|
||||
stats: {
|
||||
totalTasks: number;
|
||||
completedTasks: number;
|
||||
processingTimeMs: number;
|
||||
};
|
||||
}
|
||||
|
||||
/** Result of batch indexing */
|
||||
export interface BatchIndexResult {
|
||||
type: "batchResult";
|
||||
results: {
|
||||
filePath: string;
|
||||
taskCount: number;
|
||||
}[];
|
||||
stats: {
|
||||
totalFiles: number;
|
||||
totalTasks: number;
|
||||
processingTimeMs: number;
|
||||
};
|
||||
}
|
||||
|
||||
/** Result of a worker operation or error */
|
||||
export type IndexerResult = TaskParseResult | BatchIndexResult | ErrorResult;
|
||||
|
||||
/** Error response */
|
||||
export interface ErrorResult {
|
||||
type: "error";
|
||||
error: string;
|
||||
filePath?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Regular expressions for parsing task components
|
||||
*/
|
||||
const TASK_REGEX = /^([\s>]*- \[(.)\])\s*(.*)$/m;
|
||||
const START_DATE_REGEX = /📅 (\d{4}-\d{2}-\d{2})/;
|
||||
const COMPLETED_DATE_REGEX = /✅ (\d{4}-\d{2}-\d{2})/;
|
||||
const DUE_DATE_REGEX = /⏳ (\d{4}-\d{2}-\d{2})/;
|
||||
const SCHEDULED_DATE_REGEX = /⏰ (\d{4}-\d{2}-\d{2})/;
|
||||
const RECURRENCE_REGEX = /🔁 (.*?)(?=\s|$)/;
|
||||
const TAG_REGEX = /#[\w\/-]+/g;
|
||||
const CONTEXT_REGEX = /@[\w-]+/g;
|
||||
const PRIORITY_REGEX = /🔼|⏫|🔽|⏬️|🔺|\[#[A-C]\]/;
|
||||
const PRIORITY_MAP: Record<string, number> = {
|
||||
"⏫": 3, // High
|
||||
"🔼": 2, // Medium
|
||||
"🔽": 1, // Low
|
||||
"⏬️": 1, // Lowest
|
||||
"🔺": 5, // Highest
|
||||
"[#A]": 4, // High (letter format)
|
||||
"[#B]": 3, // Medium (letter format)
|
||||
"[#C]": 2, // Low (letter format)
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse tasks from file content
|
||||
*/
|
||||
function parseTasksFromContent(filePath: string, content: string): Task[] {
|
||||
const lines = content.split(/\r?\n/);
|
||||
const tasks: Task[] = [];
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
const taskMatch = line.match(TASK_REGEX);
|
||||
|
||||
if (taskMatch) {
|
||||
const [, prefix, status, content] = taskMatch;
|
||||
const completed = status.toLowerCase() === "x";
|
||||
|
||||
// Generate a deterministic ID based on file path and line number
|
||||
// This helps with task tracking across worker calls
|
||||
const id = `${filePath}-L${i}`;
|
||||
|
||||
// Basic task info
|
||||
const task: Task = {
|
||||
id,
|
||||
content: content.trim(),
|
||||
filePath,
|
||||
line: i,
|
||||
completed,
|
||||
originalMarkdown: line,
|
||||
tags: [],
|
||||
children: [],
|
||||
};
|
||||
|
||||
// Extract metadata
|
||||
extractDates(task, content);
|
||||
extractTags(task, content);
|
||||
extractContext(task, content);
|
||||
extractPriority(task, content);
|
||||
|
||||
tasks.push(task);
|
||||
}
|
||||
}
|
||||
|
||||
// Build parent-child relationships
|
||||
buildTaskHierarchy(tasks);
|
||||
|
||||
return tasks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract dates from task content
|
||||
*/
|
||||
function extractDates(task: Task, content: string): void {
|
||||
// Start date
|
||||
const startDateMatch = content.match(START_DATE_REGEX);
|
||||
if (startDateMatch) {
|
||||
task.startDate = new Date(startDateMatch[1]).getTime();
|
||||
}
|
||||
|
||||
// Due date
|
||||
const dueDateMatch = content.match(DUE_DATE_REGEX);
|
||||
if (dueDateMatch) {
|
||||
task.dueDate = new Date(dueDateMatch[1]).getTime();
|
||||
}
|
||||
|
||||
// Scheduled date
|
||||
const scheduledDateMatch = content.match(SCHEDULED_DATE_REGEX);
|
||||
if (scheduledDateMatch) {
|
||||
task.scheduledDate = new Date(scheduledDateMatch[1]).getTime();
|
||||
}
|
||||
|
||||
// Completion date
|
||||
const completedDateMatch = content.match(COMPLETED_DATE_REGEX);
|
||||
if (completedDateMatch) {
|
||||
task.completedDate = new Date(completedDateMatch[1]).getTime();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract tags from task content
|
||||
*/
|
||||
function extractTags(task: Task, content: string): void {
|
||||
const tagMatches = content.match(TAG_REGEX) || [];
|
||||
task.tags = tagMatches.map((tag) => tag.trim());
|
||||
|
||||
// Check for project tags
|
||||
const projectTag = task.tags.find((tag) => tag.startsWith("#project/"));
|
||||
if (projectTag) {
|
||||
task.project = projectTag.substring("#project/".length);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract context from task content
|
||||
*/
|
||||
function extractContext(task: Task, content: string): void {
|
||||
const contextMatches = content.match(CONTEXT_REGEX) || [];
|
||||
if (contextMatches.length > 0) {
|
||||
// Use the first context tag as the primary context
|
||||
task.context = contextMatches[0]?.substring(1); // Remove the @ symbol
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract priority from task content
|
||||
*/
|
||||
function extractPriority(task: Task, content: string): void {
|
||||
const priorityMatch = content.match(PRIORITY_REGEX);
|
||||
if (priorityMatch) {
|
||||
task.priority = PRIORITY_MAP[priorityMatch[0]] || undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build parent-child relationships between tasks
|
||||
*/
|
||||
function buildTaskHierarchy(tasks: Task[]): void {
|
||||
// Sort tasks by line number
|
||||
tasks.sort((a, b) => a.line - b.line);
|
||||
|
||||
// Build parent-child relationships based on indentation
|
||||
for (let i = 0; i < tasks.length; i++) {
|
||||
const currentTask = tasks[i];
|
||||
const currentIndent = getIndentLevel(currentTask.originalMarkdown);
|
||||
|
||||
// Look for potential parent tasks (must be before current task and have less indentation)
|
||||
for (let j = i - 1; j >= 0; j--) {
|
||||
const potentialParent = tasks[j];
|
||||
const parentIndent = getIndentLevel(
|
||||
potentialParent.originalMarkdown
|
||||
);
|
||||
|
||||
if (parentIndent < currentIndent) {
|
||||
// Found a parent
|
||||
currentTask.parent = potentialParent.id;
|
||||
potentialParent.children.push(currentTask.id);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get indentation level of a line
|
||||
*/
|
||||
function getIndentLevel(line: string): number {
|
||||
const match = line.match(/^(\s*)/);
|
||||
return match ? match[1].length : 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single file
|
||||
*/
|
||||
function processFile(
|
||||
filePath: string,
|
||||
content: string,
|
||||
stats: FileStats
|
||||
): TaskParseResult {
|
||||
const startTime = performance.now();
|
||||
|
||||
try {
|
||||
const tasks = parseTasksFromContent(filePath, content);
|
||||
const completedTasks = tasks.filter((t) => t.completed).length;
|
||||
|
||||
return {
|
||||
type: "parseResult",
|
||||
filePath,
|
||||
tasks,
|
||||
stats: {
|
||||
totalTasks: tasks.length,
|
||||
completedTasks,
|
||||
processingTimeMs: Math.round(performance.now() - startTime),
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
console.error(`Error processing file ${filePath}:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process multiple files in batch
|
||||
*/
|
||||
function processBatch(
|
||||
files: { path: string; content: string; stats: FileStats }[]
|
||||
): BatchIndexResult {
|
||||
const startTime = performance.now();
|
||||
const results: { filePath: string; taskCount: number }[] = [];
|
||||
let totalTasks = 0;
|
||||
|
||||
for (const file of files) {
|
||||
try {
|
||||
const parseResult = processFile(
|
||||
file.path,
|
||||
file.content,
|
||||
file.stats
|
||||
);
|
||||
totalTasks += parseResult.stats.totalTasks;
|
||||
results.push({
|
||||
filePath: file.path,
|
||||
taskCount: parseResult.stats.totalTasks,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`Error in batch processing for file ${file.path}:`,
|
||||
error
|
||||
);
|
||||
// Continue with other files even if one fails
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
type: "batchResult",
|
||||
results,
|
||||
stats: {
|
||||
totalFiles: files.length,
|
||||
totalTasks,
|
||||
processingTimeMs: Math.round(performance.now() - startTime),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Web worker message handler
|
||||
*/
|
||||
self.onmessage = async (event) => {
|
||||
try {
|
||||
const message = event.data as IndexerCommand;
|
||||
|
||||
if (message.type === "parseTasks") {
|
||||
const result = processFile(
|
||||
message.filePath,
|
||||
message.content,
|
||||
message.stats
|
||||
);
|
||||
self.postMessage(result);
|
||||
} else if (message.type === "batchIndex") {
|
||||
const result = processBatch(message.files);
|
||||
self.postMessage(result);
|
||||
} else {
|
||||
self.postMessage({
|
||||
type: "error",
|
||||
error: `Unknown command type: ${(message as any).type}`,
|
||||
} as ErrorResult);
|
||||
}
|
||||
} catch (error) {
|
||||
self.postMessage({
|
||||
type: "error",
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
} as ErrorResult);
|
||||
}
|
||||
};
|
||||
457
src/utils/workers/TaskWorkerManager.ts
Normal file
457
src/utils/workers/TaskWorkerManager.ts
Normal file
|
|
@ -0,0 +1,457 @@
|
|||
/**
|
||||
* Manager for task indexing web workers
|
||||
*/
|
||||
|
||||
import { TFile, Vault } from "obsidian";
|
||||
import { Task } from "../types/TaskIndex";
|
||||
import {
|
||||
BatchIndexCommand,
|
||||
BatchIndexResult,
|
||||
ErrorResult,
|
||||
IndexerCommand,
|
||||
IndexerResult,
|
||||
ParseTasksCommand,
|
||||
TaskParseResult,
|
||||
} from "./TaskIndexWorker";
|
||||
|
||||
// Import worker
|
||||
import TaskWorker from "./TaskIndexWorker";
|
||||
|
||||
/**
|
||||
* Options for worker pool
|
||||
*/
|
||||
export interface WorkerPoolOptions {
|
||||
/** Maximum number of workers to use */
|
||||
maxWorkers: number;
|
||||
/** Target CPU utilization (0.1 to 1.0) */
|
||||
cpuUtilization: number;
|
||||
/** Whether to enable debug logging */
|
||||
debug?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Default worker pool options
|
||||
*/
|
||||
export const DEFAULT_WORKER_OPTIONS: WorkerPoolOptions = {
|
||||
maxWorkers: Math.max(1, (navigator.hardwareConcurrency || 2) - 1),
|
||||
cpuUtilization: 0.75,
|
||||
debug: false,
|
||||
};
|
||||
|
||||
/**
|
||||
* Task callback type
|
||||
*/
|
||||
export type TaskCallback<T> = (error: Error | null, result?: T) => void;
|
||||
|
||||
/**
|
||||
* Task request interface
|
||||
*/
|
||||
interface TaskRequest {
|
||||
/** Command to process */
|
||||
command: IndexerCommand;
|
||||
/** Callback to call with result */
|
||||
callback: TaskCallback<IndexerResult>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Worker data for pool
|
||||
*/
|
||||
interface WorkerData {
|
||||
/** Worker instance */
|
||||
worker: Worker;
|
||||
/** Whether the worker is busy */
|
||||
busy: boolean;
|
||||
/** Last time this worker finished a task */
|
||||
lastTaskFinished: number;
|
||||
/** Number of tasks processed */
|
||||
tasksProcessed: number;
|
||||
/** Total processing time */
|
||||
totalProcessingTime: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Worker pool for task processing
|
||||
*/
|
||||
export class TaskWorkerManager {
|
||||
/** Worker pool */
|
||||
private workers: WorkerData[] = [];
|
||||
/** Queue of pending tasks */
|
||||
private taskQueue: TaskRequest[] = [];
|
||||
/** Whether the pool is currently active */
|
||||
private active: boolean = true;
|
||||
/** Options for the worker pool */
|
||||
private options: WorkerPoolOptions;
|
||||
/** Vault instance */
|
||||
private vault: Vault;
|
||||
|
||||
/**
|
||||
* Create a new worker pool
|
||||
*/
|
||||
constructor(vault: Vault, options: Partial<WorkerPoolOptions> = {}) {
|
||||
this.options = { ...DEFAULT_WORKER_OPTIONS, ...options };
|
||||
this.vault = vault;
|
||||
|
||||
// Initialize workers up to max
|
||||
this.initializeWorkers();
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize workers in the pool
|
||||
*/
|
||||
private initializeWorkers(): void {
|
||||
const workerCount = Math.min(
|
||||
this.options.maxWorkers,
|
||||
navigator.hardwareConcurrency || 2
|
||||
);
|
||||
|
||||
for (let i = 0; i < workerCount; i++) {
|
||||
try {
|
||||
const worker = new Worker(
|
||||
URL.createObjectURL(
|
||||
new Blob([`importScripts('${TaskWorker}')`], {
|
||||
type: "application/javascript",
|
||||
})
|
||||
)
|
||||
);
|
||||
|
||||
// Setup worker message handler
|
||||
worker.onmessage = (event) => {
|
||||
this.handleWorkerMessage(worker, event.data);
|
||||
};
|
||||
|
||||
worker.onerror = (event) => {
|
||||
console.error("Worker error:", event);
|
||||
};
|
||||
|
||||
this.workers.push({
|
||||
worker,
|
||||
busy: false,
|
||||
lastTaskFinished: Date.now(),
|
||||
tasksProcessed: 0,
|
||||
totalProcessingTime: 0,
|
||||
});
|
||||
|
||||
this.log(`Initialized worker #${i + 1}`);
|
||||
} catch (error) {
|
||||
console.error("Failed to initialize worker:", error);
|
||||
}
|
||||
}
|
||||
|
||||
this.log(
|
||||
`Initialized ${this.workers.length} workers (requested ${workerCount})`
|
||||
);
|
||||
|
||||
// Check if we have any workers
|
||||
if (this.workers.length === 0) {
|
||||
console.warn(
|
||||
"No workers could be initialized, falling back to main thread processing"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single file for tasks
|
||||
*/
|
||||
public processFile(file: TFile, callback: TaskCallback<Task[]>): void {
|
||||
this.vault
|
||||
.cachedRead(file)
|
||||
.then((content) => {
|
||||
// Create file stats (adapting to the worker's needs)
|
||||
const stats = {
|
||||
ctime: file.stat.ctime,
|
||||
mtime: file.stat.mtime,
|
||||
size: file.stat.size,
|
||||
};
|
||||
|
||||
const command: ParseTasksCommand = {
|
||||
type: "parseTasks",
|
||||
filePath: file.path,
|
||||
content,
|
||||
stats,
|
||||
};
|
||||
|
||||
this.queueTask(command, (error, result) => {
|
||||
if (error) {
|
||||
callback(error);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!result || result.type === "error") {
|
||||
callback(
|
||||
new Error(
|
||||
(result as ErrorResult)?.error ||
|
||||
"Unknown error"
|
||||
)
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (result.type === "parseResult") {
|
||||
const parseResult = result as TaskParseResult;
|
||||
callback(null, parseResult.tasks);
|
||||
} else {
|
||||
callback(
|
||||
new Error(`Unexpected result type: ${result.type}`)
|
||||
);
|
||||
}
|
||||
});
|
||||
})
|
||||
.catch((error) => {
|
||||
callback(error);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Process multiple files in a batch
|
||||
*/
|
||||
public processBatch(
|
||||
files: TFile[],
|
||||
callback: TaskCallback<Map<string, Task[]>>
|
||||
): void {
|
||||
// Read all files first
|
||||
Promise.all(
|
||||
files.map((file) =>
|
||||
this.vault.cachedRead(file).then((content) => ({
|
||||
file,
|
||||
content,
|
||||
}))
|
||||
)
|
||||
)
|
||||
.then((fileContents) => {
|
||||
const command: BatchIndexCommand = {
|
||||
type: "batchIndex",
|
||||
files: fileContents.map(({ file, content }) => ({
|
||||
path: file.path,
|
||||
content,
|
||||
stats: {
|
||||
ctime: file.stat.ctime,
|
||||
mtime: file.stat.mtime,
|
||||
size: file.stat.size,
|
||||
},
|
||||
})),
|
||||
};
|
||||
|
||||
this.queueTask(command, (error, result) => {
|
||||
if (error) {
|
||||
callback(error);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!result || result.type === "error") {
|
||||
callback(
|
||||
new Error(
|
||||
(result as ErrorResult)?.error ||
|
||||
"Unknown error"
|
||||
)
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (result.type === "batchResult") {
|
||||
const batchResult = result as BatchIndexResult;
|
||||
// This is just a summary, not the actual tasks
|
||||
callback(
|
||||
new Error("Batch result does not include tasks")
|
||||
);
|
||||
} else if (result.type === "parseResult") {
|
||||
// Single file result, convert to map
|
||||
const parseResult = result as TaskParseResult;
|
||||
const resultMap = new Map<string, Task[]>();
|
||||
resultMap.set(parseResult.filePath, parseResult.tasks);
|
||||
callback(null, resultMap);
|
||||
} else {
|
||||
callback(
|
||||
new Error(`Unexpected result type: ${result.type}`)
|
||||
);
|
||||
}
|
||||
});
|
||||
})
|
||||
.catch((error) => {
|
||||
callback(error);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Queue a task for processing by a worker
|
||||
*/
|
||||
private queueTask(
|
||||
command: IndexerCommand,
|
||||
callback: TaskCallback<IndexerResult>
|
||||
): void {
|
||||
this.taskQueue.push({
|
||||
command,
|
||||
callback,
|
||||
});
|
||||
|
||||
this.processQueue();
|
||||
}
|
||||
|
||||
/**
|
||||
* Process the task queue
|
||||
*/
|
||||
private processQueue(): void {
|
||||
if (!this.active || this.taskQueue.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Try to find an available worker
|
||||
const availableWorker = this.getAvailableWorker();
|
||||
if (!availableWorker) {
|
||||
// No available workers, will retry when a worker becomes available
|
||||
return;
|
||||
}
|
||||
|
||||
// Get the next task
|
||||
const task = this.taskQueue.shift();
|
||||
if (!task) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Mark the worker as busy
|
||||
availableWorker.busy = true;
|
||||
|
||||
try {
|
||||
// Send the task to the worker
|
||||
availableWorker.worker.postMessage(task.command);
|
||||
|
||||
// Store the callback with the worker so we can call it when the worker responds
|
||||
(availableWorker as any).currentTask = task;
|
||||
(availableWorker as any).taskStartTime = Date.now();
|
||||
|
||||
this.log(`Sent task to worker: ${task.command.type}`);
|
||||
} catch (error) {
|
||||
// Failed to send task to worker
|
||||
console.error("Failed to send task to worker:", error);
|
||||
|
||||
// Mark the worker as available again
|
||||
availableWorker.busy = false;
|
||||
|
||||
// Call the callback with the error
|
||||
task.callback(error as Error);
|
||||
|
||||
// Try to process the next task
|
||||
this.processQueue();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle a message from a worker
|
||||
*/
|
||||
private handleWorkerMessage(worker: Worker, data: IndexerResult): void {
|
||||
// Find the worker data
|
||||
const workerData = this.workers.find((w) => w.worker === worker);
|
||||
if (!workerData) {
|
||||
console.error("Received message from unknown worker:", data);
|
||||
return;
|
||||
}
|
||||
|
||||
// Get the task that was being processed
|
||||
const task = (workerData as any).currentTask as TaskRequest | undefined;
|
||||
const taskStartTime = (workerData as any).taskStartTime as
|
||||
| number
|
||||
| undefined;
|
||||
|
||||
// Update worker stats
|
||||
workerData.busy = false;
|
||||
workerData.lastTaskFinished = Date.now();
|
||||
workerData.tasksProcessed++;
|
||||
|
||||
if (taskStartTime) {
|
||||
const processingTime = Date.now() - taskStartTime;
|
||||
workerData.totalProcessingTime += processingTime;
|
||||
|
||||
// Apply throttling based on CPU utilization
|
||||
const delay = Math.round(
|
||||
(processingTime * (1 - this.options.cpuUtilization)) /
|
||||
this.options.cpuUtilization
|
||||
);
|
||||
if (delay > 0) {
|
||||
// Delay before this worker processes another task
|
||||
setTimeout(() => {
|
||||
// Process next task if there are any
|
||||
this.processQueue();
|
||||
}, delay);
|
||||
|
||||
this.log(
|
||||
`Worker throttled for ${delay}ms (processed in ${processingTime}ms)`
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Clear the current task
|
||||
(workerData as any).currentTask = undefined;
|
||||
(workerData as any).taskStartTime = undefined;
|
||||
|
||||
// Call the callback with the result
|
||||
if (task) {
|
||||
if (data.type === "error") {
|
||||
task.callback(new Error((data as ErrorResult).error));
|
||||
} else {
|
||||
task.callback(null, data);
|
||||
}
|
||||
} else {
|
||||
console.error(
|
||||
"Received message from worker with no associated task:",
|
||||
data
|
||||
);
|
||||
}
|
||||
|
||||
// Process next task if there are any
|
||||
this.processQueue();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an available worker
|
||||
*/
|
||||
private getAvailableWorker(): WorkerData | undefined {
|
||||
// First, look for a non-busy worker
|
||||
for (const worker of this.workers) {
|
||||
if (!worker.busy) {
|
||||
return worker;
|
||||
}
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shutdown the worker pool
|
||||
*/
|
||||
public shutdown(): void {
|
||||
this.active = false;
|
||||
|
||||
// Terminate all workers
|
||||
for (const worker of this.workers) {
|
||||
worker.worker.terminate();
|
||||
}
|
||||
|
||||
// Clear the workers array
|
||||
this.workers = [];
|
||||
|
||||
// Clear the task queue and call all callbacks with an error
|
||||
for (const task of this.taskQueue) {
|
||||
task.callback(new Error("Worker pool shut down"));
|
||||
}
|
||||
|
||||
this.taskQueue = [];
|
||||
|
||||
this.log("Worker pool shut down");
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the number of pending tasks
|
||||
*/
|
||||
public getPendingTaskCount(): number {
|
||||
return this.taskQueue.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Log a message if debugging is enabled
|
||||
*/
|
||||
private log(message: string): void {
|
||||
if (this.options.debug) {
|
||||
console.log(`[TaskWorkerManager] ${message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
22
src/utils/workers/deferred.ts
Normal file
22
src/utils/workers/deferred.ts
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
/** A promise that can be resolved directly. */
|
||||
export type Deferred<T> = Promise<T> & {
|
||||
resolve: (value: T) => void;
|
||||
reject: (error: any) => void;
|
||||
};
|
||||
|
||||
/** Create a new deferred object, which is a resolvable promise. */
|
||||
export function deferred<T>(): Deferred<T> {
|
||||
let resolve: (value: T) => void;
|
||||
let reject: (error: any) => void;
|
||||
|
||||
const promise = new Promise((res, rej) => {
|
||||
resolve = res;
|
||||
reject = rej;
|
||||
});
|
||||
|
||||
const deferred = promise as any as Deferred<T>;
|
||||
deferred.resolve = resolve!;
|
||||
deferred.reject = reject!;
|
||||
|
||||
return deferred;
|
||||
}
|
||||
234
src/utils/workers/importer.ts
Normal file
234
src/utils/workers/importer.ts
Normal file
|
|
@ -0,0 +1,234 @@
|
|||
/** Controls and creates Dataview file importers, allowing for asynchronous loading and parsing of files. */
|
||||
|
||||
import ImportWorker from "index/web-worker/importer.worker";
|
||||
import { Component, FileManager, MetadataCache, TFile, Vault } from "obsidian";
|
||||
import { CanvasImport, MarkdownImport } from "index/web-worker/message";
|
||||
import { Deferred, deferred } from "utils/deferred";
|
||||
|
||||
import { Queue } from "@datastructures-js/queue";
|
||||
|
||||
/** Settings for throttling import. */
|
||||
export interface ImportThrottle {
|
||||
/** The number of workers to use for imports. */
|
||||
workers: number;
|
||||
/** A number between 0.1 and 1 which indicates total cpu utilization target; 0.1 means spend 10% of time */
|
||||
utilization: number;
|
||||
}
|
||||
|
||||
/** Default throttle configuration. */
|
||||
export const DEFAULT_THROTTLE: ImportThrottle = {
|
||||
workers: 2,
|
||||
utilization: 0.75,
|
||||
};
|
||||
|
||||
/** Multi-threaded file parser which debounces rapid file requests automatically. */
|
||||
export class FileImporter extends Component {
|
||||
/* Background workers which do the actual file parsing. */
|
||||
workers: Map<number, PoolWorker>;
|
||||
/** The next worker ID to hand out. */
|
||||
nextWorkerId: number;
|
||||
/** If true, the importer is now inactive and will not process further files. */
|
||||
shutdown: boolean;
|
||||
|
||||
/** List of files which have been queued for a reload. */
|
||||
queue: Queue<[TFile, Deferred<any>]>;
|
||||
/** Outstanding loads indexed by path. */
|
||||
outstanding: Map<string, Promise<any>>;
|
||||
/** Throttle settings. */
|
||||
throttle: () => ImportThrottle;
|
||||
|
||||
public constructor(
|
||||
public vault: Vault,
|
||||
public fileManager: FileManager,
|
||||
public metadataCache: MetadataCache,
|
||||
throttle?: () => ImportThrottle
|
||||
) {
|
||||
super();
|
||||
this.workers = new Map();
|
||||
this.shutdown = false;
|
||||
this.nextWorkerId = 0;
|
||||
this.throttle = throttle ?? (() => DEFAULT_THROTTLE);
|
||||
|
||||
this.queue = new Queue();
|
||||
this.outstanding = new Map();
|
||||
}
|
||||
|
||||
/**
|
||||
* Queue the given file for importing. Multiple import requests for the same file in a short time period will be de-bounced
|
||||
* and all be resolved by a single actual file reload.
|
||||
*/
|
||||
public import<T>(file: TFile): Promise<T> {
|
||||
// De-bounce repeated requests for the same file.
|
||||
let existing = this.outstanding.get(file.path);
|
||||
if (existing) return existing;
|
||||
|
||||
let promise = deferred<T>();
|
||||
|
||||
this.outstanding.set(file.path, promise);
|
||||
this.queue.enqueue([file, promise]);
|
||||
this.schedule();
|
||||
return promise;
|
||||
}
|
||||
|
||||
/** Reset any active throttles on the importer (such as if the utilization changes). */
|
||||
public unthrottle() {
|
||||
for (let worker of this.workers.values()) {
|
||||
worker.availableAt = Date.now();
|
||||
}
|
||||
}
|
||||
|
||||
/** Poll from the queue and execute if there is an available worker. */
|
||||
private async schedule() {
|
||||
if (this.queue.size() == 0 || this.shutdown) return;
|
||||
|
||||
const worker = this.availableWorker();
|
||||
if (!worker) return;
|
||||
|
||||
const [file, promise] = this.queue.dequeue()!;
|
||||
worker.active = [file, promise, Date.now()];
|
||||
|
||||
try {
|
||||
switch (file.extension) {
|
||||
case "markdown":
|
||||
case "md": {
|
||||
const contents = await this.vault.cachedRead(file);
|
||||
worker!.worker.postMessage({
|
||||
type: "markdown",
|
||||
path: file.path,
|
||||
contents: contents,
|
||||
stat: file.stat,
|
||||
metadata: this.metadataCache.getFileCache(file),
|
||||
} as MarkdownImport);
|
||||
break;
|
||||
}
|
||||
case "canvas": {
|
||||
const contents = await this.vault.cachedRead(file);
|
||||
worker!.worker.postMessage({
|
||||
type: "canvas",
|
||||
path: file.path,
|
||||
contents: contents,
|
||||
stat: file.stat,
|
||||
index: this.fileManager.linkUpdaters.canvas.canvas.index
|
||||
.index[file.path],
|
||||
} as CanvasImport);
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (ex) {
|
||||
console.log("Datacore: Background file reloading failed. " + ex);
|
||||
|
||||
// Message failed, release this worker.
|
||||
worker.active = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** Finish the parsing of a file, potentially queueing a new file. */
|
||||
private finish(worker: PoolWorker, data: any) {
|
||||
if (!worker.active) {
|
||||
console.log(
|
||||
"Datacore: Received a stale worker message. Ignoring.",
|
||||
data
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const [file, promise, start] = worker.active!;
|
||||
|
||||
// Resolve promises to let users know this file has finished.
|
||||
if ("$error" in data) promise.reject(data["$error"]);
|
||||
else promise.resolve(data);
|
||||
|
||||
// Remove file from outstanding.
|
||||
this.outstanding.delete(file.path);
|
||||
|
||||
// Remove this worker if we are over capacity.
|
||||
// Otherwise, notify the queue this file is available for new work.
|
||||
if (this.workers.size > this.throttle().workers) {
|
||||
this.workers.delete(worker.id);
|
||||
terminate(worker);
|
||||
} else {
|
||||
const now = Date.now();
|
||||
const throttle = Math.max(0.1, this.throttle().utilization) - 1.0;
|
||||
const delay = (now - start) * throttle;
|
||||
|
||||
worker.active = undefined;
|
||||
|
||||
if (delay <= 1e-10) {
|
||||
worker.availableAt = now;
|
||||
this.schedule();
|
||||
} else {
|
||||
worker.availableAt = now + delay;
|
||||
|
||||
// Note: I'm pretty sure this will garauntee that this executes AFTER delay milliseconds,
|
||||
// so this should be fine; if it's not, we'll have to swap to an external timeout loop
|
||||
// which infinitely reschedules itself to the next available execution time.
|
||||
setTimeout(this.schedule.bind(this), delay);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Obtain an available worker, returning undefined if one does not exist. */
|
||||
private availableWorker(): PoolWorker | undefined {
|
||||
const now = Date.now();
|
||||
for (let worker of this.workers.values()) {
|
||||
if (!worker.active && worker.availableAt <= now) {
|
||||
return worker;
|
||||
}
|
||||
}
|
||||
|
||||
// Make a new worker if we can.
|
||||
if (this.workers.size < this.throttle().workers) {
|
||||
let worker = this.newWorker();
|
||||
this.workers.set(worker.id, worker);
|
||||
return worker;
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Create a new worker bound to this importer. */
|
||||
private newWorker(): PoolWorker {
|
||||
let worker: PoolWorker = {
|
||||
id: this.nextWorkerId++,
|
||||
availableAt: Date.now(),
|
||||
worker: new ImportWorker(),
|
||||
};
|
||||
|
||||
worker.worker.onmessage = (evt) => this.finish(worker, evt.data);
|
||||
return worker;
|
||||
}
|
||||
|
||||
/** Reject all outstanding promises and close all workers on close. */
|
||||
public onunload(): void {
|
||||
for (let worker of this.workers.values()) {
|
||||
terminate(worker);
|
||||
}
|
||||
|
||||
while (!this.queue.isEmpty()) {
|
||||
const [_file, promise] = this.queue.pop()!;
|
||||
promise.reject("Terminated");
|
||||
}
|
||||
|
||||
this.shutdown = true;
|
||||
}
|
||||
}
|
||||
|
||||
/** A worker in the pool of executing workers. */
|
||||
interface PoolWorker {
|
||||
/** The id of this worker. */
|
||||
id: number;
|
||||
/** The raw underlying worker. */
|
||||
worker: Worker;
|
||||
/** UNIX time indicating the next time this worker is available for execution according to target utilization. */
|
||||
availableAt: number;
|
||||
/** The active promise this worker is working on, if any. */
|
||||
active?: [TFile, Deferred<any>, number];
|
||||
}
|
||||
|
||||
/** Terminate a pool worker. */
|
||||
function terminate(worker: PoolWorker) {
|
||||
worker.worker.terminate();
|
||||
|
||||
if (worker.active) worker.active[1].reject("Terminated");
|
||||
worker.active = undefined;
|
||||
}
|
||||
47
src/utils/workers/importer.worker.ts
Normal file
47
src/utils/workers/importer.worker.ts
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
import { canvasImport } from "index/import/canvas";
|
||||
import { markdownImport } from "index/import/markdown";
|
||||
import {
|
||||
CanvasImportResult,
|
||||
ImportCommand,
|
||||
MarkdownImportResult,
|
||||
} from "index/web-worker/message";
|
||||
|
||||
/** Web worker entry point for importing. */
|
||||
onmessage = async (event) => {
|
||||
try {
|
||||
const message = event.data as ImportCommand;
|
||||
|
||||
if (message.type === "markdown") {
|
||||
const markdown = markdownImport(
|
||||
message.path,
|
||||
message.contents,
|
||||
message.metadata,
|
||||
message.stat
|
||||
);
|
||||
|
||||
postMessage({
|
||||
type: "markdown",
|
||||
result: markdown,
|
||||
} as MarkdownImportResult);
|
||||
} else if (message.type === "canvas") {
|
||||
const canvas = canvasImport(
|
||||
message.path,
|
||||
message.contents,
|
||||
message.index,
|
||||
message.stat
|
||||
);
|
||||
|
||||
postMessage({
|
||||
type: "canvas",
|
||||
result: canvas,
|
||||
} as CanvasImportResult);
|
||||
} else {
|
||||
postMessage({ $error: "Unsupported import method." });
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`Datacore Indexer failed to index ${event.data.path}: ${error}`
|
||||
);
|
||||
postMessage({ $error: error.message });
|
||||
}
|
||||
};
|
||||
58
src/utils/workers/message.ts
Normal file
58
src/utils/workers/message.ts
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
import { CanvasMetadataIndex, JsonCanvas } from "index/types/json/canvas";
|
||||
import { JsonMarkdownPage } from "index/types/json/markdown";
|
||||
import { CachedMetadata, FileStats } from "obsidian";
|
||||
|
||||
/** A command to import a markdown file. */
|
||||
export interface MarkdownImport {
|
||||
type: "markdown";
|
||||
|
||||
/** The path we are importing. */
|
||||
path: string;
|
||||
/** The file contents to import. */
|
||||
contents: string;
|
||||
/** The stats for the file. */
|
||||
stat: FileStats;
|
||||
/** Metadata for the file. */
|
||||
metadata: CachedMetadata;
|
||||
}
|
||||
|
||||
/** A command to import a canvas file. */
|
||||
export interface CanvasImport {
|
||||
type: "canvas";
|
||||
|
||||
/** The path we are importing. */
|
||||
path: string;
|
||||
/** The raw JSON contents we are importing. */
|
||||
contents: string;
|
||||
/** The stats for the file. */
|
||||
stat: FileStats;
|
||||
/** the canvas's metadata cache */
|
||||
index: CanvasMetadataIndex["any"];
|
||||
}
|
||||
|
||||
/** Available import commands to be sent to an import web worker. */
|
||||
export type ImportCommand = MarkdownImport | CanvasImport;
|
||||
|
||||
/** The result of importing a file of some variety. */
|
||||
export interface MarkdownImportResult {
|
||||
/** The type of import. */
|
||||
type: "markdown";
|
||||
/** The result of importing. */
|
||||
result: JsonMarkdownPage;
|
||||
}
|
||||
export interface CanvasImportResult {
|
||||
type: "canvas";
|
||||
result: JsonCanvas;
|
||||
}
|
||||
export interface ImportFailure {
|
||||
/** Failed to import. */
|
||||
type: "error";
|
||||
|
||||
/** The error that the worker indicated on failure. */
|
||||
$error: string;
|
||||
}
|
||||
|
||||
export type ImportResult =
|
||||
| MarkdownImportResult
|
||||
| CanvasImportResult
|
||||
| ImportFailure;
|
||||
0
src/utils/workers/normalizers.ts
Normal file
0
src/utils/workers/normalizers.ts
Normal file
|
|
@ -39,5 +39,6 @@
|
|||
"6.1.0": "0.15.2",
|
||||
"6.2.0": "0.15.2",
|
||||
"6.2.1": "0.15.2",
|
||||
"6.2.2": "0.15.2"
|
||||
"6.2.2": "0.15.2",
|
||||
"7.0.0": "0.15.2"
|
||||
}
|
||||
Loading…
Reference in a new issue