obsidian-plugin/nathonius_obsidian-github-link
1
0
Fork 0
mirror of https://github.com/nathonius/obsidian-github-link.git synced 2026-07-22 09:20:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

✨ feat: #174 Migrate cache from data.json to IndexedDB

Browse source 
Move the API response cache out of Obsidian's data.json (which gets
synced) into IndexedDB (which stays local). This prevents cache data
from bloating sync and keeps device-specific data where it belongs.

- Replace in-memory Record-based RequestCache with IndexedDB-backed
  storage using the idb library, with an in-memory layer for fast
  synchronous reads
- Remove periodic cache save interval (cacheIntervalSeconds setting)
  since IndexedDB persists on write
- Add one-time migration: existing data.json cache entries are imported
  into IndexedDB, then cleared from data.json along with the legacy
  cacheIntervalSeconds setting
- Preserve all existing cache behavior: TTL cleanup on startup
  (maxCacheAgeHours), request freshness checks (minRequestSeconds),
  and HTTP conditional requests (ETag/Last-Modified)
This commit is contained in:
Jacobo de Vera 2026-04-18 21:47:18 +01:00
parent a5ad979881
commit a1ed9bfd5e
10 changed files with 197 additions and 152 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
jest.config.mjs
View file

@ -134,7 +134,10 @@ const config = {
// runner: "jest-runner",
// The paths to modules that run some code to configure or set up the testing environment before each test
// setupFiles: [],
setupFiles: ["fake-indexeddb/auto", "<rootDir>/jest.setup.mjs"],
// Options that will be passed to the testEnvironment
// testEnvironmentOptions: {},
// A list of paths to modules that run some code to configure or set up the testing framework before each test
// setupFilesAfterEnv: [],

4
jest.setup.mjs Normal file
View file

@ -0,0 +1,4 @@
// jsdom doesn't expose structuredClone, but fake-indexeddb needs it
if (typeof globalThis.structuredClone === "undefined") {
globalThis.structuredClone = (val) => JSON.parse(JSON.stringify(val));
}

18
package-lock.json generated
View file

@ -14,6 +14,7 @@
"@octokit/openapi-types": "^19.1.0",
"@octokit/plugin-rest-endpoint-methods": "^10.3.0",
"@octokit/request": "^8.2.0",
"idb": "^8.0.3",
"lodash": "^4.17.21",
"queue": "^7.0.0"
},
@ -31,6 +32,7 @@
"eslint-plugin-import": "^2.29.1",
"eslint-plugin-jest": "^28.2.0",
"eslint-plugin-unused-imports": "^3.1.0",
"fake-indexeddb": "^6.2.5",
"jest": "^29.7.0",
"jest-environment-jsdom": "^29.7.0",
"obsidian": "latest",
@ -4260,6 +4262,16 @@
"node": "^14.15.0 || ^16.10.0 || >=18.0.0"
}
},
"node_modules/fake-indexeddb": {
"version": "6.2.5",
"resolved": "https://registry.npmjs.org/fake-indexeddb/-/fake-indexeddb-6.2.5.tgz",
"integrity": "sha512-CGnyrvbhPlWYMngksqrSSUT1BAVP49dZocrHuK0SvtR0D5TMs5wP0o3j7jexDJW01KSadjBp1M/71o/KR3nD1w==",
"dev": true,
"license": "Apache-2.0",
"engines": {
"node": ">=18"
}
},
"node_modules/fast-deep-equal": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
@ -4814,6 +4826,12 @@
"node": ">=0.10.0"
}
},
"node_modules/idb": {
"version": "8.0.3",
"resolved": "https://registry.npmjs.org/idb/-/idb-8.0.3.tgz",
"integrity": "sha512-LtwtVyVYO5BqRvcsKuB2iUMnHwPVByPCXFXOpuU96IZPPoPN6xjOGxZQ74pgSVVLQWtUOYgyeL4GE98BY5D3wg==",
"license": "ISC"
},
"node_modules/ignore": {
"version": "5.3.1",
"resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz",

4
package.json
View file

@ -30,6 +30,7 @@
"eslint-plugin-import": "^2.29.1",
"eslint-plugin-jest": "^28.2.0",
"eslint-plugin-unused-imports": "^3.1.0",
"fake-indexeddb": "^6.2.5",
"jest": "^29.7.0",
"jest-environment-jsdom": "^29.7.0",
"obsidian": "latest",
@ -46,7 +47,8 @@
"@octokit/openapi-types": "^19.1.0",
"@octokit/plugin-rest-endpoint-methods": "^10.3.0",
"@octokit/request": "^8.2.0",
"idb": "^8.0.3",
"lodash": "^4.17.21",
"queue": "^7.0.0"
}
}
}

4
src/github/api.ts
View file

@ -241,10 +241,10 @@ export class GitHubApi {
// Check for 304 response, return cached value
if (cachedValue?.response && response.status === 304) {
getCache().update(config);
await getCache().update(config);
return this.getPaginationMeta(cachedValue.response);
} else if (isSuccessResponse(response.status)) {
getCache().set(config, response);
await getCache().set(config, response);
}
// Handle rate limit
const retryAfterSeconds = parseInt(response.headers["retry-after"]);

165
src/github/cache.ts
View file

@ -1,15 +1,30 @@
import type { RequestUrlParam, RequestUrlResponse } from "obsidian";
import { openDB } from "idb";
import type { DBSchema, IDBPDatabase } from "idb";
import { logger } from "../plugin";
import { isSuccessResponse, sanitizeObject } from "../util";
interface CacheParams {
request: RequestUrlParam;
const DB_NAME = "obsidian-github-link";
const STORE_NAME = "request-cache";
const DB_VERSION = 1;
export interface CacheEntryData {
url: string;
requestBody?: string;
response: RequestUrlResponse;
retrieved: number;
etag: string | null;
lastModified: string | null;
}
interface CacheDB extends DBSchema {
[STORE_NAME]: {
key: string;
value: CacheEntryData;
indexes: { "by-retrieved": number };
};
}
export class CacheEntry {
constructor(
public readonly request: RequestUrlParam,
@ -19,11 +34,26 @@ export class CacheEntry {
public readonly lastModified: string | null,
) {}
public static fromStored(data: CacheEntryData): CacheEntry {
return new CacheEntry(
{ url: data.url, body: data.requestBody },
data.response,
new Date(data.retrieved),
data.etag,
data.lastModified,
);
}
public static fromJSON(json: string): CacheEntry | null {
let result: CacheEntry | null = null;
try {
const parsed = JSON.parse(json) as CacheParams;
result = new CacheEntry(
const parsed = JSON.parse(json) as {
request: RequestUrlParam;
response: RequestUrlResponse;
retrieved: number;
etag: string | null;
lastModified: string | null;
};
return new CacheEntry(
parsed.request,
parsed.response,
new Date(parsed.retrieved),
@ -33,57 +63,59 @@ export class CacheEntry {
} catch (err) {
logger.error("Failure reconstructing cache!");
logger.error(err);
return null;
}
return result;
}
public toJSON(): string {
const params: CacheParams = {
request: this.request,
public toStored(): CacheEntryData {
return {
url: this.request.url,
requestBody: this.request.body as string | undefined,
response: this.response,
retrieved: this.retrieved.getTime(),
etag: this.etag,
lastModified: this.lastModified,
};
return JSON.stringify(params);
}
}
/**
* Cache of responses to simple, non-search requests
* Cache of responses to simple, non-search requests, backed by IndexedDB.
*/
export class RequestCache {
public cacheUpdated = false;
private readonly entries: Record<string, CacheEntry> = {};
private db: IDBPDatabase<CacheDB> | null = null;
private memCache: Record<string, CacheEntry> = {};
constructor(storedCache: string[] | null) {
if (storedCache) {
try {
for (const entryString of storedCache) {
const entry = CacheEntry.fromJSON(entryString);
if (!entry) {
return;
}
this.entries[this.getCacheKey(entry.request)] = entry;
}
} catch (err) {
logger.warn("Could not read stored cache data, cache will be cleared.");
logger.warn(err);
}
/**
* Open the IndexedDB database. Must be called before using the cache.
*/
public async init(): Promise<void> {
this.db = await openDB<CacheDB>(DB_NAME, DB_VERSION, {
upgrade(db) {
const store = db.createObjectStore(STORE_NAME, { keyPath: "url" });
store.createIndex("by-retrieved", "retrieved");
},
});
// Load all entries into memory for fast synchronous reads
const all = await this.db.getAll(STORE_NAME);
for (const data of all) {
this.memCache[data.url] = CacheEntry.fromStored(data);
}
}
/**
* Synchronous read from in-memory cache.
*/
public get(request: RequestUrlParam): CacheEntry | null {
const entry: CacheEntry | null = this.entries[this.getCacheKey(request)] ?? null;
// Ensure headers are defined; some old cache entries might not have them
const entry: CacheEntry | null = this.memCache[this.getCacheKey(request)] ?? null;
if (entry && !entry.response.headers) {
entry.response.headers = {};
}
return entry;
}
public set(request: RequestUrlParam, response: RequestUrlResponse): void {
// Don't store bad responses
public async set(request: RequestUrlParam, response: RequestUrlResponse): Promise<void> {
if (!isSuccessResponse(response.status)) {
logger.warn(`Attempted to cache a non-successful request: ${request.url}`);
return;
@ -92,7 +124,6 @@ export class RequestCache {
const etag = response.headers.etag ?? null;
const lastModified = response.headers["last-modified"] ?? null;
// Slim down the data we store
const _request: Partial<RequestUrlParam> = { url: request.url, body: request.body };
const _response: Partial<RequestUrlResponse> = {
json: response.json,
@ -107,45 +138,69 @@ export class RequestCache {
etag,
lastModified,
);
this.entries[this.getCacheKey(request)] = entry;
this.cacheUpdated = true;
const key = this.getCacheKey(request);
this.memCache[key] = entry;
await this.db?.put(STORE_NAME, entry.toStored());
}
public remove(request: RequestUrlParam | string): void {
if (typeof request === "string") {
delete this.entries[request];
} else {
delete this.entries[this.getCacheKey(request)];
}
this.cacheUpdated = true;
public async remove(request: RequestUrlParam | string): Promise<void> {
const key = typeof request === "string" ? request : this.getCacheKey(request);
delete this.memCache[key];
await this.db?.delete(STORE_NAME, key);
}
public clean(maxAge: Date): number {
public async clean(maxAge: Date): Promise<number> {
let entriesDeleted = 0;
for (const [k, v] of Object.entries(this.entries)) {
const keysToDelete: string[] = [];
for (const [k, v] of Object.entries(this.memCache)) {
if (v.retrieved < maxAge) {
delete this.entries[k];
entriesDeleted += 1;
keysToDelete.push(k);
}
}
for (const key of keysToDelete) {
delete this.memCache[key];
await this.db?.delete(STORE_NAME, key);
entriesDeleted += 1;
}
return entriesDeleted;
}
public update(request: RequestUrlParam | string): void {
let entry: CacheEntry | null = null;
if (typeof request === "string") {
entry = this.entries[request];
} else {
entry = this.entries[this.getCacheKey(request)];
}
public async update(request: RequestUrlParam | string): Promise<void> {
const key = typeof request === "string" ? request : this.getCacheKey(request);
const entry = this.memCache[key];
if (entry) {
entry.retrieved = new Date();
await this.db?.put(STORE_NAME, entry.toStored());
}
this.cacheUpdated = true;
}
public toJSON(): string[] {
return Object.values(this.entries).map((e) => e.toJSON());
/**
* Import entries from the old JSON-based cache (data.json migration).
*/
public async importFromJSON(storedCache: string[]): Promise<number> {
let imported = 0;
for (const entryString of storedCache) {
const entry = CacheEntry.fromJSON(entryString);
if (entry) {
const key = this.getCacheKey(entry.request);
this.memCache[key] = entry;
await this.db?.put(STORE_NAME, entry.toStored());
imported += 1;
}
}
return imported;
}
/**
* Close the IndexedDB connection.
*/
public close(): void {
this.db?.close();
this.db = null;
}
private getCacheKey(request: RequestUrlParam): string {

43
src/plugin.spec.ts
View file

@ -1,4 +1,4 @@
import { expect, jest, test, describe, beforeEach } from "@jest/globals";
import { expect, jest, test, describe, beforeEach, afterEach } from "@jest/globals";
import type { Plugin, RequestUrlResponse } from "obsidian";
import { App } from "obsidian";
import type { PluginMock } from "../__mocks__/obsidian/Plugin";
@ -6,7 +6,7 @@ import * as manifest from "../manifest.json";
import { GithubLinkPlugin, PluginData, PluginSettings, getCache } from "./plugin";
import type { GithubLinkPluginSettings } from "./settings";
import { DEFAULT_SETTINGS } from "./settings";
import { CacheEntry, RequestCache } from "./github/cache";
import { RequestCache } from "./github/cache";
import { LogLevel } from "./logger";
import { DATA_VERSION } from "./settings/types";
@ -24,6 +24,13 @@ describe("GithubLinkPlugin", () => {
app = new App();
});
afterEach(() => {
// Close the cache DB connection so subsequent tests can open a fresh one
if (getCache()) {
getCache().close();
}
});
test("should create", () => {
plugin = new GithubLinkPlugin(app, manifest);
expect(plugin).toBeTruthy();
@ -42,7 +49,6 @@ describe("GithubLinkPlugin", () => {
expect(plugin.registerMarkdownPostProcessor).toHaveBeenCalled();
expect(plugin.registerEditorExtension).toHaveBeenCalled();
expect(plugin.registerMarkdownCodeBlockProcessor).toHaveBeenCalled();
expect(plugin.registerInterval).toHaveBeenCalled();
});
describe("loadData", () => {
@ -50,28 +56,33 @@ describe("GithubLinkPlugin", () => {
plugin = new GithubLinkPlugin(app, manifest);
await plugin.onload();
expect(PluginData).toBeDefined();
expect(PluginData.cache).toEqual(null);
expect(PluginData.settings).toEqual(DEFAULT_SETTINGS);
expect(PluginData.dataVersion).toEqual(DATA_VERSION);
});
test("should load stored cache", async () => {
const cacheEntry = new CacheEntry(
{ url: "mock" },
{ json: "mock", headers: {} } as RequestUrlResponse,
new Date(),
null,
null,
);
test("should migrate stored cache from data.json to IndexedDB", async () => {
const now = new Date();
const cacheJson = JSON.stringify({
request: { url: "https://api.github.com/repos/test/test/issues/1" },
response: { json: { title: "mock" }, headers: {}, status: 200 },
retrieved: now.getTime(),
etag: null,
lastModified: null,
});
plugin = new GithubLinkPlugin(app, manifest);
mockedPlugin(plugin).data = { cache: [cacheEntry.toJSON()], dataVersion: DATA_VERSION };
mockedPlugin(plugin).data = { cache: [cacheJson], dataVersion: DATA_VERSION };
await plugin.onload();
expect(PluginData.cache).toEqual([cacheEntry.toJSON()]);
expect(getCache().get(cacheEntry.request)).toEqual(cacheEntry);
// Cache should be accessible
const entry = getCache().get({ url: "https://api.github.com/repos/test/test/issues/1" });
expect(entry).not.toBeNull();
expect(entry?.response.json).toEqual({ title: "mock" });
// data.json cache should be cleared
expect(PluginData.cache).toBeUndefined();
});
test.each<{ stored: Partial<GithubLinkPluginSettings>; name: string }>([
{ stored: { cacheIntervalSeconds: 69 }, name: "cacheIntervalSeconds" },
{ stored: { defaultPageSize: 69 }, name: "defaultPageSize" },
{ stored: { tagTooltips: !DEFAULT_SETTINGS.tagTooltips }, name: "tagTooltips" },
{ stored: { minRequestSeconds: 69 }, name: "minRequestSeconds" },

59
src/plugin.ts
View file

@ -12,7 +12,7 @@ import { QueryProcessor } from "./query/processor";
import { DATA_VERSION } from "./settings/types";
export const PluginSettings: GithubLinkPluginSettings = { ...DEFAULT_SETTINGS };
export const PluginData: GithubLinkPluginData = { cache: null, settings: PluginSettings, dataVersion: DATA_VERSION };
export const PluginData: GithubLinkPluginData = { settings: PluginSettings, dataVersion: DATA_VERSION };
export const logger = new Logger();
let cache: RequestCache;
export function getCache(): RequestCache {
@ -20,7 +20,6 @@ export function getCache(): RequestCache {
}
export class GithubLinkPlugin extends Plugin {
public cacheInterval: number | undefined;
async onload() {
const data = (await this.loadData()) || {};
@ -28,26 +27,43 @@ export class GithubLinkPlugin extends Plugin {
Object.assign(PluginData, data);
logger.logLevel = PluginSettings.logLevel;
cache = new RequestCache(PluginData.cache);
cache = new RequestCache();
await cache.init();
// Migrate cache from data.json to IndexedDB (one-time)
let needsSave = false;
if (data.cache && Array.isArray(data.cache) && data.cache.length > 0) {
const imported = await cache.importFromJSON(data.cache as string[]);
logger.info(`Migrated ${imported} cache entries from data.json to IndexedDB.`);
needsSave = true;
}
// Clean up legacy keys from data.json
if (data.cache !== undefined || (data.settings as unknown as Record<string, unknown>)?.cacheIntervalSeconds !== undefined) {
delete (PluginSettings as unknown as Record<string, unknown>).cacheIntervalSeconds;
PluginData.cache = undefined;
needsSave = true;
}
if (needsSave) {
await this.saveData({ settings: PluginSettings, dataVersion: PluginData.dataVersion });
}
if (data.dataVersion === undefined || PluginData.dataVersion < DATA_VERSION) {
// Always clear cache when data version changes
const entriesDeleted = cache.clean(new Date());
PluginData.cache = null;
const entriesDeleted = await cache.clean(new Date());
PluginData.dataVersion = DATA_VERSION;
await this.saveData(PluginData);
await this.saveData({ settings: PluginSettings, dataVersion: DATA_VERSION });
new Notice(
`GitHub link data schema migrated to version ${DATA_VERSION}. Removed ${entriesDeleted} stored items from GitHub Link cache.`,
3000,
);
}
// Clean cache
// Clean cache on startup
const maxAge = new Date(new Date().getTime() - PluginSettings.maxCacheAgeHours * 60 * 60 * 1000);
const entriesDeleted = cache.clean(maxAge);
const entriesDeleted = await cache.clean(maxAge);
if (entriesDeleted > 0) {
PluginData.cache = cache.toJSON();
await this.saveData(PluginData);
logger.info(`Cleaned ${entriesDeleted} entries from request cache.`);
}
@ -57,28 +73,5 @@ export class GithubLinkPlugin extends Plugin {
this.registerMarkdownPostProcessor(InlineRenderer);
this.registerEditorExtension(createInlineViewPlugin(this));
this.registerMarkdownCodeBlockProcessor("github-query", QueryProcessor);
this.setCacheInterval();
}
/**
* Save cache at regular interval
*/
public setCacheInterval(): void {
const checkCache = async () => {
logger.debug("Checking if cache needs a save.");
if (cache.cacheUpdated) {
PluginData.cache = cache.toJSON();
await this.saveData(PluginData);
cache.cacheUpdated = false;
logger.info(`Saved request cache with ${PluginData.cache?.length} items.`);
}
};
window.clearInterval(this.cacheInterval);
this.cacheInterval = this.registerInterval(
window.setInterval(() => {
void checkCache();
}, PluginSettings.cacheIntervalSeconds * 1000),
);
}
}

42
src/settings/settings-tab.ts
View file

@ -181,43 +181,6 @@ export class GithubLinkPluginSettingsTab extends PluginSettingTab {
containerEl.createEl("h3", { text: "Cache settings" });
new Setting(containerEl)
.setClass("github-link-sub-setting")
.setName("Cache save interval (seconds)")
.setDesc(
"If it has been updated, cache will be saved to disk after this number of seconds while Obsidian is open.",
)
.addExtraButton((button) => {
button.setIcon("rotate-ccw");
button.setTooltip("Restore default");
button.onClick(async () => {
PluginSettings.cacheIntervalSeconds = DEFAULT_SETTINGS.cacheIntervalSeconds;
await this.saveSettings();
this.plugin.setCacheInterval();
this.display();
});
})
.addSlider((slider) => {
const manualInput = createEl("input", { attr: { type: "number" }, cls: "github-link-slider-input" });
manualInput.value = PluginSettings.cacheIntervalSeconds.toString();
manualInput.addEventListener("change", (e) => {
const value = parseInt((e.target as HTMLInputElement).value, 10);
PluginSettings.cacheIntervalSeconds = value;
slider.setValue(value);
void this.saveSettings();
});
slider.sliderEl.parentElement?.prepend(manualInput);
slider.setValue(PluginSettings.cacheIntervalSeconds);
slider.setLimits(10, 1200, 10);
slider.setDynamicTooltip();
slider.onChange(async (value) => {
PluginSettings.cacheIntervalSeconds = value;
manualInput.value = value.toString();
await this.saveSettings();
this.plugin.setCacheInterval();
});
});
new Setting(containerEl)
.setClass("github-link-sub-setting")
.setName("Max cache age (hours)")
@ -228,7 +191,6 @@ export class GithubLinkPluginSettingsTab extends PluginSettingTab {
button.onClick(async () => {
PluginSettings.maxCacheAgeHours = DEFAULT_SETTINGS.maxCacheAgeHours;
await this.saveSettings();
this.plugin.setCacheInterval();
this.display();
});
})
@ -295,8 +257,7 @@ export class GithubLinkPluginSettingsTab extends PluginSettingTab {
button.setIcon("trash");
button.setButtonText("Clear cache");
button.onClick(async () => {
const itemsDeleted = getCache().clean(new Date());
PluginData.cache = null;
const itemsDeleted = await getCache().clean(new Date());
await this.saveSettings();
new Notice(`Removed ${itemsDeleted} stored items from GitHub Link cache.`, 3000);
});
@ -331,7 +292,6 @@ export class GithubLinkPluginSettingsTab extends PluginSettingTab {
private saveSettings() {
const newData: GithubLinkPluginData = {
cache: PluginData.cache,
settings: PluginSettings,
dataVersion: DATA_VERSION,
};

5
src/settings/types.ts
View file

@ -13,7 +13,8 @@ export interface GithubAccount {
export interface GithubLinkPluginData {
settings: GithubLinkPluginSettings;
cache: string[] | null;
/** @deprecated Cache is now stored in IndexedDB. Only used during migration. */
cache?: string[] | null;
dataVersion: number;
}
@ -29,7 +30,6 @@ export interface GithubLinkPluginSettings {
tagShowPRMergeable: boolean;
tagShowFileBranchName: boolean;
tagShowFileLineNumber: boolean;
cacheIntervalSeconds: number;
maxCacheAgeHours: number;
minRequestSeconds: number;
}
@ -45,7 +45,6 @@ export const DEFAULT_SETTINGS: GithubLinkPluginSettings = {
tagShowPRMergeable: false,
tagShowFileBranchName: true,
tagShowFileLineNumber: true,
cacheIntervalSeconds: 60,
maxCacheAgeHours: 120,
minRequestSeconds: 60,
};