From a0feafc3dc27860daab1d0552d5b0f1f09dc937c Mon Sep 17 00:00:00 2001 From: Shane Lamb Date: Mon, 21 Apr 2025 15:21:15 +0300 Subject: [PATCH] util to build repo context --- README.md | 4 ++ package-lock.json | 46 +++++++++++++++-- package.json | 1 + src/llm-doc.ts | 19 ++++--- src/repo-context.ts | 53 ++++++++++++++++++++ test/fixtures/repos/calculator/.gitignore | 1 + test/fixtures/repos/calculator/README.md | 21 ++++++++ test/fixtures/repos/calculator/package.json | 14 ++++++ test/fixtures/repos/calculator/src/index.ts | 22 ++++++++ test/fixtures/repos/calculator/src/utils.ts | 13 +++++ test/fixtures/repos/calculator/tsconfig.json | 15 ++++++ test/repo-context.test.ts | 50 ++++++++++++++++++ 12 files changed, 247 insertions(+), 12 deletions(-) create mode 100644 src/repo-context.ts create mode 100644 test/fixtures/repos/calculator/.gitignore create mode 100644 test/fixtures/repos/calculator/README.md create mode 100644 test/fixtures/repos/calculator/package.json create mode 100644 test/fixtures/repos/calculator/src/index.ts create mode 100644 test/fixtures/repos/calculator/src/utils.ts create mode 100644 test/fixtures/repos/calculator/tsconfig.json create mode 100644 test/repo-context.test.ts diff --git a/README.md b/README.md index 9c103ae..31c3ee4 100644 --- a/README.md +++ b/README.md @@ -31,3 +31,7 @@ Type in your prompt and click "Complete" to generate a response! (or bind a keyb ## If you like it, I'm grateful for your support! Buy Me a Coffee at ko-fi.com + + +## User Guide (2.0) +- Use inbuilt "Duplicate current file" command to fork diff --git a/package-lock.json b/package-lock.json index da02983..9b2ca9d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,7 @@ "name": "obsidian-llm-docs", "version": "1.0.0", "dependencies": { + "ignore": "^7.0.3", "node-fetch": "^3.3.2" }, "devDependencies": { @@ -956,6 +957,16 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/@eslint/eslintrc/node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "peer": true, + "engines": { + "node": ">= 4" + } + }, "node_modules/@eslint/js": { "version": "8.57.1", "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.1.tgz", @@ -1691,6 +1702,15 @@ } } }, + "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "engines": { + "node": ">= 4" + } + }, "node_modules/@typescript-eslint/parser": { "version": "5.29.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.29.0.tgz", @@ -2713,6 +2733,16 @@ "node": ">=4.0" } }, + "node_modules/eslint/node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "peer": true, + "engines": { + "node": ">= 4" + } + }, "node_modules/espree": { "version": "9.6.1", "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", @@ -3192,6 +3222,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/globby/node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "engines": { + "node": ">= 4" + } + }, "node_modules/graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", @@ -3242,10 +3281,9 @@ } }, "node_modules/ignore": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", - "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", - "dev": true, + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.3.tgz", + "integrity": "sha512-bAH5jbK/F3T3Jls4I0SO1hmPR0dKU0a7+SY6n1yzRtG54FLO8d6w/nxLFX2Nb7dBu6cCWXPaAME6cYqFUMmuCA==", "engines": { "node": ">= 4" } diff --git a/package.json b/package.json index 5a029fd..106d132 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "typescript": "4.7.4" }, "dependencies": { + "ignore": "^7.0.3", "node-fetch": "^3.3.2" } } diff --git a/src/llm-doc.ts b/src/llm-doc.ts index 396d855..cb1bd8a 100644 --- a/src/llm-doc.ts +++ b/src/llm-doc.ts @@ -7,6 +7,7 @@ import { resolveConnectionForModel } from './connection-models' export interface LlmDocProperties { model: string + repo_dir?: string } type CompletionStream = OpenaiChatCompletionStream @@ -80,16 +81,18 @@ export class LlmDoc { await this.write() - const stream = new completionStream( - connectionSettings, - this.properties.model, - await preprocessMessages( - this.messages, - getDocLinkResolver(this.app, this.file.path), - getImageLinkResolver(this.app, this.file.path), - ), + const messages = await preprocessMessages( + this.messages, + getDocLinkResolver(this.app, this.file.path), + getImageLinkResolver(this.app, this.file.path), ) + if (this.properties.repo_dir) { + // todo: enrich prompt with repo context + } + + const stream = new completionStream(connectionSettings, this.properties.model, messages) + this.currentStream = stream let headingAdded = !userIsLast diff --git a/src/repo-context.ts b/src/repo-context.ts new file mode 100644 index 0000000..191adc7 --- /dev/null +++ b/src/repo-context.ts @@ -0,0 +1,53 @@ +import * as ignore from 'ignore' +import * as fs from 'node:fs' +import * as path from 'node:path' + +export async function buildRepoContext(pathToRepo: string): Promise { + const ignored = ignore().add(getIfExists(path.join(pathToRepo, '.gitignore'))) + const included = ignore().add(getIfExists(path.join(pathToRepo, '.contextinclude'))) + + let prompt = 'The repository contains the following list of files:\n' + const files = getFilesRecursive(pathToRepo) + const filteredFiles = ignored.filter(files) + + prompt += filteredFiles.join('\n') + + const includedFiles = filteredFiles.filter(file => included.ignores(file)) + + prompt += '\n\n' + prompt += includedFiles.map(file => getFileContent(pathToRepo, file)).join('\n\n') + + return prompt +} + +function getFileContent(rootDir:string, filePath: string): string { + const content = fs.readFileSync(path.join(rootDir, filePath)).toString() + return filePath + ' contents:\n' + content +} + +function getFilesRecursive(rootDir: string, subDir = ''): string[] { + const objs = fs.readdirSync(path.join(rootDir, subDir), { withFileTypes: true }) + let files = [] + let dirs = [] + for (const obj of objs) { + if (obj.isFile()) { + files.push(path.join(subDir, obj.name)) + } else { + dirs.push(path.join(subDir, obj.name)) + } + } + // sort files and dirs alphabetically + files = files.sort((a, b) => a.localeCompare(b)) + dirs = dirs.sort((a, b) => a.localeCompare(b)) + for (const dir of dirs) { + files.push(...getFilesRecursive(rootDir, dir)) + } + return files +} + +function getIfExists(filePath: string) { + if (fs.existsSync(filePath)) { + return fs.readFileSync(filePath).toString() + } + return '' +} diff --git a/test/fixtures/repos/calculator/.gitignore b/test/fixtures/repos/calculator/.gitignore new file mode 100644 index 0000000..caf0a18 --- /dev/null +++ b/test/fixtures/repos/calculator/.gitignore @@ -0,0 +1 @@ +.contextinclude diff --git a/test/fixtures/repos/calculator/README.md b/test/fixtures/repos/calculator/README.md new file mode 100644 index 0000000..43cef93 --- /dev/null +++ b/test/fixtures/repos/calculator/README.md @@ -0,0 +1,21 @@ +# Calculator CLI + +Run this application using ts-node and provide options to perform addition or subtraction. + +## Commands Examples: + +Addition: +``` +ts-node src -o add --xOperand 4 --yOperand 5 +``` + +Subtraction: +``` +ts-node src -o sub --xOperand 4 --yOperand 5 +``` + +## Install Dependencies Before Running: + +```bash +npm install +``` diff --git a/test/fixtures/repos/calculator/package.json b/test/fixtures/repos/calculator/package.json new file mode 100644 index 0000000..e95b314 --- /dev/null +++ b/test/fixtures/repos/calculator/package.json @@ -0,0 +1,14 @@ +{ + "name": "calculator-cli", + "version": "1.0.0", + "main": "src/index.ts", + "scripts": { + "run:calc": "ts-node src" + }, + "devDependencies": { + "commander": "^9.0.0", + "typescript": "^4.5.2", + "@types/node": "^16.11.7", + "ts-node": "^10.4.0" + } +} diff --git a/test/fixtures/repos/calculator/src/index.ts b/test/fixtures/repos/calculator/src/index.ts new file mode 100644 index 0000000..aab82ec --- /dev/null +++ b/test/fixtures/repos/calculator/src/index.ts @@ -0,0 +1,22 @@ +#!/usr/bin/env ts-node + +import { program } from 'commander' +import * as calcUtils from './utils' + +// Configure CLI options using Commander +program + .requiredOption('-o --operation ', 'The type of mathematical operation to perform. Supported: add, sub') + .option('-x --xOperand ', 'The first operand', parseFloat) + .option('-y --yOperand ', 'The second operand', parseFloat) + +// Parse the arguments before using them +program.parse(process.argv) +const options = program.opts() + +if (options.operation === 'add' && !isNaN(options.xOperand) && !isNaN(options.yOperand)) { + console.log(calcUtils.add(options.xOperand, options.yOperand)) +} else if (options.operation === 'sub' && !isNaN(options.xOperand) && !isNaN(options.yOperand)) { + console.log(calcUtils.sub(options.xOperand, options.yOperand)) +} else { + throw new Error('Invalid or missing arguments.') +} diff --git a/test/fixtures/repos/calculator/src/utils.ts b/test/fixtures/repos/calculator/src/utils.ts new file mode 100644 index 0000000..346b6e2 --- /dev/null +++ b/test/fixtures/repos/calculator/src/utils.ts @@ -0,0 +1,13 @@ +/** + * File: calc/utils.ts + * Description: + * This file exports functions used by main.ts to perform mathematical operations. + */ + +export function add(x: number, y: number): number { + return x + y; +} + +export function sub(x: number, y: number): number { + return x - y; +} diff --git a/test/fixtures/repos/calculator/tsconfig.json b/test/fixtures/repos/calculator/tsconfig.json new file mode 100644 index 0000000..f3a454e --- /dev/null +++ b/test/fixtures/repos/calculator/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES6", + "module": "commonjs", + "outDir": "./dist", + "rootDir": ".", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + }, + "include": [ + "src/**/*.ts" + ] +} diff --git a/test/repo-context.test.ts b/test/repo-context.test.ts new file mode 100644 index 0000000..328e5c4 --- /dev/null +++ b/test/repo-context.test.ts @@ -0,0 +1,50 @@ +import { beforeEach, describe, expect, test } from '@jest/globals' +import { buildRepoContext } from '../src/repo-context' +import * as path from 'node:path' +import * as fs from 'node:fs' + +describe('building repo context', () => { + beforeEach(() => { + // reset git status + }) + + test('calculator repo', async () => { + const repoDir = path.join(__dirname, 'fixtures', 'repos', 'calculator') + + // setup + fs.writeFileSync( + path.join(repoDir, '.contextinclude'), + ` +# README gives the intent/purpose of the repo/project +README.md + +# include the whole of the source code as context as it's small enough +src/ +`, + ) + + // execute + const context = await buildRepoContext(repoDir) + + // assert + expect(context).toEqual( + ` +The repository contains the following list of files: +.gitignore +package.json +README.md +tsconfig.json +src/index.ts +src/utils.ts + +README.md contents: +${fs.readFileSync(path.join(repoDir, 'README.md'))} + +src/index.ts contents: +${fs.readFileSync(path.join(repoDir, 'src', 'index.ts'))} + +src/utils.ts contents: +${fs.readFileSync(path.join(repoDir, 'src', 'utils.ts'))}`.trimStart(), + ) + }) +})