util to build repo context

README.md

@@ -31,3 +31,7 @@ Type in your prompt and click "Complete" to generate a response! (or bind a keyb
 ## If you like it, I'm grateful for your support!
 
 <a href='https://ko-fi.com/V7V019UAWY' target='_blank'><img height='36' style='border:0px;height:36px;' src='https://storage.ko-fi.com/cdn/kofi6.png?v=6' border='0' alt='Buy Me a Coffee at ko-fi.com' /></a>
+
+
+## User Guide (2.0)
+- Use inbuilt "Duplicate current file" command to fork

package-lock.json

@@ -8,6 +8,7 @@
 			"name": "obsidian-llm-docs",
 			"version": "1.0.0",
 			"dependencies": {
+				"ignore": "^7.0.3",
 				"node-fetch": "^3.3.2"
 			},
 			"devDependencies": {
@@ -956,6 +957,16 @@
 				"url": "https://opencollective.com/eslint"
 			}
 		},
+		"node_modules/@eslint/eslintrc/node_modules/ignore": {
+			"version": "5.3.2",
+			"resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz",
+			"integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==",
+			"dev": true,
+			"peer": true,
+			"engines": {
+				"node": ">= 4"
+			}
+		},
 		"node_modules/@eslint/js": {
 			"version": "8.57.1",
 			"resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.1.tgz",
@@ -1691,6 +1702,15 @@
 				}
 			}
 		},
+		"node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": {
+			"version": "5.3.2",
+			"resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz",
+			"integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==",
+			"dev": true,
+			"engines": {
+				"node": ">= 4"
+			}
+		},
 		"node_modules/@typescript-eslint/parser": {
 			"version": "5.29.0",
 			"resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.29.0.tgz",
@@ -2713,6 +2733,16 @@
 				"node": ">=4.0"
 			}
 		},
+		"node_modules/eslint/node_modules/ignore": {
+			"version": "5.3.2",
+			"resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz",
+			"integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==",
+			"dev": true,
+			"peer": true,
+			"engines": {
+				"node": ">= 4"
+			}
+		},
 		"node_modules/espree": {
 			"version": "9.6.1",
 			"resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz",
@@ -3192,6 +3222,15 @@
 				"url": "https://github.com/sponsors/sindresorhus"
 			}
 		},
+		"node_modules/globby/node_modules/ignore": {
+			"version": "5.3.2",
+			"resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz",
+			"integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==",
+			"dev": true,
+			"engines": {
+				"node": ">= 4"
+			}
+		},
 		"node_modules/graceful-fs": {
 			"version": "4.2.11",
 			"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
@@ -3242,10 +3281,9 @@
 			}
 		},
 		"node_modules/ignore": {
-			"version": "5.3.2",
-			"resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz",
-			"integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==",
-			"dev": true,
+			"version": "7.0.3",
+			"resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.3.tgz",
+			"integrity": "sha512-bAH5jbK/F3T3Jls4I0SO1hmPR0dKU0a7+SY6n1yzRtG54FLO8d6w/nxLFX2Nb7dBu6cCWXPaAME6cYqFUMmuCA==",
 			"engines": {
 				"node": ">= 4"
 			}

package.json

@@ -29,6 +29,7 @@
 		"typescript": "4.7.4"
 	},
 	"dependencies": {
+		"ignore": "^7.0.3",
 		"node-fetch": "^3.3.2"
 	}
 }

src/llm-doc.ts

@@ -7,6 +7,7 @@ import { resolveConnectionForModel } from './connection-models'
 
 export interface LlmDocProperties {
 	model: string
+	repo_dir?: string
 }
 
 type CompletionStream = OpenaiChatCompletionStream
@@ -80,16 +81,18 @@ export class LlmDoc {
 
 		await this.write()
 
-		const stream = new completionStream(
-			connectionSettings,
-			this.properties.model,
-			await preprocessMessages(
-				this.messages,
-				getDocLinkResolver(this.app, this.file.path),
-				getImageLinkResolver(this.app, this.file.path),
-			),
+		const messages = await preprocessMessages(
+			this.messages,
+			getDocLinkResolver(this.app, this.file.path),
+			getImageLinkResolver(this.app, this.file.path),
 		)
 
+		if (this.properties.repo_dir) {
+			// todo: enrich prompt with repo context
+		}
+
+		const stream = new completionStream(connectionSettings, this.properties.model, messages)
+
 		this.currentStream = stream
 
 		let headingAdded = !userIsLast

src/repo-context.ts

@@ -0,0 +1,53 @@
+import * as ignore from 'ignore'
+import * as fs from 'node:fs'
+import * as path from 'node:path'
+
+export async function buildRepoContext(pathToRepo: string): Promise<string> {
+	const ignored = ignore().add(getIfExists(path.join(pathToRepo, '.gitignore')))
+	const included = ignore().add(getIfExists(path.join(pathToRepo, '.contextinclude')))
+
+	let prompt = 'The repository contains the following list of files:\n'
+	const files = getFilesRecursive(pathToRepo)
+	const filteredFiles = ignored.filter(files)
+
+	prompt += filteredFiles.join('\n')
+
+	const includedFiles = filteredFiles.filter(file => included.ignores(file))
+
+	prompt += '\n\n'
+	prompt += includedFiles.map(file => getFileContent(pathToRepo, file)).join('\n\n')
+
+	return prompt
+}
+
+function getFileContent(rootDir:string, filePath: string): string {
+	const content = fs.readFileSync(path.join(rootDir, filePath)).toString()
+	return filePath + ' contents:\n' + content
+}
+
+function getFilesRecursive(rootDir: string, subDir = ''): string[] {
+	const objs = fs.readdirSync(path.join(rootDir, subDir), { withFileTypes: true })
+	let files = []
+	let dirs = []
+	for (const obj of objs) {
+		if (obj.isFile()) {
+			files.push(path.join(subDir, obj.name))
+		} else {
+			dirs.push(path.join(subDir, obj.name))
+		}
+	}
+	// sort files and dirs alphabetically
+	files = files.sort((a, b) => a.localeCompare(b))
+	dirs = dirs.sort((a, b) => a.localeCompare(b))
+	for (const dir of dirs) {
+		files.push(...getFilesRecursive(rootDir, dir))
+	}
+	return files
+}
+
+function getIfExists(filePath: string) {
+	if (fs.existsSync(filePath)) {
+		return fs.readFileSync(filePath).toString()
+	}
+	return ''
+}

test/fixtures/repos/calculator/.gitignore

@@ -0,0 +1 @@
+.contextinclude

test/fixtures/repos/calculator/README.md

@@ -0,0 +1,21 @@
+# Calculator CLI
+
+Run this application using ts-node and provide options to perform addition or subtraction.
+
+## Commands Examples:
+
+Addition:
+```
+ts-node src -o add --xOperand 4 --yOperand 5
+```
+
+Subtraction:
+```
+ts-node src -o sub --xOperand 4 --yOperand 5
+```
+
+## Install Dependencies Before Running:
+
+```bash
+npm install
+```

test/fixtures/repos/calculator/package.json

@@ -0,0 +1,14 @@
+{
+	"name": "calculator-cli",
+	"version": "1.0.0",
+	"main": "src/index.ts",
+	"scripts": {
+		"run:calc": "ts-node src"
+	},
+	"devDependencies": {
+		"commander": "^9.0.0",
+		"typescript": "^4.5.2",
+		"@types/node": "^16.11.7",
+		"ts-node": "^10.4.0"
+	}
+}

test/fixtures/repos/calculator/src/index.ts

@@ -0,0 +1,22 @@
+#!/usr/bin/env ts-node
+
+import { program } from 'commander'
+import * as calcUtils from './utils'
+
+// Configure CLI options using Commander
+program
+	.requiredOption('-o --operation <type>', 'The type of mathematical operation to perform. Supported: add, sub')
+	.option('-x --xOperand <number>', 'The first operand', parseFloat)
+	.option('-y --yOperand <number>', 'The second operand', parseFloat)
+
+// Parse the arguments before using them
+program.parse(process.argv)
+const options = program.opts()
+
+if (options.operation === 'add' && !isNaN(options.xOperand) && !isNaN(options.yOperand)) {
+	console.log(calcUtils.add(options.xOperand, options.yOperand))
+} else if (options.operation === 'sub' && !isNaN(options.xOperand) && !isNaN(options.yOperand)) {
+	console.log(calcUtils.sub(options.xOperand, options.yOperand))
+} else {
+	throw new Error('Invalid or missing arguments.')
+}

test/fixtures/repos/calculator/src/utils.ts

@@ -0,0 +1,13 @@
+/**
+ * File: calc/utils.ts
+ * Description:
+ * This file exports functions used by main.ts to perform mathematical operations.
+ */
+
+export function add(x: number, y: number): number {
+	return x + y;
+}
+
+export function sub(x: number, y: number): number {
+	return x - y;
+}

test/fixtures/repos/calculator/tsconfig.json

@@ -0,0 +1,15 @@
+{
+	"compilerOptions": {
+		"target": "ES6",
+		"module": "commonjs",
+		"outDir": "./dist",
+		"rootDir": ".",
+		"strict": true,
+		"esModuleInterop": true,
+		"skipLibCheck": true,
+		"forceConsistentCasingInFileNames": true
+	},
+	"include": [
+		"src/**/*.ts"
+	]
+}

test/repo-context.test.ts

@@ -0,0 +1,50 @@
+import { beforeEach, describe, expect, test } from '@jest/globals'
+import { buildRepoContext } from '../src/repo-context'
+import * as path from 'node:path'
+import * as fs from 'node:fs'
+
+describe('building repo context', () => {
+	beforeEach(() => {
+		// reset git status
+	})
+
+	test('calculator repo', async () => {
+		const repoDir = path.join(__dirname, 'fixtures', 'repos', 'calculator')
+
+		// setup
+		fs.writeFileSync(
+			path.join(repoDir, '.contextinclude'),
+			`
+# README gives the intent/purpose of the repo/project
+README.md
+ 
+# include the whole of the source code as context as it's small enough
+src/
+`,
+		)
+
+		// execute
+		const context = await buildRepoContext(repoDir)
+
+		// assert
+		expect(context).toEqual(
+			`
+The repository contains the following list of files:
+.gitignore
+package.json
+README.md
+tsconfig.json
+src/index.ts
+src/utils.ts
+
+README.md contents:
+${fs.readFileSync(path.join(repoDir, 'README.md'))}
+
+src/index.ts contents:
+${fs.readFileSync(path.join(repoDir, 'src', 'index.ts'))}
+
+src/utils.ts contents:
+${fs.readFileSync(path.join(repoDir, 'src', 'utils.ts'))}`.trimStart(),
+		)
+	})
+})