Switch diff dependency for semantic cleanup

package-lock.json

@@ -16,7 +16,8 @@
 				"@langchain/core": "^0.3.16",
 				"@langchain/ollama": "^0.1.1",
 				"@langchain/openai": "^0.3.11",
-				"diff": "^7.0.0",
+				"@types/diff-match-patch": "^1.0.36",
+				"diff-match-patch": "^1.0.5",
 				"dotenv": "^16.4.5",
 				"jest-diff": "^29.7.0",
 				"simple-text-diff": "^1.7.0"
@@ -767,6 +768,11 @@
 			"integrity": "sha512-K0Oqlrq3kQMaO2RhfrNQX5trmt+XLyom88zS0u84nnIcLvFnRUMRRHmrGny5GSM+kNO9IZLARsdQHDzkhAgmrQ==",
 			"dev": true
 		},
+		"node_modules/@types/diff-match-patch": {
+			"version": "1.0.36",
+			"resolved": "https://registry.npmjs.org/@types/diff-match-patch/-/diff-match-patch-1.0.36.tgz",
+			"integrity": "sha512-xFdR6tkm0MWvBfO8xXCSsinYxHcqkQUlcHeSpMC2ukzOb6lwQAfDmW+Qt0AvlGd8HpsS28qKsB+oPeJn9I39jg=="
+		},
 		"node_modules/@types/estree": {
 			"version": "1.0.6",
 			"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz",
@@ -1299,13 +1305,10 @@
 				"node": ">=0.4.0"
 			}
 		},
-		"node_modules/diff": {
-			"version": "7.0.0",
-			"resolved": "https://registry.npmjs.org/diff/-/diff-7.0.0.tgz",
-			"integrity": "sha512-PJWHUb1RFevKCwaFA9RlG5tCd+FO5iRh9A8HEtkmBH2Li03iJriB6m6JIN4rGz3K3JLawI7/veA1xzRKP6ISBw==",
-			"engines": {
-				"node": ">=0.3.1"
-			}
+		"node_modules/diff-match-patch": {
+			"version": "1.0.5",
+			"resolved": "https://registry.npmjs.org/diff-match-patch/-/diff-match-patch-1.0.5.tgz",
+			"integrity": "sha512-IayShXAgj/QMXgB0IWmKx+rOPuGMhqm5w6jvFxmVenXKIzRqTAAsbBPT3kWQeGANj3jGgvcvv4yK6SxqYmikgw=="
 		},
 		"node_modules/diff-sequences": {
 			"version": "29.6.3",

package.json

@@ -30,7 +30,8 @@
 		"@langchain/core": "^0.3.16",
 		"@langchain/ollama": "^0.1.1",
 		"@langchain/openai": "^0.3.11",
-		"diff": "^7.0.0",
+		"@types/diff-match-patch": "^1.0.36",
+		"diff-match-patch": "^1.0.5",
 		"dotenv": "^16.4.5",
 		"jest-diff": "^29.7.0",
 		"simple-text-diff": "^1.7.0"

src/modules/diffExtension.ts

@@ -12,13 +12,12 @@ import {
     ViewPlugin,
     ViewUpdate,
 } from "@codemirror/view";
-import { diffWords, Change } from "diff";
+import DiffMatchPatch from "diff-match-patch";
 
 import { acceptTooltipEffect, dismissTooltipEffect } from "./WidgetExtension";
 import { generatedResponseState, setGeneratedResponseEffect } from "./AIExtension";
 import { currentSelectionState } from "./SelectionSate";
 
-
 /**
  * Widget to display added or removed content.
  * Improves accessibility by using appropriate ARIA attributes.
@@ -51,7 +50,8 @@ class ChangeContentWidget extends WidgetType {
 }
 
 /**
- * Generates a DecorationSet representing the diff between AI response and context.
+ * Generates a DecorationSet representing the diff between AI response and context,
+ * using diff_match_patch with semantic cleanup.
  * @param state - The current editor state.
  * @returns A DecorationSet with the appropriate widgets.
  */
@@ -69,27 +69,34 @@ function generateDiffView(state: EditorState): DecorationSet {
             aiResponse: response,
             contextText: context,
         });
-        const diffResult: Change[] = diffWords(contextText, aiText);
+
+        // Use diff_match_patch instead of diffWords
+        const dmp = new DiffMatchPatch();
+        let diffs = dmp.diff_main(contextText, aiText);
+
+        // Perform semantic cleanup
+        dmp.diff_cleanupSemantic(diffs);
 
         // Initialize RangeSetBuilder for efficient decoration construction
         const builder = new RangeSetBuilder<Decoration>();
         let currentPos = context?.from ?? 0;
 
-        diffResult.forEach((part) => {
-            const { added, removed, value } = part;
-            const length = value.length;
 
-            if (added) {
-                // Highlight added text (AI response)
-                const widget = new ChangeContentWidget(value, 'added');
+        diffs.forEach(([op, text]) => {
+            const length = text.length;
+
+            if (op === DiffMatchPatch.DIFF_INSERT) {
+                // AI text added
+                const widget = new ChangeContentWidget(text, "added");
                 builder.add(
                     currentPos,
                     currentPos,
                     Decoration.widget({ widget, side: 1 })
                 );
-            } else if (removed) {
-                // Highlight removed text (from context)
-                const widget = new ChangeContentWidget(value, 'removed');
+            } else if (op === DiffMatchPatch.DIFF_DELETE) {
+                // Context text removed
+                const widget = new ChangeContentWidget(text, "removed");
+                // Attach the widget over the removed range
                 builder.add(
                     currentPos,
                     currentPos + length,
@@ -97,6 +104,7 @@ function generateDiffView(state: EditorState): DecorationSet {
                 );
                 currentPos += length;
             } else {
+                // No change, move currentPos forward
                 currentPos += length;
             }
         });
@@ -128,13 +136,11 @@ function dispatchAIChanges(state: EditorState, view: EditorView): void {
         view.dispatch({
             changes: { from: selectionFrom, to: selectionTo, insert: aiText },
         });
-
     } catch (error) {
         console.error("Error applying diff changes:", error);
     }
 }
 
-
 export const diffDecorationState = StateField.define<DecorationSet>({
     create(): DecorationSet {
         return Decoration.none;

tsconfig.json

@@ -11,7 +11,8 @@
 		"importHelpers": true,
 		"isolatedModules": true,
 		"strict": true,
-		"lib": ["DOM", "ESNext"]
+		"lib": ["DOM", "ESNext"],
+		"allowSyntheticDefaultImports": true
 	},
 	"include": ["src/**/*"],
 	"exclude": ["node_modules", "public"]