feat: add the dependencies

package.json

@@ -9,12 +9,15 @@
     "typescript": "^5.0.0"
   },
   "dependencies": {
+    "deep-diff": "^1.0.2",
     "jsdom": "^26.1.0"
   },
   "scripts": {
     "extract": "DETECTION_MODE=extract bun run src/detect_section_selector_masters/index.ts",
     "detect": "DETECTION_MODE=detect bun run src/detect_section_selector_masters/index.ts",
     "fix": "DETECTION_MODE=fix bun run src/detect_section_selector_masters/index.ts",
-    "nav": "DETECTION_MODE=extractNav bun run src/detect_section_selector_masters/index.ts"
+    "nav": "DETECTION_MODE=extractNav bun run src/detect_section_selector_masters/index.ts",
+    "diff": "bun run src/detect_section_selector_masters/diffWithDeepDiff.ts",
+    "diff:ai": "BASELINE_ID=29279960 npm run diff"
   }
 }
\ No newline at end of file

src/detect_section_selector_masters/diffWithDeepDiff.ts

+// @ts-nocheck
+import { diff } from 'deep-diff';
+import { readFileSync, writeFileSync, mkdirSync, rmSync } from 'fs';
+import { join } from 'path';
+import { section_selectors } from '../constant/section_selectors';
+
+// Directory containing JSON outputs to compare
+const srcDir = 'src/detect_section_selector_masters/extractOutput';
+// Directory to write diff results
+const outDir = 'src/detect_section_selector_masters/diffWithBaseline';
+
+// Remove existing diffWithBaseline folder if it exists to start fresh
+rmSync(outDir, { recursive: true, force: true });
+
+// Ensure output directory exists
+mkdirSync(outDir, { recursive: true });
+
+// Baseline section ID (override via BASELINE_ID env variable)
+const EN_SITE_ID = 27328697;
+const baselineId = process.env.BASELINE_ID ? Number(process.env.BASELINE_ID) : EN_SITE_ID;
+const baselineFile = `${baselineId}_extract.json`;
+
+// Read baseline data
+const baselinePath = join(srcDir, baselineFile);
+const baselineData = JSON.parse(readFileSync(baselinePath, 'utf-8'));
+
+// Fields to ignore during diff
+const ignoreFields = ['pageUid', 'pageTitle', 'id'];
+
+section_selectors
+  .filter(id => Number(id) !== baselineId)
+  .forEach(otherId => {
+    const otherFile = `${otherId}_extract.json`;
+    const otherPath = join(srcDir, otherFile);
+    const otherData = JSON.parse(readFileSync(otherPath, 'utf-8'));
+
+    const changes = diff(baselineData, otherData, (path, key) => ignoreFields.includes(key)) || [];
+
+    // Annotate each change with context
+    const annotated = changes.map(change => {
+      const context: any = {};
+      const first = change.path && change.path[0];
+      if (typeof first === 'number') {
+        const pageA = baselineData[first] || {};
+        const pageB = otherData[first] || {};
+        context.baselineData = { pageUid: pageA.pageUid, pageTitle: pageA.pageTitle, sectionIndex: pageA.sectionIndex };
+        context.testData = { pageUid: pageB.pageUid, pageTitle: pageB.pageTitle, sectionIndex: pageB.sectionIndex };
+        const idx = change.path.indexOf('result');
+        if (idx !== -1) {
+          const itemIdx = change.path[idx + 1];
+          if (typeof itemIdx === 'number') {
+            context.baselineData.id = pageA.result?.[itemIdx]?.id;
+            context.testData.id = pageB.result?.[itemIdx]?.id;
+          }
+        }
+      }
+      return { ...change, context };
+    });
+
+    // Convert to semantic format
+    const semantic = annotated.map(change => {
+      const map: Record<string, string> = { E: 'modified', N: 'added', D: 'deleted', A: 'arrayChange' };
+      const type = map[change.kind] || change.kind;
+      const pathStr = (change.path || []).map(seg => typeof seg === 'number' ? `[${seg}]` : seg).join('.');
+      let oldV: any, newV: any;
+      if (change.kind === 'E') { oldV = (change as any).lhs; newV = (change as any).rhs; }
+      else if (change.kind === 'N') { newV = (change as any).rhs; }
+      else if (change.kind === 'D') { oldV = (change as any).lhs; }
+      else if (change.kind === 'A') { oldV = (change as any).item.lhs; newV = (change as any).item.rhs; }
+      return { changeType: type, path: pathStr, oldValue: oldV, newValue: newV, context: change.context };
+    });
+
+    const outFile = `${baselineId}_vs_${otherId}_diff.json`;
+    const outPath = join(outDir, outFile);
+    writeFileSync(outPath, JSON.stringify(semantic, null, 2), 'utf-8');
+    console.log(`✅ Diff between ${baselineId} and ${otherId} written to ${join(outDir, outFile)}`);
+  });
\ No newline at end of file

src/detect_section_selector_masters/index.ts

-import { writeFileSync, mkdirSync, existsSync } from 'fs'
+import { writeFileSync, mkdirSync, existsSync, rmSync } from 'fs'
 import { fetchSiteData } from '../clients/bobcat/SiteInfo'
 import {
     section_selectors,
@@ -29,7 +29,11 @@ if (commandMode === 'detect') {
     outDir = 'src/detect_section_selector_masters/extractNavOutput'
 }
 
-if (!existsSync(outDir)) mkdirSync(outDir, { recursive: true })
+// Remove existing diffWithBaseline folder if it exists to start fresh
+rmSync(outDir, { recursive: true, force: true })
+
+// Ensure output directory exists
+mkdirSync(outDir, { recursive: true })
 
 await Promise.all(
     section_selectors.map(async (siteId, index) => {