Fix: updating CSS selectors for Perplexity extraction (#286)

* fix: updating CSS content selector

* fix: targeting new source list

* fix: extracting sources list and extracting URLs using imgs <a> tag when available (as a workaround)

* attempt: capture target URL

* chore: cleaning code

* fix: switch to Search tab to extract user answer

* 3.8.8

Author: Hugo-COLLIN

package.json

@@ -1,6 +1,6 @@
 {
   "name": "save-my-chatbot",
-  "version": "3.8.7",
+  "version": "3.8.8",
   "license": "RMD-C 1.1",
   "author": "Hugo COLLIN",
   "homepage": "https://save.hugocollin.com",

public/files/updateNotes.md

@@ -1,4 +1,8 @@
 # Update notes
+# 3.8.8
+🧩 Fixing Perplexity export
+Perplexity changed its interface again, but we're now back!
+
 # 3.8.7
 🚀 Perplexity sources extraction working again!
 Perplexity changed its interface again, but the extension is now back on track!

src/core/services/format/formatMarkdown.ts

@@ -59,8 +59,11 @@ export function formatMarkdown(html: string | Node): any | string | string {
  * @returns {string} formatted link
  */
 export function formatLink(url: string | HTMLElement, message: string): string {
+  // console.log(message)
   // @ts-ignore
-  return "[" + message.replaceAll("`", "") + "](" + url?.replace(/\)/g, "%29") + ")";
+  const res = "[" + message.replaceAll("`", "") + "](" + url?.replace(/\)/g, "%29") + ")";
+  console.log(res)
+  return res;
 }
 
 /**

src/core/services/pageExtractor/extractSection.ts

@@ -1,5 +1,5 @@
 import {capitalizeFirst, formatLineBreaks} from "../format/formatText";
-import {safeExecute} from "../../utils/jsShorteners";
+import {safeExecute, sleep} from "../../utils/jsShorteners";
 import {extractSources} from "./extractSources";
 
 /**
@@ -31,6 +31,10 @@ export async function extractSection(content: { hasChildNodes: () => any; }, for
 async function extractSearchSection(content: HTMLElement, format: (arg0: any) => string, metadata: { extractor: any; sourcesExtraction?: any; }) {
   const options = metadata.extractor;
 
+  // @ts-ignore
+  content.querySelector('.whitespace-nowrap.absolute')?.click();
+  await sleep(100);
+
   // Extract and format the user question
   const userQuestionSelector = options.userQuestionSelector ?? 'span, textarea';
   const userQuestionElement = content.querySelector(userQuestionSelector) ?? "";

src/core/services/pageExtractor/extractSources.ts

@@ -18,12 +18,12 @@ export async function extractSources(content: HTMLElement, format: any, data: {
   res = "";
   i = 1;
 
-  for (const {open, close, selector, extractionType, paginationSelector, content: msgContent} of data.selectors) {
+  for (const {open, close, selector, extractionType, paginationSelector, content: msgContent, scope: scopeType } of data.selectors) {
     open && await safeExecute(await selectAndClick(open, content));
 
     switch (extractionType) {
       case 'list':
-        res = await safeExecute(await extractFromList(format, content, selector ?? msgContent)) as unknown as string;
+        res = await safeExecute(await extractFromList(format, content, selector ?? msgContent, scopeType)) as unknown as string;
         break;
       case 'tile-list':
         res = await safeExecute(await extractFromTileList(format, content, selector)) as unknown as string;
@@ -103,24 +103,27 @@ function extractFromLinks(links: any[] | NodeListOf<any>, format: (arg0: any) =>
  * @param selectorOrContent
  * @returns {Promise<string>}
  */
-async function extractFromList(format: any, content: HTMLElement, selectorOrContent: { selector: any; scope: any; }): Promise<string> {
+async function extractFromList(format: any, content: HTMLElement, selectorOrContent: { selector: any; scope: any; }, scopeType: string): Promise<string> {
   let res = '';
   let i = 1;
 
   const selector = typeof selectorOrContent === "object"
     ? selectorOrContent.selector
     : selectorOrContent;
 
-  const scope = typeof selectorOrContent === "object"
-    ? selectorOrContent.scope
-    : "document";
+  const scope = scopeType === "content"
+    ? "content"
+    : typeof selectorOrContent === "object"
+      ? selectorOrContent.scope
+      : "document";
 
   const qs = scope === "document"
     ? document.querySelectorAll(selector)
     : content.querySelectorAll(selector);
 
   // console.log(content, scope, qs)
   for (const tile of qs) {
+    // console.log("THE TILE : ", tile)
     res += await formatSources(i, format, tile);
     i++;
   }
@@ -162,11 +165,13 @@ async function extractFromTileList(format: any, content: HTMLElement, selector:
  * @returns {Promise<string>}
  */
 export async function formatSources(i: string | number, format: (arg0: any) => string, tile: Element): Promise<string> {
-  const elt: HTMLElement = tile.querySelector("div.default") as HTMLElement //Perplexity
+  const elt: HTMLElement = tile.querySelector("a") as HTMLElement //Perplexity
     || tile;
 
+  const title: HTMLElement = tile.querySelector('.font-display') as HTMLElement
+
   const text = "(" + i + ") "
-    + format(elt.innerText
+    + format(title.innerText
       .replaceAll("\n", " ")
       .replaceAll('"', '')
       .replace(/^\d+/, "") // Removes numbers at the beginning
@@ -206,53 +211,16 @@ export async function formatSources(i: string | number, format: (arg0: any) => s
 
   // Export content
   let res = "- ";
+  // console.log(tile)
   // @ts-ignore TODO
-  if (tile && tile.href)
+  // if (tile && tile.href)
     // @ts-ignore TODO
-    res += formatLink(tile.href, text) + "\n";
-  else {
-    const url: HTMLElement = await safeExecute(extractYoutubeLink(tile as HTMLElement)) as unknown as HTMLElement;
-    res += url
-      ? formatLink(url, text) + "\n"
-      : text + "\n";
-  }
+    res += (elt.href ? formatLink(elt.href, text) : text) + "\n";
+  // else {
+  //   const url: HTMLElement = await safeExecute(extractYoutubeLink(tile as HTMLElement)) as unknown as HTMLElement;
+  //   res += url
+  //     ? formatLink(url, text) + "\n"
+  //     : text + "\n";
+  // }
   return res;
 }
-
-
-// async function extractSourcesOld(msgContent, searchResults, res, format) {
-//   const buttonsInCard = msgContent[2].querySelectorAll("button");
-//   for (const btn of buttonsInCard) {
-//     if (btn.textContent.toLowerCase() === "view all search results") {
-//       // Open modal
-//       btn.click();
-//       await sleep(0); // Needed to wait for the modal to open (even if it's 0!)
-//
-//       // Export sources and all search results, put correct index in front of each link
-//       let i = 1;
-//       let allResults = "**All search results:**";
-//
-//       const dialogLinks = Array.from(document.querySelectorAll("[role='dialog'] a"));
-//       const p2Array = Array.from(searchResults);
-//       dialogLinks.forEach((link) => {
-//         // If the link is in the sources, add it to the sources with the correct index
-//         if (p2Array.find((elt) => elt.getAttribute("href") === link.getAttribute("href"))) {
-//           res += "\n- " + format(link.outerHTML).replace("[", `[(${i}) `);
-//         }
-//
-//         // Add the link to the all search results with the correct index
-//         allResults += "\n- " + format(link.outerHTML).replace("[", `[(${i}) `);
-//         i++;
-//       });
-//
-//       // Append all search results after the sources
-//       res += "\n\n" + allResults;
-//
-//       // Close modal
-//       document.querySelectorAll("[role='dialog'] [type='button']").forEach((btn) => {
-//         if (btn.textContent.toLowerCase() === "close") btn.click();
-//       });
-//     }
-//   }
-//   return res;
-// }

src/core/utils/captureTargetUrl.ts

@@ -0,0 +1,31 @@
+export function captureTargetUrl(selecteurElement: { click: () => void; }) {
+  // Stocker l'URL capturée
+  let urlCapturee = null;
+
+  // Créer un proxy pour window.open
+  const openOriginal = window.open;
+  window.open = function (url, target) {
+    // Intercepter et stocker l'URL
+    urlCapturee = url;
+
+    // Ne pas ouvrir réellement le nouvel onglet
+    console.log('URL interceptée :', urlCapturee);
+
+    // Restaurer la méthode originale immédiatement
+    window.open = openOriginal;
+
+    return null;
+  };
+  console.log(window.open)
+
+  // @ts-ignore
+  const eventHandlers = getEventListeners(selecteurElement)
+  eventHandlers.click.forEach((evt: { listener: { (this: Window, ev: MouseEvent): any; (): null; }; }) => {
+    console.log(evt)
+    removeEventListener('click', evt.listener)
+    evt.listener = () => null
+  })
+  console.log(eventHandlers)
+  selecteurElement.click();
+  throw new Error("stop")
+}

src/features/scraper/domains/Perplexity.json

@@ -3,7 +3,7 @@
     "selector": "h1"
   },
   "domainName": "Perplexity.ai",
-  "contentSelector": "main .mx-auto > div > div > div > div > div .col-span-8",
+  "contentSelector": "main .scrollable-container > div > div > div > div > div > div:not([class])",
   "turndown": {
     "init": {
       "blankReplacement": "getBlankReplacement"
@@ -40,8 +40,9 @@
           "wait": 50
         }],
         "close": [{"selector": "[data-testid=\"close-modal\"], div.w-full.relative button[aria-label].h-8:not([aria-label=\"Submit\"])", "scope": "document", "wait": 50}],
-        "selector": ".fixed > div > [class] > div > div > div > div > div > .group, div.w-full.relative > .group",
-        "extractionType": "list"
+        "selector": ".relative > div > .flex-col.flex > div.items-start",
+        "extractionType": "list",
+        "scope": "content"
       },
       {
         "open": [{"selector": "div.grid > div.flex:nth-last-of-type(1)", "scope": "content", "wait": 50}],

src/features/scraper/rules/rules.ts

@@ -90,7 +90,7 @@ export function filter_PreserveLineBreaksInPre_Claude(node: { nodeName: string;
 }
 
 export function replacement_PreserveLineBreaksInPre_Perplexity(content: any, node: { querySelector: (arg0: string) => any; }) {
-  console.log("replacement_PreserveLineBreaksInPre_Perplexity", node)
+  // console.log("replacement_PreserveLineBreaksInPre_Perplexity", node)
   const codeBlock = node.querySelector('code');
   const codeContent = codeBlock.textContent.trim();
   const codeLang = codeBlock.parentNode.parentNode.parentNode.querySelector("div").textContent.trim();