Browser disconnects unexpectedly while scraping

[yassinbahri]

So I am trying to cloud connection to scrape TradingView, but each time I run the script it fails, should I be doing this with a locally run lightpanda browser or what is the issue?

 * ------------------------------------------------
 *
 * Description:
 *  - Uses Puppeteer Core connected to a remote browser (Lightpanda Cloud or similar)
 *  - Captures TradingView chart screenshots with optional indicators
 *  - Manages a browser session with auto-timeout and a request queue
 *
 * Environment Variables:
 *  - LIGHTPANDA_TOKEN: Token for remote browser connection (required)
 *  - TV_SESSION_ID: (optional) TradingView session cookie
 *  - TV_SESSION_ID_SIGN: (optional) TradingView session signature cookie
 *  - IGNORE_INDICATORS_CHARTS: (optional) Comma-separated chart IDs to skip indicator loading
 *  - BASE_URL: (optional) Override default TradingView base URL
 */

const puppeteer = require("puppeteer-core");

const BASE_URL = process.env.BASE_URL || "https://www.tradingview.com";

let browserPage = undefined;
let browser = undefined;
let browserLastUsed = Date.now();
let isLightpanda = false;

const BROWSER_IDLE_TIMEOUT = 10 * 60 * 1000; // 10 minutes
const screenshotQueue = [];
let isProcessingScreenshot = false;

// Get or create Puppeteer browser connection
const getBrowser = async () => {
  console.log("[Puppeteer] Setup start:", new Date().toISOString());

  try {
    if (!browser) {
      const lightpandaToken = process.env.LIGHTPANDA_TOKEN;
      if (!lightpandaToken) throw new Error("LIGHTPANDA_TOKEN is required");

      console.log("[Puppeteer] Connecting to remote browser (Lightpanda)...");
      const wsEndpoint = `wss://euwest.cloud.lightpanda.io/ws?token=${lightpandaToken}`;

      browser = await puppeteer.connect({
        browserWSEndpoint: wsEndpoint,
        defaultViewport: null
      });

      isLightpanda = true;
      console.log("[Puppeteer] Connected successfully");

      browser.on("disconnected", () => {
        console.error("[Puppeteer] Browser disconnected unexpectedly");
        browser = null;
        browserPage = null;
        isLightpanda = false;
      });
    }

    // Create new page if needed
    if (!browserPage || browserPage.isClosed()) {
      console.log("[Puppeteer] Creating new page...");
      browserPage = await browser.newPage();
      await browserPage.setViewport({ width: 1920, height: 1080 });

      // Optional cookies
      if (process.env.TV_SESSION_ID && process.env.TV_SESSION_ID_SIGN) {
        await browserPage.setCookie(
          { name: "sessionid", value: process.env.TV_SESSION_ID, domain: new URL(BASE_URL).hostname, path: "/" },
          { name: "sessionid_sign", value: process.env.TV_SESSION_ID_SIGN, domain: new URL(BASE_URL).hostname, path: "/" }
        );
        console.log("[Puppeteer] Added TradingView cookies");
      }

      // Clipboard permissions
      try {
        const context = browser.defaultBrowserContext();
        await context.overridePermissions(BASE_URL, ["clipboard-read", "clipboard-write"]);
      } catch {
        console.log("[Puppeteer] Clipboard permissions not supported (non-critical)");
      }

      // Request interception (skip ads/tracking)
      await browserPage.setRequestInterception(true);
      browserPage.on("request", (request) => {
        const url = request.url();
        const blockList = ["adzerk", "doubleclick", "google-analytics", "mixpanel", "zedo"];
        if (blockList.some((e) => url.includes(e))) request.abort();
        else request.continue();
      });
    }

    console.log("[Puppeteer] Browser ready");
    return browserPage;
  } catch (error) {
    console.error("[Puppeteer] Setup failed:", error.message);
    throw error;
  }
};

// Close browser cleanly
const closeBrowser = async () => {
  try {
    if (browserPage) await browserPage.close();
    if (browser) await browser.disconnect();
    browser = undefined;
    browserPage = undefined;
    console.log("[Puppeteer] Browser closed");
  } catch (error) {
    console.error("[Puppeteer] Error closing browser:", error.message);
  }
};

// Auto-close after idle timeout
setInterval(() => {
  if (browserPage && Date.now() - browserLastUsed > BROWSER_IDLE_TIMEOUT) {
    console.log("[Puppeteer] Idle timeout reached — closing browser");
    closeBrowser();
  }
}, 2 * 60 * 1000);

// Process queue
const processScreenshotQueue = async () => {
  if (isProcessingScreenshot || screenshotQueue.length === 0) return;
  isProcessingScreenshot = true;

  const { chart, ticker, timeframe, indicator, resolve, reject } = screenshotQueue.shift();
  console.log(`[Queue] Processing screenshot (remaining: ${screenshotQueue.length})`);

  try {
    const result = await screenshotInternal(chart, ticker, timeframe, indicator);
    resolve(result);
  } catch (error) {
    reject(error);
  } finally {
    isProcessingScreenshot = false;
    if (screenshotQueue.length > 0) setImmediate(processScreenshotQueue);
  }
};

// Public screenshot API
const screenshot = async (chart, ticker, timeframe = null, indicator = null) => {
  return new Promise((resolve, reject) => {
    screenshotQueue.push({ chart, ticker, timeframe, indicator, resolve, reject });
    console.log(`[Queue] Queued screenshot (length: ${screenshotQueue.length})`);
    processScreenshotQueue();
  });
};

// Internal screenshot function
const screenshotInternal = async (chart, ticker, timeframe = null, indicator = null) => {
  const id = `${Date.now()}-${Math.random().toString(36).slice(2, 6)}`;
  console.log(`[Screenshot:${id}] Start for ${chart}/${ticker}`);

  const url = `${BASE_URL}/chart/${chart}?symbol=${ticker}${timeframe ? `&interval=${timeframe}` : ""}`;
  try {
    browserPage = browserPage || (await getBrowser());
    browserLastUsed = Date.now();

    await browserPage.goto(url, { waitUntil: "domcontentloaded" });
    await browserPage.waitForSelector("#header-toolbar-screenshot", { visible: true, timeout: 15000 });

    console.log(`[Screenshot:${id}] Chart loaded successfully`);

    // Optional: add indicator
    const ignoredCharts = (process.env.IGNORE_INDICATORS_CHARTS || "").split(",");
    if (indicator && !ignoredCharts.includes(chart)) {
      try {
        await browserPage.keyboard.press("Slash");
        await browserPage.waitForSelector("#indicators-dialog-search-input", { visible: true });
        await browserPage.type("#indicators-dialog-search-input", indicator, { delay: 20 });
        await browserPage.waitForSelector(`[data-role="list-item"][data-title*="${indicator}"]`, { visible: true });
        await browserPage.click(`[data-role="list-item"][data-title*="${indicator}"]`);
        await browserPage.keyboard.press("Escape");
        console.log(`[Screenshot:${id}] Indicator '${indicator}' added`);
      } catch {
        console.log(`[Screenshot:${id}] Failed to add indicator (non-critical)`);
      }
    }

    // Trigger TradingView screenshot
    await browserPage.keyboard.down("Alt");
    await browserPage.keyboard.press("KeyS");
    await browserPage.keyboard.up("Alt");

    await browserPage.waitForFunction(
      () => document.body.innerText.includes("copied to clipboard"),
      { timeout: 30000 }
    );

    const imageUrl = await browserPage.evaluate("navigator.clipboard.readText()");
    const imageId = imageUrl.split("/").reverse()[1];
    const finalUrl = `${BASE_URL.replace("www", "s3")}/snapshots/${imageId[0].toLowerCase()}/${imageId}.png`;

    console.log(`[Screenshot:${id}] Completed: ${finalUrl}`);
    return finalUrl;
  } catch (error) {
    console.error(`[Screenshot:${id}] Failed: ${error.message}`);
    return undefined;
  }
};

module.exports = { screenshot, closeBrowser };

output:

[QUEUE] Screenshot queued. Queue length: 1
[QUEUE] Processing screenshot request. Queue length: 0
[SCREENSHOT:1761576969880] START
[SCREENSHOT:1761576969880] Chart: *******, Ticker: ********, Timeframe: 30, Indicator: none
[SCREENSHOT:1761576969880] URL: https://www.tradingview.com/chart/*******?symbol=******&interval=30
[SCREENSHOT:1761576969880] Getting browser page...
[INFO] Setup start: 2025-10-27T14:56:09.880Z
[INFO] Connecting to remote browser...
[INFO] Connected to remote browser successfully
[INFO] Creating new page...
[INFO] New page created
[INFO] Cookies added (sessionid + sessionid_sign)
[INFO] Clipboard permissions not supported - continuing anyway
[INFO] Skipping request interception for remote browser
[INFO] Setup complete: 2025-10-27T14:56:11.019Z
[SCREENSHOT:1761576969880] Browser page ready
[SCREENSHOT:1761576969880] Navigating to chart URL...
[ERROR] Browser disconnected unexpectedly
[SCREENSHOT:1761576969880] SCREENSHOT FAILED
[ERROR] Navigating frame was detached
    at LifecycleWatcher.js
    at CdpFrame.goto
    at screenshotInternal
    at processScreenshotQueue
[SCREENSHOT:1761576969880] Full error: { "cause": {} }
[INFO] Screenshot failed - returned undefined
[INFO] Screenshot request processing complete (partial failure)

[krichprollsch]

Hello @yassinbahri,

Did you check the browser's logs on https://console.lightpanda.io?

[yassinbahri]

Hey @krichprollsch , I have checked it and it's empty just this:
2025-10-27T14:21:57.316016631Z 2025/10/27 14:21:57 INFO command args="lightpanda serve --host 127.0.0.1 --port 9223 --log_format logfmt --timeout 400 --http_proxy http://127.0.0.1:8001"

[krichprollsch]

@yassinbahri I'm testing w/ a lightpanda locally, one issue is with the await context.overridePermissions line which we don't support and return a CDP error.
It seems after the call, CDP disconnects and set browserPage to null, so the request interception and all following calls are failing.

Can you comment the part above and make a retry?

//      // Clipboard permissions
//      try {
//        const context = browser.defaultBrowserContext();
//        await context.overridePermissions(BASE_URL, ["clipboard-read", "clipboard-write"]);
//      } catch {
//         console.log("[Puppeteer] Clipboard permissions not supported (non-critical)");
//       }

With this change, I get the following output (similar to what I get w/ chrome)

$ node test.js
[Queue] Queued screenshot (length: 1)
[Queue] Processing screenshot (remaining: 0)
[Screenshot:1761658180495-6i2m] Start for your-chart-id/NASDAQ:AAPL
[Puppeteer] Setup start: 2025-10-28T13:29:40.495Z
[Puppeteer] Connecting to remote browser (Lightpanda)...
[Puppeteer] Connected successfully
[Puppeteer] Creating new page...
[Puppeteer] Browser ready
[Screenshot:1761658180495-6i2m] Failed: Waiting for selector `#header-toolbar-screenshot` failed
Screenshot URL: undefined
[Puppeteer] Browser disconnected unexpectedly
[Puppeteer] Browser closed

[yassinbahri]

@krichprollsch thank you so much that is a problem but now I see this in the lightpanda logs

2025-10-29T14:59:32.304934479Z 2025/10/29 14:59:32 INFO command args="lightpanda serve --host 127.0.0.1 --port 9223 --log_format logfmt --timeout 400 --http_proxy http://127.0.0.1:8001"
2025-10-29T15:19:24.423842768Z $time=25771681993 $scope=user_script $level=warn $msg="eval script" url=https://www.tradingview.com/chart/IasrCR2q/?symbol=EURCAD&interval=30 err="ReferenceError: WebSocket is not defined" cacheable=false
2025-10-29T15:19:24.945196438Z $time=25771682515 $scope=user_script $level=warn $msg="eval script" url=https://static.tradingview.com/static/bundles/main_chart.f6312c7bf77ab88821ae.js err="TypeError: Cannot read properties of undefined (reading 'onReconnect')" cacheable=true
2025-10-29T15:22:05.900457836Z $time=25771843470 $scope=user_script $level=warn $msg="eval script" url=https://www.tradingview.com/chart/IasrCR2q/?symbol=EURCAD&interval=30 err="ReferenceError: WebSocket is not defined" cacheable=false
2025-10-29T15:22:06.571121978Z $time=25771844140 $scope=user_script $level=warn $msg="eval script" url=https://static.tradingview.com/static/bundles/main_chart.f6312c7bf77ab88821ae.js err="TypeError: Cannot read properties of undefined (reading 'onReconnect')" cacheable=true
2025-10-29T15:22:06.948849291Z thread 14 panic: integer overflow
2025-10-29T15:22:07.241167770Z /home/runner/work/browser/browser/src/browser/html/window.zig:349:9: 0x250d53f in createTimeout (lightpanda)
2025-10-29T15:22:07.241244539Z /home/runner/work/browser/browser/src/browser/html/window.zig:223:34: 0x23da7d2 in callback (lightpanda)
2025-10-29T15:22:07.241304688Z ???:?:?: 0x3b49b46 in ??? (???)
2025-10-29T15:22:07.257926337Z Unwind error at address `:0x3b49b46` (error.MissingFDE), trace may be incomplete
2025-10-29T15:22:07.257966326Z 
2025-10-29T15:22:07.257973956Z ???:?:?: 0x3b480d6 in ??? (???)
2025-10-29T15:22:07.257981106Z ???:?:?: 0x3b8a46d in ??? (???)
2025-10-29T15:22:07.257991756Z ???:?:?: 0x3c728e9 in ??? (???)
2025-10-29T15:22:07.257999336Z ???:?:?: 0x3b7824c in ??? (???)
2025-10-29T15:22:07.258042005Z ???:?:?: 0x3b450ea in ??? (???)
2025-10-29T15:22:07.258110614Z ???:?:?: 0x27d39c7 in ??? (???)
2025-10-29T15:22:07.258135013Z ???:?:?: 0x27d43c0 in ??? (???)
2025-10-29T15:22:07.258180163Z ???:?:?: 0x27d4538 in ??? (???)
2025-10-29T15:22:07.258221122Z ???:?:?: 0x3985e79 in ??? (???)
2025-10-29T15:22:07.258269771Z ???:?:?: 0x3985cd1 in ??? (???)
2025-10-29T15:22:07.258519177Z /home/runner/.cache/zig/p/v8-0.0.0-xddH63bVAwBSEobaUok9J0er1FqsvEujCDDVy6ItqKQ5/src/v8.zig:363:50: 0x231cfea in processMessage__anon_102475 (lightpanda)
2025-10-29T15:22:07.258704353Z /home/runner/work/browser/browser/src/cdp/cdp.zig:163:32: 0x22cefb5 in processData (lightpanda)
2025-10-29T15:22:07.258849931Z /home/runner/work/browser/browser/src/server.zig:243:32: 0x22d20a0 in readSocket (lightpanda)
2025-10-29T15:22:07.259006568Z /home/runner/work/browser/browser/src/server.zig:151:46: 0x22d5a32 in run (lightpanda)
2025-10-29T15:22:07.259175285Z /home/runner/work/browser/browser/src/main.zig:150:23: 0x2302556 in run (lightpanda)
2025-10-29T15:22:07.259328272Z /home/runner/work/browser/browser/src/main.zig:45:8: 0x2309b17 in main (lightpanda)