Check page URLs for extension before direct fetch attempt

src/crawler.ts

@@ -47,6 +47,7 @@ import {
   ExitCodes,
   InterruptReason,
   BxFunctionBindings,
+  DIRECT_FETCH_EXT,
 } from "./util/constants.js";
 
 import { AdBlockRules, BlockRuleDecl, BlockRules } from "./util/blockrules.js";
@@ -1012,6 +1013,22 @@ self.__bx_behaviors.selectMainBehavior();
     return "";
   }
 
+  shouldDirectFetchByExt(url: string) {
+    const urlFull = new URL(url);
+    const extParts = urlFull.pathname.split(".");
+    if (extParts.length <= 1) {
+      return false;
+    }
+    const ext = extParts[1];
+
+    // known files that should be direct fetched
+    if (DIRECT_FETCH_EXT.includes(ext)) {
+      return true;
+    }
+
+    return false;
+  }
+
   async crawlPage(opts: WorkerState): Promise<void> {
     await this.writeStats();
 
@@ -1033,7 +1050,7 @@ self.__bx_behaviors.selectMainBehavior();
     data.logDetails = logDetails;
     data.workerid = workerid;
 
-    if (recorder) {
+    if (recorder && this.shouldDirectFetchByExt(url)) {
       try {
         const headers = auth
           ? { Authorization: auth, ...this.headers }

src/util/constants.ts

@@ -22,6 +22,16 @@ export const DETECT_SITEMAP = "<detect>";
 
 export const EXTRACT_TEXT_TYPES = ["to-pages", "to-warc", "final-to-warc"];
 
+export const DIRECT_FETCH_EXT = [
+  "pdf",
+  "xml",
+  "jpg",
+  "webm",
+  "docx",
+  "mp4",
+  "zip",
+];
+
 export enum BxFunctionBindings {
   BehaviorLogFunc = "__bx_log",
   AddLinkFunc = "__bx_addLink",

src/util/recorder.ts

@@ -1361,7 +1361,11 @@ export class Recorder extends EventEmitter {
       url &&
       method === "GET" &&
       !isRedirectStatus(status) &&
-      !(await this.crawlState.addIfNoDupe(WRITE_DUPE_KEY, url, status))
+      !(await this.crawlState.addIfNoDupe(
+        WRITE_DUPE_KEY,
+        url,
+        status === 206 || !status ? 200 : status,
+      ))
     ) {
       logNetwork("Skipping dupe", { url, status, ...this.logDetails });
       return;
@@ -1515,7 +1519,11 @@ class AsyncFetcher {
       if (
         reqresp.method === "GET" &&
         url &&
-        !(await crawlState.addIfNoDupe(ASYNC_FETCH_DUPE_KEY, url, status))
+        !(await crawlState.addIfNoDupe(
+          ASYNC_FETCH_DUPE_KEY,
+          url,
+          status === 206 || !status ? 200 : status,
+        ))
       ) {
         if (!this.ignoreDupe) {
           this.reqresp.asyncLoading = false;