More robust configuration options for FetchRequest getUrl functions (#4353).

src.ts/utils/fetch.ts

@@ -23,7 +23,7 @@ import { assert, assertArgument } from "./errors.js";
 import { defineProperties } from "./properties.js";
 import { toUtf8Bytes, toUtf8String } from "./utf8.js"
 
-import { getUrl } from "./geturl.js";
+import { createGetUrl } from "./geturl.js";
 
 /**
  *  An environments implementation of ``getUrl`` must return this type.
@@ -77,7 +77,7 @@ const MAX_ATTEMPTS = 12;
 const SLOT_INTERVAL = 250;
 
 // The global FetchGetUrlFunc implementation.
-let getUrlFunc: FetchGetUrlFunc = getUrl;
+let defaultGetUrlFunc: FetchGetUrlFunc = createGetUrl();
 
 const reData = new RegExp("^data:([^;:]*)?(;base64)?,(.*)$", "i");
 const reIpfs = new RegExp("^ipfs:/\/(ipfs/)?(.*)$", "i");
@@ -201,6 +201,8 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> {
 
     #throttle: Required<FetchThrottleParams>;
 
+    #getUrlFunc: null | FetchGetUrlFunc;
+
     /**
      *  The fetch URI to requrest.
      */
@@ -429,6 +431,28 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> {
         this.#retry = retry;
     }
 
+    /**
+     *  This function is called to fetch content from HTTP and
+     *  HTTPS URLs and is platform specific (e.g. nodejs vs
+     *  browsers).
+     *
+     *  This is by default the currently registered global getUrl
+     *  function, which can be changed using [[registerGetUrl]].
+     *  If this has been set, setting is to ``null`` will cause
+     *  this FetchRequest (and any future clones) to revert back to
+     *  using the currently registered global getUrl function.
+     *
+     *  Setting this is generally not necessary, but may be useful
+     *  for developers that wish to intercept requests or to
+     *  configurege a proxy or other agent.
+     */
+    get getUrlFunc(): FetchGetUrlFunc {
+        return this.#getUrlFunc || defaultGetUrlFunc;
+    }
+    set getUrlFunc(value: null | FetchGetUrlFunc) {
+        this.#getUrlFunc = value;
+    }
+
     /**
      *  Create a new FetchRequest instance with default values.
      *
@@ -448,6 +472,8 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> {
             slotInterval: SLOT_INTERVAL,
             maxAttempts: MAX_ATTEMPTS
         };
+
+        this.#getUrlFunc = null;
     }
 
     toString(): string {
@@ -510,7 +536,7 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> {
         // We have a preflight function; update the request
         if (this.preflightFunc) { req = await this.preflightFunc(req); }
 
-        const resp = await getUrlFunc(req, checkSignal(_request.#signal));
+        const resp = await this.getUrlFunc(req, checkSignal(_request.#signal));
         let response = new FetchResponse(resp.statusCode, resp.statusMessage, resp.headers, resp.body, _request);
 
         if (response.statusCode === 301 || response.statusCode === 302) {
@@ -641,6 +667,8 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> {
         clone.#process = this.#process;
         clone.#retry = this.#retry;
 
+        clone.#getUrlFunc = this.#getUrlFunc;
+
         return clone;
     }
 
@@ -686,7 +714,22 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> {
      */
     static registerGetUrl(getUrl: FetchGetUrlFunc): void {
         if (locked) { throw new Error("gateways locked"); }
-        getUrlFunc = getUrl;
+        defaultGetUrlFunc = getUrl;
+    }
+
+    /**
+     *  Creates a getUrl function that fetches content from HTTP and
+     *  HTTPS URLs.
+     *
+     *  The available %%options%% are dependent on the platform
+     *  implementation of the default getUrl function.
+     *
+     *  This is not generally something that is needed, but is useful
+     *  when trying to customize simple behaviour when fetching HTTP
+     *  content.
+     */
+    static createGetUrlFunc(options?: Record<string, any>): FetchGetUrlFunc {
+        return createGetUrl(options);
     }
 
     /**

src.ts/utils/geturl-browser.ts

@@ -1,6 +1,8 @@
 import { assert } from "./errors.js";
 
-import type { FetchRequest, FetchCancelSignal, GetUrlResponse } from "./fetch.js";
+import type {
+    FetchGetUrlFunc, FetchRequest, FetchCancelSignal, GetUrlResponse
+} from "./fetch.js";
 
 
 declare global {
@@ -27,7 +29,9 @@ declare global {
 
 // @TODO: timeout is completely ignored; start a Promise.any with a reject?
 
-export async function getUrl(req: FetchRequest, _signal?: FetchCancelSignal): Promise<GetUrlResponse> {
+export function createGetUrl(options?: Record<string, any>): FetchGetUrlFunc {
+
+    async function getUrl(req: FetchRequest, _signal?: FetchCancelSignal): Promise<GetUrlResponse> {
         const protocol = req.url.split(":")[0].toLowerCase();
 
         assert(protocol === "http" || protocol === "https", `unsupported protocol ${ protocol }`, "UNSUPPORTED_OPERATION", {
@@ -68,5 +72,15 @@ export async function getUrl(req: FetchRequest, _signal?: FetchCancelSignal): Pr
             statusMessage: resp.statusText,
             headers, body
         };
+    }
+
+    return getUrl;
+}
+
+// @TODO: remove in v7; provided for backwards compat
+const defaultGetUrl: FetchGetUrlFunc = createGetUrl({ });
+
+export async function getUrl(req: FetchRequest, _signal?: FetchCancelSignal): Promise<GetUrlResponse> {
+    return defaultGetUrl(req, _signal);
 }
 

src.ts/utils/geturl.ts

@@ -5,12 +5,16 @@ import { gunzipSync } from "zlib";
 import { assert } from "./errors.js";
 import { getBytes } from "./data.js";
 
-import type { FetchRequest, FetchCancelSignal, GetUrlResponse } from "./fetch.js";
+import type {
+    FetchGetUrlFunc, FetchRequest, FetchCancelSignal, GetUrlResponse
+} from "./fetch.js";
 
 /**
  *  @_ignore:
  */
-export async function getUrl(req: FetchRequest, signal?: FetchCancelSignal): Promise<GetUrlResponse> {
+export function createGetUrl(options?: Record<string, any>): FetchGetUrlFunc {
+
+    async function getUrl(req: FetchRequest, signal?: FetchCancelSignal): Promise<GetUrlResponse> {
 
         const protocol = req.url.split(":")[0].toLowerCase();
 
@@ -26,9 +30,12 @@ export async function getUrl(req: FetchRequest, signal?: FetchCancelSignal): Pro
         const method = req.method;
         const headers = Object.assign({ }, req.headers);
 
-    const options: any = { method, headers };
+        const reqOptions: any = { method, headers };
+        if (options) {
+            if (options.agent) { reqOptions.agent = options.agent; }
+        }
 
-    const request = ((protocol === "http") ? http: https).request(req.url, options);
+        const request = ((protocol === "http") ? http: https).request(req.url, reqOptions);
 
         request.setTimeout(req.timeout);
 
@@ -92,5 +99,18 @@ export async function getUrl(req: FetchRequest, signal?: FetchCancelSignal): Pro
 
             request.on("error", (error) => { reject(error); });
         });
+    }
+
+    return getUrl;
+}
+
+// @TODO: remove in v7; provided for backwards compat
+const defaultGetUrl: FetchGetUrlFunc = createGetUrl({ });
+
+/**
+ *  @_ignore:
+ */
+export async function getUrl(req: FetchRequest, signal?: FetchCancelSignal): Promise<GetUrlResponse> {
+    return defaultGetUrl(req, signal);
 }