Unverified Commit e6a416f0 authored by Hong Minhee's avatar Hong Minhee
Browse files

Make HTTP requests with the proper `User-Agent`

parent dad1ca17

CHANGES.md

+4 −0
Original line number Diff line number Diff line
@@ -8,6 +8,10 @@ Version 1.3.0 


To be released.



 -  Fedify now makes HTTP requests with the proper `User-Agent` header.



     -  Added `getUserAgent()` function.





Version 1.2.2

-------------

src/nodeinfo/client.ts

+13 −2
Original line number Diff line number Diff line 
import { getLogger } from "@logtape/logtape";

import { parse, type SemVer } from "@std/semver";

import { getUserAgent } from "../runtime/docloader.ts";

import type { ResourceDescriptor } from "../webfinger/jrd.ts";

import type {

  InboundService,
@@ -83,7 +84,12 @@ export async function getNodeInfo( 
    let nodeInfoUrl: URL | string = url;

    if (!options.direct) {

      const wellKnownUrl = new URL("/.well-known/nodeinfo", url);

      const wellKnownResponse = await fetch(wellKnownUrl);

      const wellKnownResponse = await fetch(wellKnownUrl, {

        headers: {

          Accept: "application/json",

          "User-Agent": getUserAgent(),

        },

      });

      if (!wellKnownResponse.ok) {

        logger.error("Failed to fetch {url}: {status} {statusText}", {

          url: wellKnownUrl.href,
@@ -110,7 +116,12 @@ export async function getNodeInfo( 
      }

      nodeInfoUrl = link.href;

    }

    const response = await fetch(nodeInfoUrl);

    const response = await fetch(nodeInfoUrl, {

      headers: {

        Accept: "application/json",

        "User-Agent": getUserAgent(),

      },

    });

    if (!response.ok) {

      logger.error(

        "Failed to fetch NodeInfo document from {url}: {status} {statusText}",

src/runtime/docloader.test.ts

+62 −0
Original line number Diff line number Diff line 
import { assertEquals, assertRejects, assertThrows } from "@std/assert";

import * as mf from "mock_fetch";

import process from "node:process";

import metadata from "../deno.json" with { type: "json" };

import { MemoryKvStore } from "../federation/kv.ts";

import { verifyRequest } from "../sig/http.ts";

import { mockDocumentLoader } from "../testing/docloader.ts";
@@ -10,6 +12,7 @@ import { 
  fetchDocumentLoader,

  FetchError,

  getAuthenticatedDocumentLoader,

  getUserAgent,

  kvCache,

} from "./docloader.ts";

import { UrlError } from "./url.ts";
@@ -480,3 +483,62 @@ test("kvCache()", async (t) => { 
    );

  });

});



test("getUserAgent()", () => {

  if ("Deno" in globalThis) {

    assertEquals(

      getUserAgent(),

      `Fedify/${metadata.version} (Deno/${Deno.version.deno})`,

    );

    assertEquals(

      getUserAgent("MyApp/1.0.0"),

      `MyApp/1.0.0 (Fedify/${metadata.version}; Deno/${Deno.version.deno})`,

    );

    assertEquals(

      getUserAgent(null, "https://example.com/"),

      `Fedify/${metadata.version} (Deno/${Deno.version.deno}; +https://example.com/)`,

    );

    assertEquals(

      getUserAgent("MyApp/1.0.0", new URL("https://example.com/")),

      `MyApp/1.0.0 (Fedify/${metadata.version}; Deno/${Deno.version.deno}; +https://example.com/)`,

    );

  } else if ("Bun" in globalThis) {

    assertEquals(

      getUserAgent(),

      // @ts-ignore: `Bun` is a global variable in Bun

      `Fedify/${metadata.version} (Bun/${Bun.version})`,

    );

    assertEquals(

      getUserAgent("MyApp/1.0.0"),

      // @ts-ignore: `Bun` is a global variable in Bun

      `MyApp/1.0.0 (Fedify/${metadata.version}; Bun/${Bun.version})`,

    );

    assertEquals(

      getUserAgent(null, "https://example.com/"),

      // @ts-ignore: `Bun` is a global variable in Bun

      `Fedify/${metadata.version} (Bun/${Bun.version}; +https://example.com/)`,

    );

    assertEquals(

      getUserAgent("MyApp/1.0.0", new URL("https://example.com/")),

      // @ts-ignore: `Bun` is a global variable in Bun

      `MyApp/1.0.0 (Fedify/${metadata.version}; Bun/${Bun.version}; +https://example.com/)`,

    );

  } else {

    assertEquals(

      getUserAgent(),

      `Fedify/${metadata.version} (Node.js/${process.version})`,

    );

    assertEquals(

      getUserAgent("MyApp/1.0.0"),

      `MyApp/1.0.0 (Fedify/${metadata.version}; Node.js/${process.version})`,

    );

    assertEquals(

      getUserAgent(null, "https://example.com/"),

      `Fedify/${metadata.version} (Node.js/${process.version}; +https://example.com/)`,

    );

    assertEquals(

      getUserAgent("MyApp/1.0.0", new URL("https://example.com/")),

      `MyApp/1.0.0 (Fedify/${metadata.version}; Node.js/${process.version}; +https://example.com/)`,

    );

  }

});

src/runtime/docloader.ts

+31 −0
Original line number Diff line number Diff line 
import { HTTPHeaderLink } from "@hugoalh/http-header-link";

import { getLogger } from "@logtape/logtape";

import process from "node:process";

import metadata from "../deno.json" with { type: "json" };

import type { KvKey, KvStore } from "../federation/kv.ts";

import { signRequest } from "../sig/http.ts";

import { validateCryptoKey } from "../sig/key.ts";
@@ -75,6 +77,7 @@ function createRequest(url: string): Request { 
  return new Request(url, {

    headers: {

      Accept: "application/activity+json, application/ld+json",

      "User-Agent": getUserAgent(),

    },

    redirect: "manual",

  });
@@ -394,3 +397,31 @@ export function kvCache( 
    return cache;

  };

}



/**

 * Gets the user agent string for the given application and URL.

 * @param app An optional application name and version, e.g., `"Hollo/1.0.0"`.

 * @param url An optional URL to append to the user agent string.

 *            Usually the URL of the ActivityPub instance.

 * @returns The user agent string.

 * @since 1.3.0

 */

export function getUserAgent(

  app?: string | null,

  url?: string | URL | null,

): string {

  const fedify = `Fedify/${metadata.version}`;

  const runtime = "Deno" in globalThis

    ? `Deno/${Deno.version.deno}`

    : "Bun" in globalThis

    // @ts-ignore: `Bun` is a global variable in Bun

    ? `Bun/${Bun.version}`

    : "process" in globalThis

    ? `Node.js/${process.version}`

    : null;

  const userAgent = app == null ? [fedify] : [app, fedify];

  if (runtime != null) userAgent.push(runtime);

  if (url != null) userAgent.push(`+${url.toString()}`);

  const first = userAgent.shift();

  return `${first} (${userAgent.join("; ")})`;

}

src/webfinger/lookup.ts

+5 −1
Original line number Diff line number Diff line 
import { getLogger } from "@logtape/logtape";

import { getUserAgent } from "../runtime/docloader.ts";

import type { ResourceDescriptor } from "./jrd.ts";



const logger = getLogger(["fedify", "webfinger", "lookup"]);
@@ -34,7 +35,10 @@ export async function lookupWebFinger( 
    let response: Response;

    try {

      response = await fetch(url, {

        headers: { Accept: "application/jrd+json" },

        headers: {

          Accept: "application/jrd+json",

          "User-Agent": getUserAgent(),

        },

        redirect: "manual",

      });

    } catch (error) {