Unverified Commit cfe509f3 authored by Hong Minhee's avatar Hong Minhee
Browse files

Custom `User-Agent` headers 

https://github.com/dahlia/fedify/issues/162
parent e6a416f0

CHANGES.md

+21 −1
Original line number Diff line number Diff line
@@ -8,9 +8,29 @@ Version 1.3.0 


To be released.



 -  Fedify now makes HTTP requests with the proper `User-Agent` header.

 -  Fedify now makes HTTP requests with the proper `User-Agent` header. [[#162]]



     -  Added `getUserAgent()` function.

     -  Added `GetUserAgentOptions` interface.

     -  Added `getDocumentLoader()` function.

     -  Added `GetDocumentLoaderOptions` interface.

     -  The type of `getAuthenticatedDocumentLoader()` function's second

        parameter became `GetAuthenticatedDocumentLoaderOptions | undefined`

        (was `boolean | undefined`).

     -  Added `GetAuthenticatedDocumentLoaderOptions` interface.

     -  Deprecated `fetchDocumentLoader()` function.

     -  Added `LookupObjectOptions.userAgent` option.

     -  Added the type of `getActorHandle()` function's second parameter became

        `GetActorHandleOptions | undefined` (was `NormalizeActorHandleOptions |

        undefined`).

     -  Added `GetActorHandleOptions` interface.

     -  Added the optional second parameter to `lookupWebFinger()` function.

     -  Added `LookupWebFingerOptions` interface.

     -  Added `GetNodeInfoOptions.userAgent` option.

     -  Added `-u`/--user-agent` option to `fedify lookup` subcommand.

     -  Added `-u`/--user-agent` option to `fedify node` subcommand.



[#162]: https://github.com/dahlia/fedify/issues/162





Version 1.2.2

cli/docloader.ts

+19 −10
Original line number Diff line number Diff line 
import {

  type DocumentLoader,

  fetchDocumentLoader,

  getDocumentLoader as getDefaultDocumentLoader,

  kvCache,

} from "@fedify/fedify";

import { DenoKvStore } from "@fedify/fedify/x/denokv";

import { join } from "@std/path";

import { getCacheDir } from "./cache.ts";



let documentLoader: DocumentLoader | undefined = undefined;

const documentLoaders: Record<string, DocumentLoader> = {};



export async function getDocumentLoader(): Promise<DocumentLoader> {

  if (documentLoader) return documentLoader;

export interface DocumentLoaderOptions {

  userAgent?: string;

}



export async function getDocumentLoader(

  { userAgent }: DocumentLoaderOptions = {},

): Promise<DocumentLoader> {

  if (documentLoaders[userAgent ?? ""]) return documentLoaders[userAgent ?? ""];

  const path = join(await getCacheDir(), "kv");

  const kv = new DenoKvStore(await Deno.openKv(path));

  return documentLoader = kvCache({

  return documentLoaders[userAgent ?? ""] = kvCache({

    kv,

    rules: [

      [
@@ -50,12 +56,15 @@ export async function getDocumentLoader(): Promise<DocumentLoader> { 
        Temporal.Duration.from({ seconds: 0 }),

      ],

    ],

    loader(url) {

      return fetchDocumentLoader(url, true);

    },

    loader: getDefaultDocumentLoader({

      allowPrivateAddress: true,

      userAgent,

    }),

  });

}



export function getContextLoader(): Promise<DocumentLoader> {

  return getDocumentLoader();

export function getContextLoader(

  options: DocumentLoaderOptions = {},

): Promise<DocumentLoader> {

  return getDocumentLoader(options);

}

cli/lookup.ts

+12 −3
Original line number Diff line number Diff line
@@ -31,14 +31,19 @@ export const command = new Command() 
  .option("-e, --expand", "Expand the fetched JSON-LD document.", {

    conflicts: ["raw", "compact"],

  })

  .option("-u, --user-agent <string>", "The custom User-Agent header value.")

  .action(async (options, url: string) => {

    const spinner = ora({

      text: "Looking up the object...",

      discardStdin: false,

    }).start();

    let server: TemporaryServer | undefined = undefined;

    const documentLoader = await getDocumentLoader();

    const contextLoader = await getContextLoader();

    const documentLoader = await getDocumentLoader({

      userAgent: options.userAgent,

    });

    const contextLoader = await getContextLoader({

      userAgent: options.userAgent,

    });

    let authLoader: DocumentLoader | undefined = undefined;

    if (options.authorizedFetch) {

      spinner.text = "Generating a one-time key pair...";
@@ -87,7 +92,11 @@ export const command = new Command() 
      spinner.text = "Looking up the object...";

      const object = await lookupObject(

        url,

        { documentLoader: authLoader ?? documentLoader, contextLoader },

        {

          documentLoader: authLoader ?? documentLoader,

          contextLoader,

          userAgent: options.userAgent,

        },

      );

      spinner.succeed();

      if (object == null) {

cli/node.ts

+24 −6
Original line number Diff line number Diff line 
import { colors } from "@cliffy/ansi";

import { Command } from "@cliffy/command";

import { formatSemVer, getNodeInfo } from "@fedify/fedify";

import { formatSemVer, getNodeInfo, getUserAgent } from "@fedify/fedify";

import { createJimp } from "@jimp/core";

import webp from "@jimp/wasm-webp";

import { getLogger } from "@logtape/logtape";
@@ -34,6 +34,7 @@ export const command = new Command() 
    "Print metadata fields of the NodeInfo document.",

    { conflicts: ["raw"] },

  )

  .option("-u, --user-agent <string>", "The custom User-Agent header value.")

  .action(async (options, host: string) => {

    const spinner = ora({

      text: "Fetching a NodeInfo document...",
@@ -41,7 +42,10 @@ export const command = new Command() 
    }).start();

    const url = new URL(URL.canParse(host) ? host : `https://${host}/`);

    if (options.raw) {

      const nodeInfo = await getNodeInfo(url, { parse: "none" });

      const nodeInfo = await getNodeInfo(url, {

        parse: "none",

        userAgent: options.userAgent,

      });

      if (nodeInfo === undefined) {

        spinner.fail("No NodeInfo document found.");

        console.error("No NodeInfo document found.");
@@ -53,6 +57,7 @@ export const command = new Command() 
    }

    const nodeInfo = await getNodeInfo(url, {

      parse: options.bestEffort ? "best-effort" : "strict",

      userAgent: options.userAgent,

    });

    logger.debug("NodeInfo document: {nodeInfo}", { nodeInfo });

    if (nodeInfo == undefined) {
@@ -70,8 +75,14 @@ export const command = new Command() 
    if (options.favicon) {

      spinner.text = "Fetching the favicon...";

      try {

        const faviconUrl = await getFaviconUrl(url);

        const response = await fetch(faviconUrl);

        const faviconUrl = await getFaviconUrl(url, options.userAgent);

        const response = await fetch(faviconUrl, {

          headers: {

            "User-Agent": options.userAgent == null

              ? getUserAgent()

              : options.userAgent,

          },

        });

        if (response.ok) {

          const contentType = response.headers.get("Content-Type");

          let buffer: ArrayBuffer = await response.arrayBuffer();
@@ -208,8 +219,15 @@ const LINK_REGEXP = 
  /<link((?:\s+(?:[-a-z]+)=(?:"[^"]*"|'[^']*'|[^\s]+))*)\s*\/?>/ig;

const LINK_ATTRS_REGEXP = /(?:\s+([-a-z]+)=("[^"]*"|'[^']*'|[^\s]+))/ig;



async function getFaviconUrl(url: string | URL): Promise<URL> {

  const response = await fetch(url);

async function getFaviconUrl(

  url: string | URL,

  userAgent?: string,

): Promise<URL> {

  const response = await fetch(url, {

    headers: {

      "User-Agent": userAgent == null ? getUserAgent() : userAgent,

    },

  });

  const text = await response.text();

  for (const match of text.matchAll(LINK_REGEXP)) {

    const attrs: Record<string, string> = {};

docs/cli.md

+28 −0
Original line number Diff line number Diff line
@@ -639,6 +639,20 @@ Person { 
}

~~~~



### `-u`/`--user-agent`: Custom `User-Agent` header



*This option is available since Fedify 1.3.0.*



By default, the `fedify lookup` command sends the `User-Agent` header with the

value `Fedify/1.3.0 (Deno/2.0.4)` (version numbers may vary).  You can specify

a custom `User-Agent` header by using the `-u`/`--user-agent` option.  For

example, to send the `User-Agent` header with the value `MyApp/1.0`, run the

below command:



~~~~ sh

fedify lookup --user-agent MyApp/1.0 @fedify@hollo.social

~~~~





`fedify inbox`: Ephemeral inbox server

--------------------------------------
@@ -797,6 +811,20 @@ instance. 
The `-m`/`--metadata` option is used to show the extra metadata of the NodeInfo,

i.e., the `metadata` field of the document.



### `-u`/`--user-agent`: Custom `User-Agent` header



*This option is available since Fedify 1.3.0.*



By default, the `fedify node` command sends the `User-Agent` header with the

value `Fedify/1.3.0 (Deno/2.0.4)` (version numbers may vary).  You can specify

a custom `User-Agent` header by using the `-u`/`--user-agent` option.  For

example, to send the `User-Agent` header with the value `MyApp/1.0`, run the

below command:



~~~~ sh

fedify node --user-agent MyApp/1.0 mastodon.social

~~~~





`fedify tunnel`: Exposing a local HTTP server to the public internet

--------------------------------------------------------------------