async function scrapeGenericUrl({
  link,
  captureAs = "text",
  processAsDocument = true,
  scraperHeaders = {},
}) {
  console.log(`-- Working URL ${link} => (${captureAs}) --`);
  const content = await getPageContent({
    link,
    captureAs,
    headers: scraperHeaders,
  });

  if (!content.length) {
    console.error(`Resulting URL content was empty at ${link}.`);
    return {
      success: false,
      reason: `No URL content found at ${link}.`,
      documents: [],
    };
  }

  if (!processAsDocument) {
    return {
      success: true,
      content,
    };
  }

  const url = new URL(link);
  const decodedPathname = decodeURIComponent(url.pathname);
  const filename = `${url.hostname}${decodedPathname.replace(/\//g, "_")}`;

  const data = {
    id: v4(),
    url: "file://" + slugify(filename) + ".html",
    title: slugify(filename) + ".html",
    docAuthor: "no author found",
    description: "No description found.",
    docSource: "URL link uploaded by the user.",
    chunkSource: `link://${link}`,
    published: new Date().toLocaleString(),
    wordCount: content.split(" ").length,
    pageContent: content,
    token_count_estimate: tokenizeString(content),
  };

  const document = writeToServerDocuments(
    data,
    `url-${slugify(filename)}-${data.id}`
  );
  console.log(`[SUCCESS]: URL ${link} converted & ready for embedding.\n`);
  return { success: true, reason: null, documents: [document] };
}
Scrape a generic URL and return the content in the specified format
@param {Object} config - The configuration object
@param {string} config.link - The URL to scrape
@param {('html' | 'text')} config.captureAs - The format to capture the page content as. Default is 'text'
@param {boolean} config.processAsDocument - Whether to process the content as a document or return the content directly. Default is true
@param {{[key: string]: string}} config.scraperHeaders - Custom headers to use when making the request
@returns {Promise<Object>} - The content of the page