static determineIfCrawlableForUserAgent(userAgent, mainResource, metaElements,
      parsedRobotsTxt, robotsTxtUrl) {
    /** @type {LH.Audit.Details.Table['items']} */
    const blockingDirectives = [];

    // Prefer a meta element specific to a user agent, fallback to generic 'robots' if not present.
    // https://developers.google.com/search/blog/2007/03/using-robots-meta-tag#directing-a-robots-meta-tag-specifically-at-googlebot
    let meta;
    if (userAgent) meta = metaElements.find(meta => meta.name === userAgent.toLowerCase());
    if (!meta) meta = metaElements.find(meta => meta.name === 'robots');
    if (meta) {
      const blockingDirective = IsCrawlable.handleMetaElement(meta);
      if (blockingDirective) blockingDirectives.push(blockingDirective);
    }

    for (const header of mainResource.responseHeaders || []) {
      if (header.name.toLowerCase() !== ROBOTS_HEADER) continue;

      const directiveUserAgent = getUserAgentFromHeaderDirectives(header.value);
      if (directiveUserAgent !== userAgent && directiveUserAgent !== undefined) continue;

      let directiveWithoutUserAgentPrefix = header.value.trim();
      if (userAgent && header.value.startsWith(`${userAgent}:`)) {
        directiveWithoutUserAgentPrefix = header.value.replace(`${userAgent}:`, '');
      }
      if (!hasBlockingDirective(directiveWithoutUserAgentPrefix)) continue;

      blockingDirectives.push({source: `${header.name}: ${header.value}`});
    }

    if (parsedRobotsTxt && !parsedRobotsTxt.isAllowed(mainResource.url, userAgent)) {
      const line = parsedRobotsTxt.getMatchingLineNumber(mainResource.url) || 1;
      blockingDirectives.push({
        source: {
          type: /** @type {const} */ ('source-location'),
          url: robotsTxtUrl.href,
          urlProvider: /** @type {const} */ ('network'),
          line: line - 1,
          column: 0,
        },
      });
    }

    return blockingDirectives;
  }
@param {string|undefined} userAgent
@param {LH.Artifacts.NetworkRequest} mainResource
@param {LH.Artifacts.MetaElement[]} metaElements
@param {import('robots-parser').Robot|undefined} parsedRobotsTxt
@param {URL} robotsTxtUrl