async _checkAllowedRobots(options, depth, previousUrl) {
    if (!options.obeyRobotsTxt) return true;
    const robot = await this._getRobot(options, depth, previousUrl);
    const userAgent = await this._getUserAgent(options);
    return robot.isAllowed(options.url, userAgent);
  }
@param {!Object} options
@param {!number} depth
@param {string} previousUrl
@return {!Promise<!boolean>}
@private