class RuleMatcher {

    public boolean canCrawl(String agent, String path, String rules) {
        // Fixes #29
        if (path == null || path.isEmpty()) path = "/";

        // if path is /robots.txt then allow always
        if ("/robots.txt".equals(path)) return true;

        Content content = gson.fromJson(rules, type);
        List<Group> groups = content.getGroups();

        // if no groups found, then allow all
        if (groups.isEmpty()) return true;

        List<String> userAgents = groups.stream()
                .flatMap(group -> group.getUserAgents().stream().map(String::toLowerCase))
                .collect(Collectors.toList());

        if (userAgents.isEmpty()) return true;
        if (userAgents.parallelStream().noneMatch(ALL::equals) && ALL.equals(agent)) return true;

        agent = agent.toLowerCase();
        if (noneMatch(agent, userAgents) && noneMatch(ALL, userAgents) && !ALL.equals(agent)) return true;

        // Flag used to check if the given user agent match any saved agent.
        // Note that if no user agent match, it will use * as base user agent
        boolean foundUserAgent = false;

        for (Group group : groups) {
            // Directive priority is decided using length, that is, longest first
            Collections.sort(group.getRules());

            for (String ua : group.getUserAgents()) {
                // Check if user agent match. User agent case is ignored as
                // google documentation says
                if (agent.contains(ua.toLowerCase())) {
                    foundUserAgent = true;
                    // If rules check, then allow
                    if (checkGroupRules(group, path)) return true;
                }
            }

            // Check with default user agent (*)
            if (!foundUserAgent) {
                for (String ua : group.getUserAgents()) {
                    // Continue loop until * found
                    if (!ALL.equals(ua)) continue;
                    if (checkGroupRules(group, path)) return true;
                }
            }
        }
        return false;
    }

}