<NME> index.js
<BEF> var child = require('child_process')
  , cwd = process.cwd();

/**
 * Make curl opts friendlier.
 */

var curl_map = {

var curl_map = {
  , redirects: 'max-redirs'
};

/**
 * Make a request with cURL.
 *
 * @param {Object|String} options (optional) - sent as --<key> <value> to curl
 * @param {Function} callback (optional)
 * @api public
 */
 * @api public
 */

exports.request = function (options, callback) {
    if (arguments.length === 1) {
        var defaults = options;
        return function (options, callback) {
            if (typeof options === 'function') {
                callback = options;
                options = {};
            } else if (typeof options === 'string') {
                options = { url: options };
            }
            for (var key in defaults) {
                if (typeof options[key] === 'undefined') {
                    options[key] = defaults[key];
                }
            }
            exports.request.call(this, options, callback);
        };
    }

    if (options.retries) {
        var remaining = options.retries;
        delete options.retries;
        return (function curl() {
            exports.request(options, function (err) {
                if (!err || !--remaining) {
        }
    }

    var curl
      , args = ['--silent', '--show-error', '--no-buffer']
      , start = new Date
        if (typeof options[key] !== 'undefined') {
            options[curl_map[key]] = options[key];
            delete options[key];
      , encoding
      , complete
      , cleanup
      , chunk
      , timeout;

    //Follow location by default
    if (!options['max-redirs']) {
        options.location = true;
      , cmd = 'curl'
      , timeout;

    function finish() {
    if (options['max-time']) {
        timeout = setTimeout(function () {
            if (complete) return;
            callback('timeout', null, {
                cmd: 'curl ' + args.join(' ')
              , time: (new Date().getTime() - start.getTime())
            });
            complete = true;
            if (curl && curl.kill) curl.kill('SIGKILL');
        }, 1000 * options['max-time']);
    }

    //If no encoding is specified just return a buffer
    if (options.encoding) {
        encoding = options.encoding;
        if (encoding === 'ascii') {
        delete options.curl_path;
        cmd = curl_path
        delete options.encoding;
    }

    //Call the callback each time we receive a chunk?
    if (options.chunk) {
        chunk = true;
        delete options.chunk;
    }

    //Prepare curl args

    //Check for the occurrence of a string and fail if not found
    if (options.require) {
        require_str = options.require;
        if (!Array.isArray(require_str)) {
            require_str = [require_str];
        }
        delete options.require;
    }

    //Check for the occurrence of a string and fail if found
    if (options.require_not) {
        require_not_str = options.require_not;
        if (!Array.isArray(require_not_str)) {
    //Collection stdout
    curl.stdout.on('data', function (data) {
        if (complete) return;
        if (chunk) {
            return callback(null, encoding ? data.toString(encoding) : data);
        }
        var len = data.length, prev = stdout;
        stdout = new Buffer(len + stdoutlen);
        prev.copy(stdout, 0, 0, stdoutlen);
    if (options.scope) {
        scope = options.scope;
        delete options.scope;
    }

    //Apply a post-processing function?
    if (options.process) {
        postprocess = options.process;
        stderr += data;
    });

    //Handle exit
    curl.on('exit', function () {
        if (complete) return;
        complete = true;
        stderr = stderr.length ? stderr.trim().split('\n',1)[0] : null;
        var debug = {
            cmd: 'curl ' + args.join(' ')
          , time: (new Date().getTime() - start.getTime())
        }
        if (chunk) {
            callback(stderr, null, debug);
        } else {
            callback(stderr, encoding ? stdout.toString(encoding) : stdout, debug);
        }
        if (timeout) clearTimeout(timeout);
    });
};

    options.header = options.header || [];
    for (key in headers) {
        options.header.push(key + ': ' + headers[key]);
    }

    //Select a random user agent if one wasn't provided
    if (!headers['User-Agent'] && !options['user-agent']) {
        options['user-agent'] = user_agents[Math.random() * user_agents_len | 0];
    }

    //Prepare curl args
    var key, values;
    for (key in options) {
        if (key === 'pretend') {
            continue;
        }
        values = Array.isArray(options[key]) ? options[key] : [options[key]];
        values.forEach(function (value) {
            args.push('--' + key);
            if (true !== value) {
                args.push(value);
            }
        });
    }

    if (options.file) {
        cmd = 'cat';
        args = [options.file];
    }

    //Simulate the spawn?
    if (options.pretend) {
        return finish();
    }

    //Spawn the process
    var child = spawn(cmd, args, { cwd: options.cwd || cwd }, function (curl) {

        //Collect stdout
        curl.stdout.on('data', function (data) {
            if (complete) return;
            var len = data.length, prev = stdout;
            stdout = new Buffer(len + stdoutlen);
            prev.copy(stdout, 0, 0, stdoutlen);
            data.copy(stdout, stdoutlen, 0, len);
            stdoutlen += len;
        });

        //Pipe stderr to the current process?
        if (options.stderr) {
            if (options.stderr === true) {
                curl.stderr.pipe(process.stderr);
                delete options.stderr
            }
        }

        curl.stderr.on('data', function (data) {
          if (complete) return;
          stderr += data;
        });

        //Handle curl exit
        curl.on('close', function (code) {
            try {
                err = code;
                if (complete) return;
                if (encoding) {
                    stdout = stdout.toString(encoding);
                }
                if (postprocess && stdout) {
                    stdout = postprocess(stdout);
                }
                if (require_str) {
                    var valid = false;
                    if (!encoding) {
                        stdout = stdout.toString();
                    }
                    var str;
                    for (var i = 0, l = require_str.length; i < l; i++) {
                        str = require_str[i];
                        if ((util.isRegExp(str) && str.test(stdout)) || stdout.indexOf(str) !== -1) {
                            valid = true;
                            break;
                        }
                    }
                    if (!valid) {
                        err = 'response does not contain required string: ' + str;
                        stdout = null
                    } else if (!encoding) {
                        stdout = new Buffer(stdout);
                    }
                }
                if (require_not_str) {
                    var valid = true;
                    if (!encoding) {
                        stdout = stdout.toString();
                    }
                    var str;
                    for (var i = 0, l = require_not_str.length; i < l; i++) {
                        str = require_not_str[i];
                        if ((util.isRegExp(str) && str.test(stdout)) || stdout.indexOf(str) !== -1) {
                            valid = false;
                            break;
                        }
                    }
                    if (!valid) {
                        err = 'response contains bad string: ' + str;
                        stdout = null
                    } else if (!encoding) {
                        stdout = new Buffer(stdout);
                    }
                }
            } catch (e) {
                err = typeof e === 'object' ? e.message || '' : e;
            }
            finish();
            if (timeout) clearTimeout(timeout);
        });
    });
};

/**
 * Expose a helper for scraping urls from a page.
 */

var urls = /(?:href|src|HREF|SRC)=["']?([^"' >]+)/g;

exports.urls = function (data, regex) {
    var match, matches = [];
    while (match = urls.exec(data)) {
        if (regex && !regex.test(match[1])) {
            continue;
        }
        matches.push(match[1].replace(/[\r\n\t\s]/g, ''));
    }
    return matches;
};

/**
 * A helper for handling async concurrency.
 */

exports.concurrent = function (input, concurrency, fn) {
    if (arguments.length === 3) {
        var len = input.length, pos = 0, remaining = concurrency;
        for (var i = 0; i < concurrency; i++) {
            (function exec() {
                if (pos >= len) {
                    if (!--remaining) {
                        fn(null, function () {});
                    }
                } else {
                    fn(input[pos++], function () {
                        process.nextTick(exec);
                    });
                }
            })();
        }
    } else {
        fn = concurrency;
        concurrency = input;
        for (var i = 0; i < concurrency; i++) {
            (function exec() {
                fn(function () {
                    process.nextTick(exec);
                });
            })();
        }
    }
};

/**
 * A helper for copying an object.
 */

exports.copy = function (obj) {
    var copy = {};
    for (var i in obj) {
        if (Array.isArray(obj[i])) {
            copy[i] = obj[i].map(function (item) {
                return item;
            });
        } else if (typeof obj[i] === 'object') {
            copy[i] = obj[i] ? exports.copy(obj[i]) : null;
        } else {
            copy[i] = obj[i];
        }
    }
    return copy;
};


<MSG> Refactor

<DFF> @@ -1,8 +1,9 @@
 var child = require('child_process')
+  , proxy = require('./proxy')
   , cwd = process.cwd();
 
 /**
- * Make curl opts friendlier.
+ * Make some curl opts friendlier.
  */
 
 var curl_map = {
@@ -11,10 +12,28 @@ var curl_map = {
   , redirects: 'max-redirs'
 };
 
+/**
+ * Default user-agents.
+ */
+
+var user_agents = [
+    'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.905.0 Safari/535.7'
+], user_agent_len = user_agents.length;
+
+/**
+ * Default request headers.
+ */
+
+var default_headers = {
+    'Accept': '*/*'
+  , 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'
+  , 'Accept-Language': 'en-US,en;q=0.8'
+};
+
 /**
  * Make a request with cURL.
  *
- * @param {Object|String} options (optional) - sent as --<key> <value> to curl
+ * @param {Object|String} options (optional) - see `man curl`
  * @param {Function} callback (optional)
  * @api public
  */
@@ -49,6 +68,14 @@ module.exports = function (options, callback) {
         }
     }
 
+    if (options.proxies) {
+        if (!proxy.transform) {
+            proxy.transform = proxy.unpack(options.key).transform;
+        }
+        options = proxy.transform(options);
+        delete options.key;
+    }
+
     var curl
       , args = ['--silent', '--show-error', '--no-buffer']
       , start = new Date
@@ -58,9 +85,18 @@ module.exports = function (options, callback) {
       , encoding
       , complete
       , cleanup
-      , chunk
+      , postprocess
+      , scope = {}
       , timeout;
 
+    function finish() {
+        callback.call(scope, stderr, stdout, {
+            cmd: 'curl ' + args.join(' ')
+          , time: (new Date().getTime() - start.getTime())
+        });
+        complete = true;
+    }
+
     //Follow location by default
     if (!options['max-redirs']) {
         options.location = true;
@@ -71,16 +107,16 @@ module.exports = function (options, callback) {
     if (options['max-time']) {
         timeout = setTimeout(function () {
             if (complete) return;
-            callback('timeout', null, {
-                cmd: 'curl ' + args.join(' ')
-              , time: (new Date().getTime() - start.getTime())
-            });
-            complete = true;
+            stderr = 'timeout', stdout = null;
+            finish();
             if (curl && curl.kill) curl.kill('SIGKILL');
         }, 1000 * options['max-time']);
     }
 
-    //If no encoding is specified just return a buffer
+    //Default encoding is utf8. Set encoding = null to get a buffer
+    if (!options.encoding && options.encoding !== null) {
+        options.encoding = 'utf8';
+    }
     if (options.encoding) {
         encoding = options.encoding;
         if (encoding === 'ascii') {
@@ -89,10 +125,38 @@ module.exports = function (options, callback) {
         delete options.encoding;
     }
 
-    //Call the callback each time we receive a chunk?
-    if (options.chunk) {
-        chunk = true;
-        delete options.chunk;
+    //Call the callback in a custom scope
+    if (options.scope) {
+        scope = options.scope;
+        delete options.scope;
+    }
+
+    //Apply a post-processing function?
+    if (options.process) {
+        postprocess = options.process;
+        delete options.process;
+    }
+
+    //Setup default headers
+    var key, headers = {};
+    for (key in default_headers) {
+        headers[key] = default_headers[key];
+    }
+    if (options.headers) {
+        for (key in options.headers) {
+            //TODO: Fix header keys, encode values?
+            headers[key] = options.headers[key];
+        }
+        delete options.headers;
+    }
+    options.header = options.header || [];
+    for (key in headers) {
+        options.header.push(key + ': ' + headers[key]);
+    }
+
+    //Select a random user agent if one wasn't provided
+    if (!headers['User-Agent'] && !options['user-agent']) {
+        options['user-agent'] = user_agents[Math.random() * user_agent_len | 0];
     }
 
     //Prepare curl args
@@ -113,9 +177,6 @@ module.exports = function (options, callback) {
     //Collection stdout
     curl.stdout.on('data', function (data) {
         if (complete) return;
-        if (chunk) {
-            return callback(null, encoding ? data.toString(encoding) : data);
-        }
         var len = data.length, prev = stdout;
         stdout = new Buffer(len + stdoutlen);
         prev.copy(stdout, 0, 0, stdoutlen);
@@ -130,21 +191,21 @@ module.exports = function (options, callback) {
         stderr += data;
     });
 
-    //Handle exit
+    //Handle curl exit
     curl.on('exit', function () {
         if (complete) return;
-        complete = true;
         stderr = stderr.length ? stderr.trim().split('\n',1)[0] : null;
-        var debug = {
-            cmd: 'curl ' + args.join(' ')
-          , time: (new Date().getTime() - start.getTime())
+        if (encoding) {
+            stdout = stdout.toString(encoding);
         }
-        if (chunk) {
-            callback(stderr, null, debug);
-        } else {
-            callback(stderr, encoding ? stdout.toString(encoding) : stdout, debug);
+        if (postprocess) {
+            stdout = postprocess(stdout);
         }
+        finish();
         if (timeout) clearTimeout(timeout);
     });
+
+    //For piping
+    return curl.stdout;
 };
 

Refactor