I'm actually building an automation tool based on Selenium with Go called IGopher and I want implement a native proxy support.
However, I am facing an issue with those with authentication...
I can't send the proxy credentials to Chrome and without them it asks through an alert box for authentication that I can hardly interact with through Selenium (I'm not even sure it's possible in headless mode).
So I thought of an intermediary proxy system hosted locally by my program which will add the Proxy-Authorization header and transfer the request to the remote proxy:
I found this nodejs program which works like a charm and I would like to reproduce that in Go.
The main part is the following:
function createPortForwarder(local_host, local_port, remote_host, remote_port, buf_proxy_basic_auth, is_remote_https, ignore_https_cert) {
net.createServer({allowHalfOpen: true}, function (socket) {
var realCon = (is_remote_https ? tls : net).connect({
port: remote_port, host: remote_host, allowHalfOpen: true,
rejectUnauthorized: !ignore_https_cert /*not used when is_remote_https false*/
});
realCon.on('data', function (buf) {
socket.write(buf);
realCon.__haveGotData = true;
}).on('end', function () {
socket.end();
if (!realCon.__haveGotData && !realCon.__haveShownError) {
console.error('[LocalProxy(:' + local_port + ')][Connection to ' + remote_host + ':' + remote_port + '] Error: ended by remote peer');
realCon.__haveShownError = true;
}
}).on('close', function () {
socket.end();
if (!realCon.__haveGotData && !realCon.__haveShownError) {
console.error('[LocalProxy(:' + local_port + ')][Connection to ' + remote_host + ':' + remote_port + '] Error: reset by remote peer');
realCon.__haveShownError = true;
}
}).on('error', function (err) {
console.error('[LocalProxy(:' + local_port + ')][Connection to ' + remote_host + ':' + remote_port + '] ' + err);
realCon.__haveShownError = true;
});
var parser = new HTTPParser(HTTPParser.REQUEST);
parser[HTTPParser.kOnHeadersComplete] = function (versionMajor, versionMinor, headers, method,
url, statusCode, statusMessage, upgrade,
shouldKeepAlive) {
parser.__is_headers_complete = true;
parser.__upgrade = upgrade;
parser.__method = method;
};
var state = STATE_NONE;
socket.on('data', function (buf) {
if (!parser) {
realCon.write(buf);
return
}
var buf_ary = [], unsavedStart = 0, buf_len = buf.length;
for (var i = 0; i < buf_len; i++) {
//find first LF
if (state === STATE_NONE) {
if (buf[i] === LF) {
state = STATE_FOUND_LF;
}
continue;
}
//find second CR LF or LF
if (buf[i] === LF) {
parser.__is_headers_complete = false;
parser.execute(buf.slice(unsavedStart, i + 1));
if (parser.__is_headers_complete) {
buf_ary.push(buf.slice(unsavedStart, buf[i - 1] === CR ? i - 1 : i));
//insert auth header
buf_ary.push(buf_proxy_basic_auth);
buf_ary.push(state === STATE_FOUND_LF_CR ? BUF_CR_LF_CR_LF : BUF_LF_LF);
// stop intercepting packets if encountered TLS and WebSocket handshake
if (parser.__method === 5 /*CONNECT*/ || parser.__upgrade) {
parser.close();
parser = null;
buf_ary.push(buf.slice(i + 1));
realCon.write(Buffer.concat(buf_ary));
state = STATE_NONE;
return;
}
unsavedStart = i + 1;
state = STATE_NONE;
}
else {
state = STATE_FOUND_LF;
}
}
else if (buf[i] === CR && state === STATE_FOUND_LF) {
state = STATE_FOUND_LF_CR;
} else {
state = STATE_NONE;
}
}
if (unsavedStart < buf_len) {
buf = buf.slice(unsavedStart, buf_len);
parser.execute(buf);
buf_ary.push(buf);
}
realCon.write(Buffer.concat(buf_ary));
}).on('end', cleanup).on('close', cleanup).on('error', function (err) {
if (!socket.__cleanup) {
console.error('[LocalProxy(:' + local_port + ')][Incoming connection] ' + err);
}
});
function cleanup() {
socket.__cleanup = true;
if (parser) {
parser.close();
parser = null;
}
realCon.end();
}
}).on('error', function (err) {
console.error('[LocalProxy(:' + local_port + ')] ' + err);
process.exit(1);
}).listen(local_port, local_host === '*' ? undefined : local_host, function () {
console.log('[LocalProxy(:' + local_port + ')] OK: forward http://' + local_host + ':' + local_port + ' to http' + (is_remote_https ? 's' : '') + '://' + remote_host + ':' + remote_port);
});
}
Would it be possible to reproduce that in go? I found this gist for a TCP forwarder but I don't if it's possible to edit request header with this...
If this is not possible (which would surprise me) I can still use this node program in my own one but I would really prefer to avoid making node a dependency.
In addition, having it natively in my program would make interactions with it easier, in particular to be able to stop or restart this server.
So if anyone has an idea, advice or resources that will help me a lot! I've been stuck on this problem for a long time...
Thanks in advance!
EDIT:
I already tried with a ReverseProxy without success, it can't send the request to the remote proxy after modifying it since it's a CONNECT one and the URL is in the format //ip:port for example: //google.com:443.
Here's my code:
func printResponse(r *http.Response) error {
logrus.Infof("Response: %+v\n", r)
return nil
}
// LaunchForwardingProxy launch forward server used to inject proxy authentication header
// into outgoing requests
func LaunchForwardingProxy(localPort uint16, remoteProxy ProxyConfig) error {
localServerHost = fmt.Sprintf("localhost:%d", localPort)
remoteServerHost = fmt.Sprintf(
"http://%s:%d",
remoteProxy.IP,
remoteProxy.Port,
)
remoteServerAuth = fmt.Sprintf(
"%s:%s",
remoteProxy.Username,
remoteProxy.Password,
)
remote, err := url.Parse(remoteServerHost)
if err != nil {
panic(err)
}
proxy := httputil.NewSingleHostReverseProxy(remote)
d := func(req *http.Request) {
logrus.Infof("Pre-Edited request: %+v\n", req)
req.Host = remoteServerHost
// Inject proxy authentication headers to outgoing request into new Header
basicAuth := "Basic " + base64.StdEncoding.EncodeToString([]byte(remoteServerAuth))
req.Header.Set("Proxy-Authorization", basicAuth)
logrus.Infof("Edited Request: %+v\n", req)
logrus.Infof("Scheme: %s, Host: %s, Port: %s\n", req.URL.Scheme, req.URL.Host, req.URL.Port())
}
proxy.Director = d
proxy.ModifyResponse = printResponse
http.ListenAndServe(localServerHost, proxy)
return nil
}
curl https://google.com --proxy http://127.0.0.1:8880 output:
INFO[0013] Pre-Edited request: &{Method:CONNECT URL://google.com:443 Proto:HTTP/1.1 ProtoMajor:1 ProtoMinor:1 Header:map[Proxy-Connection:[Keep-Alive] User-Agent:[curl/7.68.0]] Body:<nil> GetBody:<nil> ContentLength:0 TransferEncoding:[] Close:false Host:google.com:443 Form:map[] PostForm:map[] MultipartForm:<nil> Trailer:map[] RemoteAddr:127.0.0.1:54126 RequestURI:google.com:443 TLS:<nil> Cancel:<nil> Response:<nil> ctx:0xc000674280} function=func1 line=59
INFO[0013] Edited Request: &{Method:CONNECT URL://google.com:443 Proto:HTTP/1.1 ProtoMajor:1 ProtoMinor:1 Header:map[Proxy-Authorization:[Basic auth] Proxy-Connection:[Keep-Alive] User-Agent:[curl/7.68.0]] Body:<nil> GetBody:<nil> ContentLength:0 TransferEncoding:[] Close:false Host:http://51.178.xx.xx:3128 Form:map[] PostForm:map[] MultipartForm:<nil> Trailer:map[] RemoteAddr:127.0.0.1:54126 RequestURI:google.com:443 TLS:<nil> Cancel:<nil> Response:<nil> ctx:0xc000674280} function=func1 line=67
INFO[0013] Scheme: , Host: google.com:443, Port: 443 function=func1 line=68
2021/03/16 21:53:51 http: proxy error: unsupported protocol scheme ""