It must be your lucky day because today I woke up and decided that I would like to help you with your cause. I first tried to get the download to work with the web browser control but unfortunately I am sure this is not possible without extending the web browser control and we don't want to do that today.
As I mentioned in the comments, the only way I really know that this is possible (without user interaction) is to log in via the HttpWebRequest method. It's pretty tricky stuff. Definitely not for beginners.
Now I must admit that this isn't the cleanest, most "proper" and user-friendly code around, so if anyone wants to suggest a better way to do things, I am all ears.
I suggest you test this first before you incorporate it into your existing app. Just create a new vb.net app and replace all of the code in Form1 with the code below. You will have to update the usernamehere and passwordhere strings with your real username and password. Also, the file is saving to C:\file.rar by default so you can change this path if you want. This code completely removes the need for the web browser control (unless you are using it for something else) so most likely you can remove that from your real application once you incorporate this properly:
Imports System.Net
Imports System.IO
Imports System.Text
Public Class Form1
Private Const gsUserAgent As String = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:35.0) Gecko/20100101 Firefox/35.0"
Const sUsername As String = "usernamehere"
Const sPassword As String = "passwordhere"
Const sMainURL As String = "http://www.mpgh.net/"
Const sCheckLoginURL As String = "http://www.mpgh.net/forum/login.php?do=login"
Const sDownloadURL As String = "http://www.mpgh.net/forum/attachment.php?attachmentid=266579&d=1417312178"
Const sCookieLoggedInMessage As String = "mpgh_imloggedin=yes"
Dim oCookieCollection As CookieCollection = Nothing
Dim sSaveFile As String = "c:\file.rar"
Private Sub Form1_Load(sender As Object, e As EventArgs) Handles MyBase.Load
StartScrape()
End Sub
Private Sub StartScrape()
Try
Dim bContinue As Boolean = True
Dim sPostData(15) As String
sPostData(0) = UrlEncode("vb_login_username")
sPostData(1) = UrlEncode(sUsername)
sPostData(2) = UrlEncode("vb_login_password")
sPostData(3) = UrlEncode(sPassword)
sPostData(4) = UrlEncode("vb_login_password_hint")
sPostData(5) = UrlEncode("Password")
sPostData(6) = UrlEncode("s")
sPostData(7) = UrlEncode("")
sPostData(8) = UrlEncode("securitytoken")
sPostData(9) = UrlEncode("guest")
sPostData(10) = UrlEncode("do")
sPostData(11) = UrlEncode("login")
sPostData(12) = UrlEncode("vb_login_md5password")
sPostData(13) = UrlEncode("")
sPostData(14) = UrlEncode("vb_login_md5password_utf")
sPostData(15) = UrlEncode("")
If GetMethod(sMainURL) = True Then
If SetMethod(sCheckLoginURL, sPostData, sMainURL) = True Then
' Login successful
If DownloadMethod(sDownloadURL, sMainURL) = True Then
MessageBox.Show("File downloaded successfully")
Else
MessageBox.Show("Error downloading file")
End If
End If
End If
Catch ex As Exception
MessageBox.Show(ex.Message, "Error", MessageBoxButtons.OK, MessageBoxIcon.Error)
End Try
End Sub
Private Function GetMethod(ByVal sPage As String) As Boolean
Dim req As HttpWebRequest
Dim resp As HttpWebResponse
Dim stw As StreamReader
Dim bReturn As Boolean = True
Try
req = HttpWebRequest.Create(sPage)
req.Method = "GET"
req.AllowAutoRedirect = False
req.UserAgent = gsUserAgent
req.Accept = "text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5"
req.Headers.Add("Accept-Language", "en-us,en;q=0.5")
req.Headers.Add("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7")
req.Headers.Add("Keep-Alive", "300")
req.KeepAlive = True
resp = req.GetResponse ' Get the response from the server
If req.HaveResponse Then
' Save the cookie info if applicable
SaveCookies(resp.Headers("Set-Cookie"))
resp = req.GetResponse ' Get the response from the server
stw = New StreamReader(resp.GetResponseStream)
stw.ReadToEnd() ' Read the response from the server, but we do not save it
Else
MessageBox.Show("No response received from host " & sPage, "Error", MessageBoxButtons.OK, MessageBoxIcon.Error)
bReturn = False
End If
Catch exc As WebException
MessageBox.Show("Network Error: " & exc.Message.ToString & " Status Code: " & exc.Status.ToString & " from " & sPage, "Error", MessageBoxButtons.OK, MessageBoxIcon.Error)
bReturn = False
End Try
Return bReturn
End Function
Private Function SetMethod(ByVal sPage As String, ByVal sPostData() As String, sReferer As String) As Boolean
Dim bReturn As Boolean = False
Dim req As HttpWebRequest
Dim resp As HttpWebResponse
Dim str As StreamWriter
Dim sPostDataValue As String = ""
Try
req = HttpWebRequest.Create(sPage)
req.Method = "POST"
req.UserAgent = gsUserAgent
req.Accept = "application/x-shockwave-flash,text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5"
req.Headers.Add("Accept-Language", "en-us,en;q=0.5")
req.Headers.Add("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7")
req.Referer = sReferer
req.ContentType = "application/x-www-form-urlencoded"
req.Headers.Add("Pragma", "no-cache")
req.Headers.Add("Keep-Alive", "300")
If oCookieCollection IsNot Nothing Then
' Pass cookie info from the login page
req.CookieContainer = SetCookieContainer(sPage)
End If
str = New StreamWriter(req.GetRequestStream)
If sPostData.Count Mod 2 = 0 Then
' There is an even number of post names and values
For i As Int32 = 0 To sPostData.Count - 1 Step 2
' Put the post data together into one string
sPostDataValue &= sPostData(i) & "=" & sPostData(i + 1) & "&"
Next i
sPostDataValue = sPostDataValue.Substring(0, sPostDataValue.Length - 1) ' This will remove the extra "&" at the end that was added from the for loop above
' Post the data to the server
str.Write(sPostDataValue)
str.Close()
' Get the response
resp = req.GetResponse
If req.HaveResponse Then
If resp.Headers("Set-Cookie").IndexOf(sCookieLoggedInMessage) > -1 Then
' Save the cookie info
SaveCookies(resp.Headers("Set-Cookie"))
bReturn = True
Else
MessageBox.Show("The email or password you entered are incorrect." & vbCrLf & vbCrLf & "Please try again.", "Unable to log in", MessageBoxButtons.OK, MessageBoxIcon.Exclamation)
bReturn = False
End If
Else
' This should probably never happen.. but if it does, we give a message
MessageBox.Show("The email or password you entered are incorrect." & vbCrLf & vbCrLf & "Please try again.", "Unable to log in", MessageBoxButtons.OK, MessageBoxIcon.Exclamation)
bReturn = False
End If
Else
' Did not specify the correct amount of parameters so we cannot continue
MessageBox.Show("POST error. Did not supply the correct amount of post data for " & sPage, "Error", MessageBoxButtons.OK, MessageBoxIcon.Error)
bReturn = False
End If
Catch ex As Exception
MessageBox.Show("POST error. " & ex.Message, "Error", MessageBoxButtons.OK, MessageBoxIcon.Error)
bReturn = False
End Try
Return bReturn
End Function
Private Function DownloadMethod(ByVal sPage As String, sReferer As String) As Boolean
Dim req As HttpWebRequest
Dim bReturn As Boolean = False
Try
req = HttpWebRequest.Create(sPage)
req.Method = "GET"
req.AllowAutoRedirect = False
req.UserAgent = gsUserAgent
req.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
req.Headers.Add("Accept-Language", "en-US,en;q=0.5")
req.Headers.Add("Accept-Encoding", "gzip, deflate")
req.Headers.Add("Keep-Alive", "300")
req.KeepAlive = True
If oCookieCollection IsNot Nothing Then
' Set cookie info so that we continue to be logged in
req.CookieContainer = SetCookieContainer(sPage)
End If
' Save file to disk
Using oResponse As System.Net.WebResponse = CType(req.GetResponse, System.Net.WebResponse)
Using responseStream As IO.Stream = oResponse.GetResponseStream
Using fs As New IO.FileStream(sSaveFile, FileMode.Create, FileAccess.Write)
Dim buffer(2047) As Byte
Dim read As Integer
Do
read = responseStream.Read(buffer, 0, buffer.Length)
fs.Write(buffer, 0, read)
Loop Until read = 0
responseStream.Close()
fs.Flush()
fs.Close()
End Using
responseStream.Close()
End Using
oResponse.Close()
End Using
bReturn = True
Catch exc As WebException
MessageBox.Show("Network Error: " & exc.Message.ToString & " Status Code: " & exc.Status.ToString & " from " & sPage, "Error", MessageBoxButtons.OK, MessageBoxIcon.Error)
bReturn = False
End Try
Return bReturn
End Function
Private Function SetCookieContainer(sPage As String) As System.Net.CookieContainer
Dim oCookieContainerObject As New System.Net.CookieContainer
Dim oCookie As System.Net.Cookie
For c As Int32 = 0 To oCookieCollection.Count - 1
If IsDate(oCookieCollection(c).Value) = True Then
' Fix dates as they seem to cause errors/problems
oCookieCollection(c).Value = Format(CDate(oCookieCollection(c).Value), "dd-MMM-yyyy hh:mm:ss")
End If
oCookie = New System.Net.Cookie
oCookie.Name = oCookieCollection(c).Name
oCookie.Value = oCookieCollection(c).Value
oCookie.Domain = New Uri(sPage).Host
oCookie.Secure = False
oCookieContainerObject.Add(oCookie)
Next
Return oCookieContainerObject
End Function
Private Sub SaveCookies(sCookieString As String)
Dim sCookieStrings() As String = sCookieString.Trim.Replace(" HttpOnly,", "").Replace(" HttpOnly", "").Replace(" domain=.mpgh.net,", "").Split(";".ToCharArray())
oCookieCollection = New CookieCollection
For Each sCookie As String In sCookieStrings
If sCookie.Trim <> "" Then
Dim sName As String = sCookie.Trim().Split("=".ToCharArray())(0)
Dim sValue As String = sCookie.Trim().Split("=".ToCharArray())(1)
oCookieCollection.Add(New Cookie(sName, sValue))
End If
Next
End Sub
Private Function UrlEncode(ByRef URLText As String) As String
Dim AscCode As Integer
Dim EncText As String = ""
Dim bStr() As Byte = Encoding.ASCII.GetBytes(URLText)
Try
For i As Long = 0 To UBound(bStr)
AscCode = bStr(i)
Select Case AscCode
Case 48 To 57, 65 To 90, 97 To 122, 46, 95
EncText = EncText & Chr(AscCode)
Case 32
EncText = EncText & "+"
Case Else
If AscCode < 16 Then
EncText = EncText & "%0" & Hex(AscCode)
Else
EncText = EncText & "%" & Hex(AscCode)
End If
End Select
Next i
Erase bStr
Catch ex As WebException
MessageBox.Show(ex.Message, "Error", MessageBoxButtons.OK, MessageBoxIcon.Error)
End Try
Return EncText
End Function
End Class
