c HttpWebRequest crawls web content through a proxy server
- 2020-05-07 20:20:45
- OfStack
Used by Intranet users or proxy Internet users
You know, with HttpWebRequest you can crawl web pages through Http, but if you're on an Intranet and you're surfing the web through a proxy, you can't do it directly.
Is there any way?
Of course there is, ha ha, see the following code:
using System.IO;
using System.Net;
public string get_html()
{
string urlStr = "http://www.domain.com"; // I will specify the address to be taken
HttpWebRequest hwr = (HttpWebRequest)HttpWebRequest.Create(urlStr); // To establish HttpWebRequest � like
hwr.Timeout = 60000; // A set of mobile phones and a set of mobile phones
WebProxy proxy = new WebProxy(); // Set the � 1 � � � �
proxy.Address = new Uri("http://proxy.domain.com:3128"); // � � � device : port
proxy.Credentials = new NetworkCredential("f3210316", "6978233"); // Use of � , Dense �
hwr.UseDefaultCredentials = true; // With � � � � �
hwr.Proxy = proxy; // � buy � �
try
{
HttpWebResponse hwrs = (HttpWebResponse)hwr.GetResponse(); // Get back to �
}
catch
{
MessageBox.Show(" Unable to connect broker! ");
return;
}
// judge HTTP The response status
if(hwrs.StatusCode != HttpStatusCode.OK)
{
MessageBox.Show(" Access failed! ");
hwrs.Close();
return;
}
else
{
Stream s = hwrs.GetResponseStream(); // Gets the stream and image of the returned stream
StreamReader sr = new StreamReader(s, Encoding.UTF8); // In order to UTF-8 � � � flow
StringBuilder content = new StringBuilder(); //
while (sr.Peek() != -1) // Every time � take 1 line , until
{ // Under the 1 Every word has its color
content.Append(sr.ReadLine()+""r"n"); // Returns the � check
} //
//return content.ToString() ;
}
// Output all Header (including server output, of course Cookie )
//for(int ii=0;ii<hwrs.Headers.Count;ii++)
//{
//MessageBox.Show(hwrs.Headers.GetKey(ii)+" : "+res.Headers[ii]);
//}
}
You know, with HttpWebRequest you can crawl web pages through Http, but if you're on an Intranet and you're surfing the web through a proxy, you can't do it directly.
Is there any way?
Of course there is, ha ha, see the following code:
string urlStr = "http://www.domain.com"; // I will specify the address to be taken
HttpWebRequest hwr = (HttpWebRequest)HttpWebRequest.Create(urlStr); // To establish HttpWebRequest � like
hwr.Timeout = 60000; // A set of mobile phones and a set of mobile phones
WebProxy proxy = new WebProxy(); // Set the � 1 � � � �
proxy.Address = new Uri("http://proxy.domain.com:3128"); // � � � device : port
proxy.Credentials = new NetworkCredential("f3210316", "6978233"); // Use of � , Dense �
hwr.UseDefaultCredentials = true; // With � � � � �
hwr.Proxy = proxy; // � buy � �
HttpWebResponse hwrs = (HttpWebResponse)hwr.GetResponse(); // Get back to �
Stream s = hwrs.GetResponseStream(); // Gets the stream and image of the returned stream
StreamReader sr = new StreamReader(s, Encoding.UTF8); // In order to UTF-8 � � � flow
StringBuilder content = new StringBuilder(); //
while (sr.Peek() != -1) // Every time � take 1 line , until
{ // Under the 1 Every word has its color
content.Append(sr.ReadLine()+""r"n"); // Returns the � check
} //
return content.ToString() ; // Returns the resulting string