c HttpWebRequest crawls web content through a proxy server

  • 2020-05-07 20:20:45
  • OfStack

Used by Intranet users or proxy Internet users
 
using System.IO; 
using System.Net; 
public string get_html() 
{ 
string urlStr = "http://www.domain.com"; // I will specify the address to be taken  
HttpWebRequest hwr = (HttpWebRequest)HttpWebRequest.Create(urlStr); // To establish HttpWebRequest � like  
hwr.Timeout = 60000; // A set of mobile phones and a set of mobile phones  
WebProxy proxy = new WebProxy(); // Set the � 1 � � � �  
proxy.Address = new Uri("http://proxy.domain.com:3128"); // � � � device : port  
proxy.Credentials = new NetworkCredential("f3210316", "6978233"); // Use of � , Dense �  
hwr.UseDefaultCredentials = true; // With � � � � �  
hwr.Proxy = proxy; // � buy � �  
try 
{ 
HttpWebResponse hwrs = (HttpWebResponse)hwr.GetResponse(); // Get back to �  
} 
catch 
{ 
MessageBox.Show(" Unable to connect broker! "); 
return; 
} 
// judge HTTP The response status  
if(hwrs.StatusCode != HttpStatusCode.OK) 
{ 
MessageBox.Show(" Access failed! "); 
hwrs.Close(); 
return; 
} 
else 
{ 
Stream s = hwrs.GetResponseStream(); // Gets the stream and image of the returned stream  
StreamReader sr = new StreamReader(s, Encoding.UTF8); // In order to UTF-8 � � � flow  
StringBuilder content = new StringBuilder(); // 
while (sr.Peek() != -1) // Every time � take 1 line , until  
{ // Under the 1 Every word has its color  
content.Append(sr.ReadLine()+""r"n"); // Returns the � check  
} // 
//return content.ToString() ; 
} 
// Output all Header (including server output, of course Cookie )  
//for(int ii=0;ii<hwrs.Headers.Count;ii++) 
//{ 
//MessageBox.Show(hwrs.Headers.GetKey(ii)+" : "+res.Headers[ii]); 
//} 
} 

You know, with HttpWebRequest you can crawl web pages through Http, but if you're on an Intranet and you're surfing the web through a proxy, you can't do it directly.
Is there any way?
Of course there is, ha ha, see the following code:
 
string urlStr = "http://www.domain.com"; // I will specify the address to be taken  
HttpWebRequest hwr = (HttpWebRequest)HttpWebRequest.Create(urlStr); // To establish HttpWebRequest � like  
hwr.Timeout = 60000; // A set of mobile phones and a set of mobile phones  
WebProxy proxy = new WebProxy(); // Set the � 1 � � � �  
proxy.Address = new Uri("http://proxy.domain.com:3128"); // � � � device : port  
proxy.Credentials = new NetworkCredential("f3210316", "6978233"); // Use of � , Dense �  
hwr.UseDefaultCredentials = true; // With � � � � �  
hwr.Proxy = proxy; // � buy � �  
HttpWebResponse hwrs = (HttpWebResponse)hwr.GetResponse(); // Get back to �  
Stream s = hwrs.GetResponseStream(); // Gets the stream and image of the returned stream  
StreamReader sr = new StreamReader(s, Encoding.UTF8); // In order to UTF-8 � � � flow  
StringBuilder content = new StringBuilder(); // 
while (sr.Peek() != -1) // Every time � take 1 line , until  
{ // Under the 1 Every word has its color  
content.Append(sr.ReadLine()+""r"n"); // Returns the � check  
} // 
return content.ToString() ; // Returns the resulting string  

Related articles: