HTTP Object and Retrieving HTML Programatically

[Guest]

I am using ASP.Net 2.0 and VB.Net (although C#is ok also).

I want to create an object/method/function that will take a URL as an input
parameter and then return all of the HTML in that page.

I also want to return the HTTP header information (response object).

Does anyone have an insight as to any code samples or .Net objects I would
use to accomplish this?

TIA

 

[Guest]

I am using ASP.Net 2.0 and VB.Net (although C#is ok also).

I want to create an object/method/function that will take a URL as an input
parameter and then return all of the HTML in that page.

I also want to return the HTTP header information (response object).

Does anyone have an insight as to any code samples or .Net objects I would
use to accomplish this?

TIA

Use WebRequest:

Dim PageUrl As String = "http://..."

Dim req As HttpWebRequest = CType(WebRequest.Create(PageUrl),
HttpWebRequest)
Dim enc As Encoding = Encoding.GetEncoding(1252)
Dim r As HttpWebResponse
Dim s As System.IO.StreamReader


r = CType(req.GetResponse(), HttpWebResponse) ' This is your response
s = New System.IO.StreamReader(r.GetResponseStream(), enc)

Dim html As String = s.ReadToEnd()

r.Close()
s.Close()

Response.Write(html)

 

[sloan]

This should help.

You can see how a querystring and a FORM POST works. The FormPost took some
time to figure out, if I recall correctly.

Rework the naming conventions.
I had to hardcode some query string and form post values, naturally you'll
fix those with either a string[] ... or maybe a Dictionary based generic in
2.0.





public class HTTPHelper
{
private int m_ConnectTimeout;
private Encoding m_enc;


public HTTPHelper(int ConnectionTimeout)
{
m_ConnectTimeout = ConnectionTimeout;
}

//This method will write a text file streamed over HTTP in incremental
chunks defined by the buffer size
//This comes in really handy when you have to transfer REALLY big HTML
files and don't want to peg system memory

public void WriteTextFile(string Url, string FilePath, long BufferSize )
{

try
{


string queryStringAll = "?empid=123&carid=1001";
Url += queryStringAll;


//create a web request
HttpWebRequest oHttpWebRequest = null;
oHttpWebRequest = (HttpWebRequest) System.Net.WebRequest.Create(Url);

//set the connection timeout
oHttpWebRequest.Timeout = m_ConnectTimeout;


this.postDataToHttpWebRequest ( oHttpWebRequest , "postkey1" ,
"postvalue1" );



//create a response object that we can read a stream from
HttpWebResponse oHttpResponse = (HttpWebResponse)
oHttpWebRequest.GetResponse();


long workingbuffersize = 1;

//if we don't get back anything from the response, throw and exception
if (oHttpResponse == null)
{
throw new Exception("Url is missing or invalid.");
}

//Define the encoding type
try
{
//see if the page will give us back an encoding type
if (oHttpResponse.ContentEncoding.Length > 0)
m_enc = Encoding.GetEncoding(oHttpResponse.ContentEncoding);
else
m_enc = Encoding.GetEncoding(1252);
}
catch
{
// *** Invalid encoding passed
m_enc = Encoding.GetEncoding(1252);
}


//create a stream reader grabbing text we get over HTTP
StreamReader sr = new
StreamReader(oHttpResponse.GetResponseStream(),m_enc);

//set the variable that we will use as a buffer to store characters in
while the file is downloading
char[] DownloadedCharChunk = new char[BufferSize];

//go ahead and create our streamwriter to write our file
StreamWriter sw = new StreamWriter(FilePath,false,m_enc);

sw.AutoFlush = false;

//when the working buffer size hits 0 then we know that the file has
finished downloading
while (workingbuffersize > 0)
{
//set the working buffer size based on the length of characters we
receive from the stream
//we will also set DownloadedCharChunk to the set of characters we
recieve from the stream
workingbuffersize = sr.Read(DownloadedCharChunk,0,(int) BufferSize);

if (workingbuffersize > 0)
{
//write DownloadedCharChunk to the file on disk
sw.Write(DownloadedCharChunk,0,(int) workingbuffersize );
}

} // while


sr.Close();
sw.Close();

}
catch(Exception e)
{
throw e;
}

}






private string buildPostString ( string fpiKey , string fpiValue)
{

StringBuilder sb = new StringBuilder();


//string postValue = Encode(Request.Form(postKey));
sb.Append( string.Format("{0}={1}&", fpiKey , fpiValue ));


return sb.ToString();
}



private void postDataToHttpWebRequest ( HttpWebRequest webRequest , string
key , string value )
{


if (null != key )
{


ASCIIEncoding encoding=new ASCIIEncoding();

byte[] data = encoding.GetBytes(this.buildPostString(key,value));

webRequest.Method = "POST";
webRequest.ContentType="application/x-www-form-urlencoded";
//oHttpWebRequest.ContentType = "text/xml";//Does Not Work

webRequest.ContentLength = data.Length;
Stream newStream=webRequest.GetRequestStream();
// Send the data.
newStream.Write(data,0,data.Length);
newStream.Close();
}



}


}