downfromhttp.c

linux下http和ftp下载工具

/***********************************************************************
* File name: DownFromHTTP.c
* Author: Version: Date: likun
* Description: 下载管理模块的实现
*
* Others:
* Function List:
* 1. ....
* History:
* 1. Date:2008-3-31
* Author:likun create this file
* Modification:
* 2. ...
*************************************************************************/
#include <stdio.h>
#include <pthread.h>
#include"DownFromHTTP.h"
#include"GenFunc.h"


#ifdef __cplusplus

extern "C" {
#endif

#define Http_Debug(fmt,arg...)  {\
           fprintf(stderr,"\r\n%s:%04d, "fmt, __FILE__, __LINE__, ##arg);\
        }

int GetStartText(char *buf,int sizebuf)
{
    int i;
	//?buf[i + 1] will overflow when i eq (sizebuf - 1)
	//for(i = 0;i <= sizebuf-1;++i)
    for(i = 0;i <= sizebuf-2;++i)
	{
        if(buf[i]=='\n'&&
		  !(buf[i+1] >= 'a'&&buf[i+1] <= 'z'||
			buf[i+1] >= 'A'&&buf[i+1] <= 'Z'))break;
  
    }
    return i+3;
}


int FindLength(char *buf)
{
    int start=0,length=0,end=0;
    
    while(buf[start]!='\0'){
        if(buf[start]=='L'){
   if(!strncmp(buf+start,"Length:",7)){
       start+=8;
       break;
   }
        }
        start++;
    }
    end=start;
    
    while(buf[end++]!='\n'){}
    buf[--end]='\0';
    length=atoi(buf+start);//Content-Length: xxxxxx\r\n

    return length;
    
}

int GetHttpUrlInfo(const char *url, const char *creferer, DLoadFileInfo *pDloadFile)
{
    char head[1024],referer[256],length1[32];
    char c_port[8];
    int iport=0;//sign the first time to see : like www.xxx.com:8000/name this is the position of :
    char dataget[512]="";
    int i,i_sock,j;
    int length;

    if((!url) || (!pDloadFile))
        return -1;
    
    if(!creferer) /*auto fill*/
        *referer = '\0';
    else
        strncpy(referer, creferer, sizeof(referer));

    length = strlen(url);
    if(!strncasecmp(url,"http://",7))
         pDloadFile->hosttype = DLOAD_PROTOCOL_HTTP;
    else
    {
        Http_Debug("You must input a url start with http://");
        return -1;
    }

    for(i = 0;i <= strlen(url)-7; ++i)
    {
    	if(*(url+7+i)==':')	
            iport=i;
    	else if(*(url+7+i)=='/')
    	{
    		if(iport!=0)
    		{
    			snprintf(pDloadFile->host,iport+1,"%s",url+7);
    			sprintf(pDloadFile->path,"/%s",url+7+i+1);
    			snprintf(c_port,i-iport,"%s",url+7+iport+1);
    			pDloadFile->httpPort=atoi(c_port);
    		}
    		else
    		{
    			snprintf(pDloadFile->host,i+1,"%s",url+7);
    			sprintf(pDloadFile->path,"/%s",url+7+i+1);
    		}
    		break;
    	}
    }
    for(i=length-1;i>=0;i--)
    {
       if(*(url+i)=='/')
       {
    	   sprintf(pDloadFile->filename,"%s",url+i+1);
    	   break;
       }
    }

    i_sock = GetConnectSocket(pDloadFile->host, pDloadFile->httpPort); 
    sprintf(head,"GET %s HTTP/1.1\r\nHost: %s:%d\r\nReferer: %s\r\n\r\n",pDloadFile->path,
        pDloadFile->host,pDloadFile->httpPort,referer,i_sock);      
    if(send(i_sock,head,strlen(head),0) != strlen(head))
    {
        Http_Debug("Error in sending HEAD");
        return -1;
    }
 
     recv(i_sock,dataget,512,0);
     close(i_sock);
     if(strstr(dataget,"\nLocation")!=NULL)
     {
         for(i=0;*(strstr(dataget,"\nLocation")+11+i)!='\n';++i);
         snprintf(url,i,"%s\0",strstr(dataget,"\nLocation")+11);
         GetHttpUrlInfo(url,creferer,pDloadFile);
         return 0;
     }
     if(strstr(dataget,"\nlocation")!=NULL)
     {
         for(i=0;*(strstr(dataget,"\nlocation")+11+i)!='\n';++i);
         snprintf(url,i,"%s\0",strstr(dataget,"\nlocation")+11);
         GetHttpUrlInfo(url,creferer,pDloadFile);
         return 0;
     }
     if(strstr(dataget,"Content-Location")!=NULL)
     {
         for(i=0;*(strstr(dataget,"Content-Location")+18+i)!='\n';++i);
         snprintf(pDloadFile->filename,i,"%s\0",strstr(dataget,"Content-Location")+18);
         if(*(pDloadFile->path+strlen(pDloadFile->path)-1)=='/')
            sprintf(pDloadFile->path,"%s%s",pDloadFile->path,pDloadFile->filename);
     }
     pDloadFile->filelength = FindLength(dataget);
     Http_Debug("host: %s,path: %s, filename:%s,user:%s, pass:%s length: %d",
         pDloadFile->host,pDloadFile->path,pDloadFile->filename, pDloadFile->user,pDloadFile->pass, pDloadFile->filelength);
     close(i_sock);
     return 0;

}

int DownLoadHttp(DloadForCreatThread *pCreatThread)//struct arg *s_info
{
#define FALSE 0
#define TRUE 1

    char get[256];
    char buf[1024]="";
    //this function used to download assigned file from a1 bytes to a2 bytes from host, try 5 times 
    int connected =FALSE;
    int i,j,starttext,boolprint=0;
    int sock,nextthread=0,boolbyte=0;
    int readen,bytesget=0;
    int offset=0;
    struct arg *s_info;

    int curThreadId = pCreatThread->threadId;
    FtpHttpDLoadInfo *pDloadInfo = pCreatThread->pInfo;
    DLoadFileInfo *pTmpFileInfo = &(pDloadInfo->FileInfo);
    s_info = &(pTmpFileInfo->astPthread[curThreadId]);
    int bytesleft=(s_info->a2)-(s_info->a1)+1;
    
    if(s_info->booldown==1&&s_info->allowotherdown==1)
    {    
        free(pCreatThread);
        return 0;
    }
    if(curThreadId!=0){        
    	while(pTmpFileInfo->astPthread[curThreadId-1].booldown==0){
    		sleep(1);
	    }
    }

    while(1){
        if((sock=GetConnectSocket(pTmpFileInfo->host,pTmpFileInfo->httpPort))<=0)
            sleep(1);
        else{
   		    connected=TRUE;
   		    break;
        }
    }

    if(connected)
    {
        sprintf(get,"GET %s HTTP/1.1\r\nRange: bytes=%d-\r\nHost: %s:%d\r\n\r\n",
            pTmpFileInfo->path,s_info->a1,pTmpFileInfo->host,pTmpFileInfo->httpPort);

        if(send(sock,get,strlen(get),0)!=strlen(get)){
		    close(sock);
		    sleep(5);
   		    DownLoadHttp(pCreatThread);
		    return 0;
        }

        s_info->fd=open(pTmpFileInfo->localpath,O_WRONLY|O_CREAT,0644);
        lseek(s_info->fd,s_info->a1,0);
	    pthread_mutex_lock(&(pDloadInfo->lock));
	    s_info->booldown=1;
  	    pthread_mutex_unlock(&(pDloadInfo->lock));
        
        memset(buf, 0, 1024);
        while((bytesleft>0) && (FTPHTTP_NORMAL==pDloadInfo->bTaskExit))
        {
		    readen =ReadEn(sock,30);
        	if(readen<1){
			    close(sock);
			    close(s_info->fd);
			    sleep(2);
			    pthread_mutex_lock(&(pDloadInfo->lock));
			    s_info->booldown=0;
			    s_info->allowotherdown=0;
    		    pthread_mutex_unlock(&(pDloadInfo->lock));
			    DownLoadHttp(pCreatThread);
			    return 0;
		    }
		
		    boolprint++;
		    if(boolprint==10)
                boolprint=0;
   		    if(bytesleft>=1024&&offset==0)
            {
    		    bytesget=recv(sock,buf,1024,MSG_WAITALL);

                /*实际数据的长度*/
      			starttext=GetStartText(buf,bytesget);
      			write(s_info->fd,buf+starttext,1024-starttext);
      			bytesleft+=starttext-bytesget;
    			pTmpFileInfo->sizeget+=bytesget-starttext;
    			s_info->a1+=bytesget-starttext;
    			offset+=bytesget-starttext;
                memset(buf, 0, 1024);
    			continue;
    		}
   		    else if(bytesleft>=1024)
            {
       			bytesget=recv(sock,buf,1024,0);
            	if(bytesget==0)boolbyte++;
       			if(bytesget<0||(bytesget==0&&boolbyte>=10)){
        			close(sock);
        			close(s_info->fd);
        			pthread_mutex_lock(&(pDloadInfo->lock));
        			s_info->booldown=0;
        			s_info->allowotherdown=0;
        			pthread_mutex_unlock(&(pDloadInfo->lock));
        			DownLoadHttp(pCreatThread);
        			return 0;	
    		    }

    			write(s_info->fd,buf,bytesget);
          		offset+=bytesget;
          		pTmpFileInfo->sizeget+=bytesget;
    			s_info->a1+=bytesget;
    			bytesleft-=bytesget;
                memset(buf, 0, 1024);
    			
    	  	}
	  	    else
            {
                if(offset==0)
                {//used only if the thread size <1024
    			    bytesget=recv(sock,buf,1024,MSG_WAITALL);
    			    starttext=GetStartText(buf,bytesget);
    		
        		    if(bytesleft!=bytesget-starttext)
                    {
        			    close(sock);
        			    close(s_info->fd);
        			    sleep(2);
        			    pthread_mutex_lock(&(pDloadInfo->lock));
        			    s_info->booldown=0;
        			    s_info->allowotherdown=0;
        				pthread_mutex_unlock(&(pDloadInfo->lock));
            			DownLoadHttp(pCreatThread);
        			    return 0;
        	        }
                        
    			    write(s_info->fd,buf+starttext,strlen(buf)-starttext);
        			offset+=bytesget-starttext;
        			pthread_mutex_lock(&(pDloadInfo->lock));
        			pTmpFileInfo->sizeget+=bytesget-starttext;
                    pthread_mutex_unlock(&(pDloadInfo->lock));
        			s_info->a1+=bytesget;
        			bytesleft-=bytesget-starttext;