control.pyc_dis

Cache服务器模块,可以完成图片的缓存处理功能. 很好用的.

#! /usr/bin/env python
# emacs-mode: -*- python-*-

import logging
import cgi
import string
import urllib
import cStringIO
import time
import urlparse
from zope.interface import implements
import twisted.internet.interfaces
import Image
import webagent
import cachemgr
from cachemgr import CacheMgr
from webagent import WebAgent
from resizeagent import ResizeAgent
import const
import config
from url_stat import url_stat
miRequestID = 0
miReqTimeoutCounter = 0
log = logging.getLogger(const.CONST_APP_LOGGER)
mlRequestList = []
miTimeoutTotal = 0
miAccessCnt = 0
miAccessTime = 0
miAccessCntLast = 0
miAccessTimeLast = 0
miResizeCnt = 0
miResizeTime = 0
miResizeCntLast = 0
miResizeTimeLast = 0
class ResizeRequest:
    implements(twisted.internet.interfaces.IProducer, twisted.internet.interfaces.IConsumer)
    csSep1 = "&"
    csSep2 = "="
    csHttpGetQuerySep = "?"
    cbStreaming = False
    coWebAgent = None
    coResizeAgent = None
    STATE_WAIT_WEB_AGENT = 1
    STATE_WAIT_WEB_DATA = 2
    STATE_WAIT_RESIZE_DATA = 4
    STATE_DONE = 5
    MAX_REDIRECT_TRY = 3

    def __init__(self, aoRequest):
        global miRequestID
        self.ciID = miRequestID
        log.debug(("img req id:%d" % self.ciID))
        miRequestID += 1
        self.coRequest = aoRequest
        self.ciState = self.STATE_WAIT_WEB_AGENT
        self.cdArgs = {}
        self.csHost = None
        self.csPath = None
        self.csScheme = None
        self.coImage = cachemgr.Image()
        self.clCookies = []
        self.cbClientDisconnected = False
        self.cdWebRspHeader = None
        self.csResizeQuery = ""
        self.cdResizeRspHeader = None
        self.csReferer = ""
        self.clDupReq = []
        self.cbDupReq = False
        self.ciStartTime = time.time()
        self.ciResizeTime = 0
        self.cbTimeout = False
        self.coWaitReq = None
        self.csReq = self.coRequest.csGetIData()
        self.csUserAgent = None
        self.cbRedirect = False
        self.csRedirectHost = None
        self.csRedirectPort = None
        self.csRedirectPath = None
        self.csRedirectFullURL = None
        self.ciRedirectTry = 0
        self.csOrigFullURL = None
        self.coProducer = None
        if (not self.cbParseQuery()):
            self.cnRspZero()
            return
        for req in mlRequestList:
            if req.cbSameReq(self):
                self.coWaitReq = req
                log.debug(("dup req(%d) exist,just wait for them" % req.ciID))
                self.cbDupReq = True
                req.cnAddSameReq(self)
                break

        mlRequestList.append(self)
        if (not self.cbDupReq):
            self.coRequest.registerProducer(self, self.cbStreaming)



    def cbSetRedirect(self, asRedirectLoc):
        """
    设置重定向
    """
        self.ciRedirectTry += 1
        if (self.ciRedirectTry > self.MAX_REDIRECT_TRY):
            log.warn(("exceed max redirect try, last redirect addr is:%s, origin is:%s" % (asRedirectLoc,
             self.csGetFullURL())))
            return False
        llParsed = list(urlparse.urlparse(asRedirectLoc))
        (scheme, host, path,) = llParsed[:3]
        if ((scheme == "") or (scheme == "http")):
            llParsed[0] = "http"
        else:
            log.debug("only http can be redirect")
            return False
        if (host == ""):
            llParsed[1] = ((self.coImage.csHost + ":") + str(self.coImage.ciPort))
        self.csRedirectFullURL = urlparse.urlunparse(llParsed)
        self.cbRedirect = True
        self.csRedirectHost = llParsed[1]
        return True



    def cbCheckTimeout(self):
        """
    自我检查是否超时了
    如果超时需要删除，那么返回true，否则返回false
    """
        global miReqTimeoutCounter
        if (self.ciState == self.STATE_DONE):
            return True
        if ((time.time() - self.ciStartTime) < config.request_timeout):
            return
        if self.cbTimeout:
            if ((time.time() - self.ciStartTime) > (config.request_timeout * 1.5)):
                log.debug(("task(%d) timeout too long,break connection" % self.ciID))
                if self.coProducer:
                    self.coProducer.stopProducing(self.ciID)
                return True
            return False
        log.debug(("task(%d)%s is timeout" % (self.ciID,
         self.coImage.csSrc)))
        miReqTimeoutCounter += 1
        self.cbTimeout = True
        if self.coWaitReq:
            self.coWaitReq.cnRemoveSameReq(self)
        self.cnRspZero(True)
        return False



    def cbSameReq(self, aoOtherReq):
        """
    判断两个请求是否相同
    """
        if (self.csReq != aoOtherReq.csReq):
            return False
        if self.cbTimeout:
            return False
        return True



    def cnAddSameReq(self, aoOtherReq):
        """
    添加重复请求
    """
        self.clDupReq.append(aoOtherReq)



    def cnRemoveSameReq(self, aoOtherReq):
        try:
            self.clDupReq.remove(aoOtherReq)
        except:
            pass



    def resumeProducing(self):
        lsSrcPath = self.csCheckCache()
        if (lsSrcPath is not None):
            self.coImage.csCacheLoc = lsSrcPath
            self.cnGoResize()
        else:
            self.ciState = self.STATE_WAIT_WEB_DATA
            self.registerproducer(WebAgent)



    def registerproducer(self, aoProducer):
        """
    添加生产者，不同阶段，我们的生产者是不一样的
    """
        if (self.ciState == self.STATE_WAIT_WEB_DATA):
            aoProducer.cnAddConsumer(self)
            self.coProducer = aoProducer
            aoProducer.resumeProducing()
        if (self.ciState == self.STATE_WAIT_RESIZE_DATA):
            aoProducer.cnAddConsumer(self)
            self.coProducer = aoProducer
            aoProducer.resumeProducing()



    def cbParseQuery(self):
        """
    解释请求，请求格式如下：
    http://192.168.15.206:10007/precache?url=http%3A%2F%2Fimg.mop.com%2Fimages%2Frss%2Frss_xml.jpg&size=170x177&color=32referer=http%3A%2F%2Fdzh.mop.com%2Ftopic%2FreadSub.jsp%3Fsid%3D5495259&
    参数串包括:
    url
    size
    color
    referer
    三个参数
    """
        lbGood = True
        lsQuery = self.coRequest.csGetIData()
        llFields = cgi.parse_qsl(lsQuery)
        for (key, value,) in llFields:
            self.cdArgs[key.lower()] = value

        (self.csScheme, self.csHost, self.csPath,) = urlparse.urlparse(self.cdArgs["url"])[0:3]
        try:
            url_stat.cnAddReq(self.csScheme, self.csHost, self.csPath)
            if self.cbBlackList(self.cdArgs["url"], self.csHost, self.csPath):
                return False
            self.coImage.csSrc = self.cdArgs["url"]
            self.coImage.csSrc = self.coImage.csSrc.replace(" ", "%20")
            self.coImage.cnSetColor(self.cdArgs["color"])
            self.coImage.cnSetRSize(self.cdArgs["size"])
            self.coImage.cnParseURL()
        except Exception,e:
            import traceback
            traceback.print_exc()
            lbGood = False
        self.csReferer = self.cdArgs.get("referer", "")
        self.csUserAgent = self.cdArgs.get("useragent", config.default_user_agent)
        self.cnParseCookie()
        self.cdArgs["url"] = self.coImage.csSrc
        llBuf = []
        for (key, value,) in self.cdArgs.items():
            if (key == "url"):
                llBuf.append(((key + "=") + self.coImage.csSrc))
            else:
                llBuf.append(((key + "=") + value))

        self.csResizeQuery = string.join(llBuf, "&")
        self.csResizeQuery = urllib.urlencode(self.cdArgs)
        self.csOrigFullURL = self.coImage.csSrc
        return lbGood



    def cbBlackList(self, request_url, request_host, request_path):
        """
    检查要图片是否在黑名单上
    """
        try:
            for (host, path,) in config.image_blacklist:
                if ((request_host == host) and request_path.lower().startswith(path)):
                    log.debug(("denied " + request_url))
                    return True

            if url_stat.cbIsBlocking(self.csScheme, self.csHost, self.csPath):
                log.debug(("denied " + request_url))
                return True
        except:
            pass
        return False