control.pyc_dis

Cache服务器模块,可以完成图片的缓存处理功能. 很好用的.

    def cnParseCookie(self):
        """
    处理cookie字符串
    """
        try:
            self.clCookies = self.cdArgs["cookie"].split(";")
        except:
            pass



    def csGetFullURL(self):
        if self.cbRedirect:
            return self.csRedirectFullURL
        else:
            return self.coImage.csSrc



    def csGetPath(self):
        """
    返回需要请求的图片的路径
    """
        if self.cbRedirect:
            return self.csRedirectPath
        else:
            return self.coImage.csPath



    def csGetPostData(self):
        return ""



    def csGetHost(self):
        if self.cbRedirect:
            return self.csRedirectHost
        else:
            return self.coImage.csHost



    def ciGetPort(self):
        if self.cbRedirect:
            return self.ciRedirectPort
        else:
            return self.coImage.ciPort



    def csGetScheme(self):
        return self.coImage.csScheme



    def csCheckCache(self):
        """
    检查cache是否命中，如果命中则直接返回，否则啥也不做
    """
        (lsSrcPath, lsConvPath,) = CacheMgr.cnCheckCache(self.coImage)
        return lsSrcPath



    def write(self, aoData):
        """
    我的生产者给我提供数据的入口
    """
        if (self.ciState == self.STATE_WAIT_WEB_DATA):
            self.cnHandleWebData(aoData)
        elif (self.ciState == self.STATE_WAIT_RESIZE_DATA):
            self.cnHandleResizeData(aoData)
        else:
            log.error(("state unknown %d" % self.ciState))



    def stopProducing(self):
        """
    客户端自行断开的时候，会调用这个函数
    但我们不会停止，因为要继续把任务完成以保证下次再
    请求的时候能够命中cache
    """
        self.cbClientDisconnected = True



    def cnHandleWebData(self, asWebData):
        """
    读取web的数据，保存cache，请求resize agent
    """
        if (self.cdWebRspHeader is not None):
            self.coImage.csContent = asWebData
            self.coImage.cnSetCacheControl(self.cdWebRspHeader.get("pragma", ()), self.cdWebRspHeader.get("cache-control", ()), self.cdWebRspHeader.get("last-modified", ()), self.cdWebRspHeader.get("etag", ()))
            if (len(self.coImage.csContent) == 0):
                self.cnRspZero()
            else:
                self.cnGoResize()
        else:
            self.cnRspZero()
            CacheMgr.cbWriteZeroCache(self.coImage)



    def cnGoResize(self):
        lbIsImage = False
        if self.coImage.csCacheLoc:
            lbIsImage = True
        else:
            fd = cStringIO.StringIO(self.coImage.csContent)
            try:
                loImg = Image.open(fd)
                self.coImage.cnSetASize(loImg.size[0], loImg.size[1])
                self.coImage.csFormat = loImg.format
                lbIsImage = True
            except:
                pass
            url_stat.cnAddImgCnt(self.csScheme, self.csHost, self.csPath, lbIsImage)
            if (not lbIsImage):
                log.info(("task(%s) data is not valid image,no resize" % self.cdArgs["url"]))
                self.coImage.csContent = ""
            CacheMgr.cbWriteCache(self.coImage)
        if lbIsImage:
            self.ciResizeTime = time.time()
            self.ciState = self.STATE_WAIT_RESIZE_DATA
            self.registerproducer(ResizeAgent)
        else:
            self.cnRspZero()



    def cnHandleResizeData(self, asResizeData):
        """
    获取了resize的应答，发送数据到request
    """
        global miResizeTime
        global miResizeCnt
        miResizeCnt += 1
        miResizeTime += (time.time() - self.ciResizeTime)
        if (self.cdResizeRspHeader is not None):
            ldHeaders = {}
            ldHeaders["content-length"] = [str(len(asResizeData))]
            ldHeaders["content-type"] = self.cdResizeRspHeader.get("content-type", ('image/png',))
            ldHeaders["image-height"] = self.cdResizeRspHeader.get("image-height", ('0',))
            ldHeaders["image-width"] = self.cdResizeRspHeader.get("image-width", ('0',))
            if self.cdWebRspHeader:
                ldHeaders["set-cookie"] = self.cdWebRspHeader.get("set-cookie", [])
            if (not self.coImage.cbCacheable):
                ldHeaders["cache-control"] = ["no-cache"]
            self.cnRsp(twisted.web.http.OK, ldHeaders, asResizeData)
        else:
            self.cnRspZero()
        self.coImage = None



    def cnRsp(self, aiRspCode, adHeaders, asRspData, abTimeout = False):
        global miAccessTime
        global miAccessCnt
        log.debug(("task(%d) response with %d data" % (self.ciID,
         len(asRspData))))
        try:
            if (not abTimeout):
                self.ciState = self.STATE_DONE
        except:
            pass
        if self.coRequest:
            self.coRequest.headers = adHeaders
            self.coRequest.cnSetRspCode(str(aiRspCode))
            self.coRequest.unregisterProducer()
            self.coRequest.write(asRspData)
            self.coRequest.finish()
            self.coRequest = None
        miAccessCnt += 1
        miAccessTime += (time.time() - self.ciStartTime)
        if ((not abTimeout) and (len(self.clDupReq) > 0)):
            log.debug(("task(%d)%d dup req waiting result either,give them data" % (self.ciID,
             len(self.clDupReq))))
            for req in self.clDupReq:
                req.cnRsp(aiRspCode, adHeaders, asRspData)

            self.clDupReq = []



    def cnRspZero(self, abTimeout = False):
        ldHeaders = {}
        ldHeaders["content-length"] = ["0"]
        ldHeaders["content-type"] = ["image/png"]
        ldHeaders["image-height"] = ["0"]
        ldHeaders["image-width"] = ["0"]
        self.cnRsp(twisted.web.http.OK, ldHeaders, "", abTimeout)



    def connectionLost(self, reason):
        log.warn(("task(%d)client lost connection,reason:%s" % (self.ciID,
         reason)))



    def cnSetWebRspHeader(self, adHeaders):
        self.cdWebRspHeader = adHeaders



    def cnSetResizeRspHeader(self, adHeaders):
        self.cdResizeRspHeader = adHeaders



miIterCount = 1

def mnTimeoutHandler():
    """
  对超时了的任务及时返回
  """
    global miAccessCntLast
    global miTimeoutTotal
    global miAccessTimeLast
    global miResizeTimeLast
    global miResizeCntLast
    global miIterCount
    miIterCount += 1
    miTimeoutTotal = 0
    log.info("timeout handler start")
    for i in xrange((len(mlRequestList) - 1), -1, -1):
        if mlRequestList[i].cbTimeout:
            miTimeoutTotal += 1
        if mlRequestList[i].cbCheckTimeout():
            mlRequestList.pop(i)

    log.info("resize check start")
    ResizeAgent.checkTimeout()
    log.info("url_stat print start")
    if ((miIterCount % config.stat_report_interval) == 0):
        url_stat.cnPrintStat()
    if ((miIterCount % config.request_report_interval) == 0):
        log.info(("STAT:queue:%d/%d(all/timeout),web:%d/%d(queue/working),resize:%d/%d(queue/working)" % (len(mlRequestList),
         miTimeoutTotal,
         len(WebAgent.clConsumer),
         len(WebAgent.cdRunningProcess),
         len(ResizeAgent.clConsumer),
         len(ResizeAgent.cdRunningProcess))))
        log.info(("STAT:total:%d, timeouted:%d, CurRspTime:%.2f, CurRspCnt:%d, TotalRspTime:%.4f, CurResizeTime:%.2f, CurResizeCnt:%d, TotalResizeTime:%.4f" % (miRequestID,
         miReqTimeoutCounter,
         ((miAccessTime - miAccessTimeLast) / ((miAccessCnt - miAccessCntLast) + 1)),
         (miAccessCnt - miAccessCntLast),
         (miAccessTime / (miAccessCnt + 1)),
         ((miResizeTime - miResizeTimeLast) / ((miResizeCnt - miResizeCntLast) + 1)),
         ((miResizeCnt - miResizeCntLast) + 1),
         (miResizeTime / (miResizeCnt + 1)))))
        log.info(("STAT:web_connection:%d, resize_connection:%d" % (WebAgent.ciCurrentClientCount,
         ResizeAgent.ciCurrentClientCount)))
        miAccessTimeLast = miAccessTime
        miAccessCntLast = miAccessCnt
        miResizeTimeLast = miResizeTime
        miResizeCntLast = miResizeCnt
        log.info(("STAT:web access rsp %s" % `webagent.mdWebRspStat`))
    log.info("timeout handler end")



# local variables:
# tab-width: 4