ngx_http_charset_filter_module.c

Nginx ("engine x") 是一个高性能的 HTTP 和 反向代理 服务器

/*
 * Copyright (C) Igor Sysoev
 */


#include <ngx_config.h>
#include <ngx_core.h>
#include <ngx_http.h>


#define NGX_HTTP_NO_CHARSET    -2
#define NGX_HTTP_CHARSET_VAR   0x10000

/* 1 byte length and up to 3 bytes for the UTF-8 encoding of the UCS-2 */
#define NGX_UTF_LEN             4

#define NGX_HTML_ENTITY_LEN     (sizeof("&#1114111;") - 1)


typedef struct {
    u_char                    **tables;
    ngx_str_t                   name;

    unsigned                    length:16;
    unsigned                    utf8:1;
} ngx_http_charset_t;


typedef struct {
    ngx_int_t                   src;
    ngx_int_t                   dst;
} ngx_http_charset_recode_t;


typedef struct {
    ngx_int_t                   src;
    ngx_int_t                   dst;
    u_char                     *src2dst;
    u_char                     *dst2src;
} ngx_http_charset_tables_t;


typedef struct {
    ngx_array_t                 charsets;       /* ngx_http_charset_t */
    ngx_array_t                 tables;         /* ngx_http_charset_tables_t */
    ngx_array_t                 recodes;        /* ngx_http_charset_recode_t */
} ngx_http_charset_main_conf_t;


typedef struct {
    ngx_int_t                   charset;
    ngx_int_t                   source_charset;
    ngx_flag_t                  override_charset;
} ngx_http_charset_loc_conf_t;


typedef struct {
    u_char                     *table;
    ngx_int_t                   charset;

    ngx_chain_t                *busy;
    ngx_chain_t                *free_bufs;
    ngx_chain_t                *free_buffers;

    size_t                      saved_len;
    u_char                      saved[NGX_UTF_LEN];

    unsigned                    length:16;
    unsigned                    from_utf8:1;
    unsigned                    to_utf8:1;
} ngx_http_charset_ctx_t;


typedef struct {
    ngx_http_charset_tables_t  *table;
    ngx_http_charset_t         *charset;
    ngx_uint_t                  characters;
} ngx_http_charset_conf_ctx_t;


static ngx_int_t ngx_http_charset_get_charset(ngx_http_charset_t *charsets,
    ngx_uint_t n, ngx_str_t *charset);
static ngx_int_t ngx_http_charset_set_charset(ngx_http_request_t *r,
    ngx_http_charset_t *charsets, ngx_int_t charset, ngx_int_t source_charset);
static ngx_uint_t ngx_http_charset_recode(ngx_buf_t *b, u_char *table);
static ngx_chain_t *ngx_http_charset_recode_from_utf8(ngx_pool_t *pool,
    ngx_buf_t *buf, ngx_http_charset_ctx_t *ctx);
static ngx_chain_t *ngx_http_charset_recode_to_utf8(ngx_pool_t *pool,
    ngx_buf_t *buf, ngx_http_charset_ctx_t *ctx);

static ngx_chain_t *ngx_http_charset_get_buf(ngx_pool_t *pool,
    ngx_http_charset_ctx_t *ctx);
static ngx_chain_t *ngx_http_charset_get_buffer(ngx_pool_t *pool,
    ngx_http_charset_ctx_t *ctx, size_t size);

static char *ngx_http_charset_map_block(ngx_conf_t *cf, ngx_command_t *cmd,
    void *conf);
static char *ngx_http_charset_map(ngx_conf_t *cf, ngx_command_t *dummy,
    void *conf);

static char *ngx_http_set_charset_slot(ngx_conf_t *cf, ngx_command_t *cmd,
    void *conf);
static ngx_int_t ngx_http_add_charset(ngx_array_t *charsets, ngx_str_t *name);

static void *ngx_http_charset_create_main_conf(ngx_conf_t *cf);
static void *ngx_http_charset_create_loc_conf(ngx_conf_t *cf);
static char *ngx_http_charset_merge_loc_conf(ngx_conf_t *cf,
    void *parent, void *child);
static ngx_int_t ngx_http_charset_postconfiguration(ngx_conf_t *cf);


static ngx_command_t  ngx_http_charset_filter_commands[] = {

    { ngx_string("charset"),
      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF
                        |NGX_HTTP_LIF_CONF|NGX_CONF_TAKE1,
      ngx_http_set_charset_slot,
      NGX_HTTP_LOC_CONF_OFFSET,
      offsetof(ngx_http_charset_loc_conf_t, charset),
      NULL },

    { ngx_string("source_charset"),
      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF
                        |NGX_HTTP_LIF_CONF|NGX_CONF_TAKE1,
      ngx_http_set_charset_slot,
      NGX_HTTP_LOC_CONF_OFFSET,
      offsetof(ngx_http_charset_loc_conf_t, source_charset),
      NULL },

    { ngx_string("override_charset"),
      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF
                        |NGX_HTTP_LIF_CONF|NGX_CONF_FLAG,
      ngx_conf_set_flag_slot,
      NGX_HTTP_LOC_CONF_OFFSET,
      offsetof(ngx_http_charset_loc_conf_t, override_charset),
      NULL },

    { ngx_string("charset_map"),
      NGX_HTTP_MAIN_CONF|NGX_CONF_BLOCK|NGX_CONF_TAKE2,
      ngx_http_charset_map_block,
      NGX_HTTP_MAIN_CONF_OFFSET,
      0,
      NULL },

      ngx_null_command
};


static ngx_http_module_t  ngx_http_charset_filter_module_ctx = {
    NULL,                                  /* preconfiguration */
    ngx_http_charset_postconfiguration,    /* postconfiguration */

    ngx_http_charset_create_main_conf,     /* create main configuration */
    NULL,                                  /* init main configuration */

    NULL,                                  /* create server configuration */
    NULL,                                  /* merge server configuration */

    ngx_http_charset_create_loc_conf,      /* create location configuration */
    ngx_http_charset_merge_loc_conf        /* merge location configuration */
};


ngx_module_t  ngx_http_charset_filter_module = {
    NGX_MODULE_V1,
    &ngx_http_charset_filter_module_ctx,   /* module context */
    ngx_http_charset_filter_commands,      /* module directives */
    NGX_HTTP_MODULE,                       /* module type */
    NULL,                                  /* init master */
    NULL,                                  /* init module */
    NULL,                                  /* init process */
    NULL,                                  /* init thread */
    NULL,                                  /* exit thread */
    NULL,                                  /* exit process */
    NULL,                                  /* exit master */
    NGX_MODULE_V1_PADDING
};


static ngx_http_output_header_filter_pt  ngx_http_next_header_filter;
static ngx_http_output_body_filter_pt    ngx_http_next_body_filter;


static ngx_int_t
ngx_http_charset_header_filter(ngx_http_request_t *r)
{
    u_char                        *ct;
    ngx_int_t                      charset, source_charset;
    ngx_str_t                     *mc, *from, *to, s;
    ngx_uint_t                     n;
    ngx_http_charset_t            *charsets;
    ngx_http_charset_ctx_t        *ctx;
    ngx_http_variable_value_t     *vv;
    ngx_http_charset_loc_conf_t   *lcf, *mlcf;
    ngx_http_charset_main_conf_t  *mcf;

    mcf = ngx_http_get_module_main_conf(r, ngx_http_charset_filter_module);

    charsets = mcf->charsets.elts;
    n = mcf->charsets.nelts;

    /* destination charset */

    if (r == r->main) {

        if (r->headers_out.content_type.len == 0) {
            return ngx_http_next_header_filter(r);
        }

        if (r->headers_out.override_charset
            && r->headers_out.override_charset->len)
        {
            charset = ngx_http_charset_get_charset(charsets, n,
                                              r->headers_out.override_charset);

            if (charset == NGX_HTTP_NO_CHARSET) {
                ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
                              "unknown charset \"%V\" to override",
                              r->headers_out.override_charset);

                return ngx_http_next_header_filter(r);
            }

        } else {
            mlcf = ngx_http_get_module_loc_conf(r,
                                                ngx_http_charset_filter_module);
            charset = mlcf->charset;

            if (charset == NGX_HTTP_NO_CHARSET) {
                return ngx_http_next_header_filter(r);
            }

            if (r->headers_out.charset.len) {
                if (mlcf->override_charset == 0) {
                    return ngx_http_next_header_filter(r);
                }

            } else {
                ct = r->headers_out.content_type.data;

                if (ngx_strncasecmp(ct, (u_char *) "text/", 5) != 0
                    && ngx_strncasecmp(ct,
                                      (u_char *) "application/x-javascript", 24)
                       != 0)
                {
                    return ngx_http_next_header_filter(r);
                }
            }

            if (charset >= NGX_HTTP_CHARSET_VAR) {
                vv = ngx_http_get_indexed_variable(r,
                                               charset - NGX_HTTP_CHARSET_VAR);

                if (vv == NULL || vv->not_found) {
                    return NGX_ERROR;
                }

                s.len = vv->len;
                s.data = vv->data;

                charset = ngx_http_charset_get_charset(charsets, n, &s);
            }
        }

    } else {
        ctx = ngx_http_get_module_ctx(r->main, ngx_http_charset_filter_module);

        if (ctx == NULL) {

            mc = &r->main->headers_out.charset;

            if (mc->len == 0) {
                return ngx_http_next_header_filter(r);
            }

            ctx = ngx_pcalloc(r->pool, sizeof(ngx_http_charset_ctx_t));
            if (ctx == NULL) {
                return NGX_ERROR;
            }

            ngx_http_set_ctx(r->main, ctx, ngx_http_charset_filter_module);

            charset = ngx_http_charset_get_charset(charsets, n, mc);

            ctx->charset = charset;

        } else {
            charset = ctx->charset;
        }
    }

    /* source charset */

    if (r->headers_out.charset.len == 0) {
        lcf = ngx_http_get_module_loc_conf(r, ngx_http_charset_filter_module);

        source_charset = lcf->source_charset;

        if (source_charset >= NGX_HTTP_CHARSET_VAR) {
            vv = ngx_http_get_indexed_variable(r,
                                        source_charset - NGX_HTTP_CHARSET_VAR);

            if (vv == NULL || vv->not_found) {
                return NGX_ERROR;
            }

            s.len = vv->len;
            s.data = vv->data;

            source_charset = ngx_http_charset_get_charset(charsets, n, &s);
        }

        if (charset != NGX_HTTP_NO_CHARSET) {
            return ngx_http_charset_set_charset(r, mcf->charsets.elts, charset,
                                                source_charset);
        }

        if (source_charset == NGX_CONF_UNSET) {
            return ngx_http_next_header_filter(r);
        }

        from = &charsets[source_charset].name;
        to = &r->main->headers_out.charset;

        goto no_charset_map;
    }

    source_charset = ngx_http_charset_get_charset(charsets, n,
                                                  &r->headers_out.charset);

    if (charset == NGX_HTTP_NO_CHARSET
        || source_charset == NGX_HTTP_NO_CHARSET)
    {
        if (charset != source_charset
            || ngx_strcasecmp(r->main->headers_out.charset.data,
                              r->headers_out.charset.data)
                != 0)
        {
            from = &r->headers_out.charset;
            to = (charset == NGX_HTTP_NO_CHARSET) ?
                                           &r->main->headers_out.charset:
                                           &charsets[charset].name;

            goto no_charset_map;
        }

        return ngx_http_next_header_filter(r);
    }

    if (source_charset != charset
        && (charsets[source_charset].tables == NULL
            || charsets[source_charset].tables[charset] == NULL))
    {
        from = &charsets[source_charset].name;
        to = &charsets[charset].name;

        goto no_charset_map;
    }

    r->headers_out.content_type.len = r->headers_out.content_type_len;

    return ngx_http_charset_set_charset(r, mcf->charsets.elts, charset,
                                        source_charset);

no_charset_map:

    ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
                  "no \"charset_map\" between the charsets \"%V\" and \"%V\"",
                  from, to);

    return ngx_http_next_header_filter(r);
}


static ngx_int_t
ngx_http_charset_get_charset(ngx_http_charset_t *charsets, ngx_uint_t n,
    ngx_str_t *charset)
{
    ngx_uint_t  i;

    for (i = 0; i < n; i++) {
        if (charsets[i].name.len != charset->len) {
            continue;
        }

        if (ngx_strncasecmp(charsets[i].name.data, charset->data, charset->len)
            == 0)
        {
            return i;
        }
    }

    return NGX_HTTP_NO_CHARSET;
}


static ngx_int_t
ngx_http_charset_set_charset(ngx_http_request_t *r,
    ngx_http_charset_t *charsets, ngx_int_t charset, ngx_int_t source_charset)
{
    ngx_http_charset_ctx_t  *ctx;

    if (r->headers_out.status == NGX_HTTP_MOVED_PERMANENTLY
        || r->headers_out.status == NGX_HTTP_MOVED_TEMPORARILY)
    {
        /*
         * do not set charset for the redirect because NN 4.x
         * use this charset instead of the next page charset
         */

        r->headers_out.charset.len = 0;

        return ngx_http_next_header_filter(r);
    }

    r->headers_out.charset = charsets[charset].name;
    r->utf8 = charsets[charset].utf8;

    if (source_charset == NGX_CONF_UNSET || source_charset == charset) {
        return ngx_http_next_header_filter(r);
    }

    ctx = ngx_pcalloc(r->pool, sizeof(ngx_http_charset_ctx_t));
    if (ctx == NULL) {
        return NGX_ERROR;
    }

    ngx_http_set_ctx(r, ctx, ngx_http_charset_filter_module);

    ctx->table = charsets[source_charset].tables[charset];
    ctx->charset = charset;
    ctx->length = charsets[charset].length;
    ctx->from_utf8 = charsets[source_charset].utf8;
    ctx->to_utf8 = charsets[charset].utf8;

    r->filter_need_in_memory = 1;

    if ((ctx->to_utf8 || ctx->from_utf8) && r == r->main) {
        ngx_http_clear_content_length(r);

    } else {
        r->filter_need_temporary = 1;
    }

    return ngx_http_next_header_filter(r);
}


static ngx_int_t
ngx_http_charset_body_filter(ngx_http_request_t *r, ngx_chain_t *in)
{
    ngx_int_t                rc;
    ngx_buf_t               *b;
    ngx_chain_t             *cl, *out, **ll;
    ngx_http_charset_ctx_t  *ctx;

    ctx = ngx_http_get_module_ctx(r, ngx_http_charset_filter_module);

    if (ctx == NULL || ctx->table == NULL) {
        return ngx_http_next_body_filter(r, in);
    }

    if ((ctx->to_utf8 || ctx->from_utf8) || ctx->busy) {

        out = NULL;
        ll = &out;

        for (cl = in; cl; cl = cl->next) {
            b = cl->buf;

            if (ngx_buf_size(b) == 0) {

                *ll = ngx_alloc_chain_link(r->pool);
                if (*ll == NULL) {
                    return NGX_ERROR;
                }

                (*ll)->buf = b;
                (*ll)->next = NULL;

                ll = &(*ll)->next;

                continue;
            }

            if (ctx->to_utf8) {
                *ll = ngx_http_charset_recode_to_utf8(r->pool, b, ctx);

            } else {
                *ll = ngx_http_charset_recode_from_utf8(r->pool, b, ctx);
            }

            if (*ll == NULL) {
                return NGX_ERROR;
            }

            while (*ll) {
                ll = &(*ll)->next;
            }
        }

        rc = ngx_http_next_body_filter(r, out);

        if (out) {
            if (ctx->busy == NULL) {
                ctx->busy = out;

            } else {
                for (cl = ctx->busy; cl->next; cl = cl->next) { /* void */ }
                cl->next = out;
            }
        }

        while (ctx->busy) {

            cl = ctx->busy;
            b = cl->buf;

            if (ngx_buf_size(b) != 0) {
                break;
            }

#if (NGX_HAVE_WRITE_ZEROCOPY)
            if (b->zerocopy_busy) {