feedprocessor.js

从国外的站点上淘的个人财务管理系统．开源．

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* ***** BEGIN LICENSE BLOCK *****
 * Version: MPL 1.1/GPL 2.0/LGPL 2.1
 *
 * The contents of this file are subject to the Mozilla Public License Version
 * 1.1 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 * http://www.mozilla.org/MPL/
 *
 * Software distributed under the License is distributed on an "AS IS" basis,
 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
 * for the specific language governing rights and limitations under the
 * License.
 *
 * The Original Code is mozilla.org code.
 *
 * The Initial Developer of the Original Code is Robert Sayre.
 * Portions created by the Initial Developer are Copyright (C) 2006
 * the Initial Developer. All Rights Reserved.
 *
 * Contributor(s):
 *   Ben Goodger <[e-mail]>
 *   Myk Melez <[e-mail]>
 *
 * Alternatively, the contents of this file may be used under the terms of
 * either the GNU General Public License Version 2 or later (the "GPL"), or
 * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
 * in which case the provisions of the GPL or the LGPL are applicable instead
 * of those above. If you wish to allow use of your version of this file only
 * under the terms of either the GPL or the LGPL, and not to allow others to
 * use your version of this file under the terms of the MPL, indicate your
 * decision by deleting the provisions above and replace them with the notice
 * and other provisions required by the GPL or the LGPL. If you do not delete
 * the provisions above, a recipient may use your version of this file under
 * the terms of any one of the MPL, the GPL or the LGPL.
 *
 * ***** END LICENSE BLOCK ***** */

function LOG(str) {
  dump("*** " + str + "\n");
}

const Ci = Components.interfaces;
const Cc = Components.classes;
const Cr = Components.results;

const IO_CONTRACTID = "@mozilla.org/network/io-service;1"
const BAG_CONTRACTID = "@mozilla.org/hash-property-bag;1"
const ARRAY_CONTRACTID = "@mozilla.org/array;1";
const SAX_CONTRACTID = "@mozilla.org/saxparser/xmlreader;1";
const UNESCAPE_CONTRACTID = "@mozilla.org/feed-unescapehtml;1";

var gIoService = Cc[IO_CONTRACTID].getService(Ci.nsIIOService);
var gUnescapeHTML = Cc[UNESCAPE_CONTRACTID].
                    getService(Ci.nsIScriptableUnescapeHTML);

const XMLNS = "http://www.w3.org/XML/1998/namespace";
const RSS090NS = "http://my.netscape.com/rdf/simple/0.9/";

/***** Some general utils *****/
function strToURI(link, base) {
  var base = base || null;
  try {
    return gIoService.newURI(link, null, base);
  }
  catch(e) {
    return null;
  }
}

function isArray(a) {
  return isObject(a) && a.constructor == Array;
}

function isObject(a) {
  return (a && typeof a == "object") || isFunction(a);
}

function isFunction(a) {
  return typeof a == "function";
}

function isIID(a, iid) {
  var rv = false;
  try {
    a.QueryInterface(iid);
    rv = true;
  }
  catch(e) {
  }
  return rv;
}

function isIArray(a) {
  return isIID(a, Ci.nsIArray);
}

function isIFeedContainer(a) {
  return isIID(a, Ci.nsIFeedContainer);
}

function stripTags(someHTML) {
  return someHTML.replace(/<[^>]+>/g,"");
}

/**
 * Searches through an array of links and returns a JS array 
 * of matching property bags.
 */
const IANA_URI = "http://www.iana.org/assignments/relation/";
function findAtomLinks(rel, links) {
  var rvLinks = [];
  for (var i = 0; i < links.length; ++i) {
    var linkElement = links.queryElementAt(i, Ci.nsIPropertyBag2);
    // atom:link MUST have @href
    if (bagHasKey(linkElement, "href")) {
      var relAttribute = null;
      if (bagHasKey(linkElement, "rel"))
        relAttribute = linkElement.getPropertyAsAString("rel")
      if ((!relAttribute && rel == "alternate") || relAttribute == rel) {
        rvLinks.push(linkElement);
        continue;
      }
      // catch relations specified by IANA URI 
      if (relAttribute == IANA_URI + rel) {
        rvLinks.push(linkElement);
      }
    }
  }
  return rvLinks;
}

function xmlEscape(s) {
  s = s.replace(/&/g, "&amp;");
  s = s.replace(/>/g, "&gt;");
  s = s.replace(/</g, "&lt;");
  s = s.replace(/"/g, "&quot;");
  s = s.replace(/'/g, "&apos;");
  return s;
}

function arrayContains(array, element) {
  for (var i = 0; i < array.length; ++i) {
    if (array[i] == element) {
      return true;
    }
  }
  return false;
}

// XXX add hasKey to nsIPropertyBag
function bagHasKey(bag, key) {
  try {
    bag.getProperty(key);
    return true;
  }
  catch (e) {
    return false;
  }
}

function makePropGetter(key) {
  return function FeedPropGetter(bag) {
    try {
      return value = bag.getProperty(key);
    }
    catch(e) {
    }
    return null;
  }
}



/**
 * XXX Thunderbird's W3C-DTF function
 *
 * Converts a W3C-DTF (subset of ISO 8601) date string to an IETF date
 * string.  W3C-DTF is described in this note:
 * http://www.w3.org/TR/NOTE-datetime IETF is obtained via the Date
 * object's toUTCString() method.  The object's toString() method is
 * insufficient because it spells out timezones on Win32
 * (f.e. "Pacific Standard Time" instead of "PST"), which Mail doesn't
 * grok.  For info, see
 * http://lxr.mozilla.org/mozilla/source/js/src/jsdate.c#1526.
 */
const HOURS_TO_MINUTES = 60;
const MINUTES_TO_SECONDS = 60;
const SECONDS_TO_MILLISECONDS = 1000;
const MINUTES_TO_MILLISECONDS = MINUTES_TO_SECONDS * SECONDS_TO_MILLISECONDS;
const HOURS_TO_MILLISECONDS = HOURS_TO_MINUTES * MINUTES_TO_MILLISECONDS;
function W3CToIETFDate(dateString) {

  var parts = dateString.match(/(\d\d\d\d)(-(\d\d))?(-(\d\d))?(T(\d\d):(\d\d)(:(\d\d)(\.(\d+))?)?(Z|([+-])(\d\d):(\d\d))?)?/);

  // Here's an example of a W3C-DTF date string and what .match returns for it.
  // 
  // date: 2003-05-30T11:18:50.345-08:00
  // date.match returns array values:
  //
  //   0: 2003-05-30T11:18:50-08:00,
  //   1: 2003,
  //   2: -05,
  //   3: 05,
  //   4: -30,
  //   5: 30,
  //   6: T11:18:50-08:00,
  //   7: 11,
  //   8: 18,
  //   9: :50,
  //   10: 50,
  //   11: .345,
  //   12: 345,
  //   13: -08:00,
  //   14: -,
  //   15: 08,
  //   16: 00

  // Create a Date object from the date parts.  Note that the Date
  // object apparently can't deal with empty string parameters in lieu
  // of numbers, so optional values (like hours, minutes, seconds, and
  // milliseconds) must be forced to be numbers.
  var date = new Date(parts[1], parts[3] - 1, parts[5], parts[7] || 0,
                      parts[8] || 0, parts[10] || 0, parts[12] || 0);

  // We now have a value that the Date object thinks is in the local
  // timezone but which actually represents the date/time in the
  // remote timezone (f.e. the value was "10:00 EST", and we have
  // converted it to "10:00 PST" instead of "07:00 PST").  We need to
  // correct that.  To do so, we're going to add the offset between
  // the remote timezone and UTC (to convert the value to UTC), then
  // add the offset between UTC and the local timezone //(to convert
  // the value to the local timezone).

  // Ironically, W3C-DTF gives us the offset between UTC and the
  // remote timezone rather than the other way around, while the
  // getTimezoneOffset() method of a Date object gives us the offset
  // between the local timezone and UTC rather than the other way
  // around.  Both of these are the additive inverse (i.e. -x for x)
  // of what we want, so we have to invert them to use them by
  // multipying by -1 (f.e. if "the offset between UTC and the remote
  // timezone" is -5 hours, then "the offset between the remote
  // timezone and UTC" is -5*-1 = 5 hours).

  // Note that if the timezone portion of the date/time string is
  // absent (which violates W3C-DTF, although ISO 8601 allows it), we
  // assume the value to be in UTC.

  // The offset between the remote timezone and UTC in milliseconds.
  var remoteToUTCOffset = 0;
  if (parts[13] && parts[13] != "Z") {
    var direction = (parts[14] == "+" ? 1 : -1);
    if (parts[15])
      remoteToUTCOffset += direction * parts[15] * HOURS_TO_MILLISECONDS;
    if (parts[16])
      remoteToUTCOffset += direction * parts[16] * MINUTES_TO_MILLISECONDS;
  }
  remoteToUTCOffset = remoteToUTCOffset * -1; // invert it

  // The offset between UTC and the local timezone in milliseconds.
  var UTCToLocalOffset = date.getTimezoneOffset() * MINUTES_TO_MILLISECONDS;
  UTCToLocalOffset = UTCToLocalOffset * -1; // invert it
  date.setTime(date.getTime() + remoteToUTCOffset + UTCToLocalOffset);

  return date.toUTCString();
}

const RDF_NS = "http://www.w3.org/1999/02/22-rdf-syntax-ns#";
// namespace map
var gNamespaces = {
  "http://webns.net/mvcb/":"admin",
  "http://backend.userland.com/rss":"",
  "http://blogs.law.harvard.edu/tech/rss":"",
  "http://www.w3.org/2005/Atom":"atom",
  "http://purl.org/atom/ns#":"atom03",
  "http://purl.org/rss/1.0/modules/content/":"content",
  "http://purl.org/dc/elements/1.1/":"dc",
  "http://purl.org/dc/terms/":"dcterms",
  "http://www.w3.org/1999/02/22-rdf-syntax-ns#":"rdf",
  "http://purl.org/rss/1.0/":"rss1",
  "http://my.netscape.com/rdf/simple/0.9/":"rss1",
  "http://wellformedweb.org/CommentAPI/":"wfw",                              
  "http://purl.org/rss/1.0/modules/wiki/":"wiki", 
  "http://www.w3.org/XML/1998/namespace":"xml"
}


function FeedResult() {}
FeedResult.prototype = {
  bozo: false,
  doc: null,
  version: null,
  headers: null,
  uri: null,
  stylesheet: null,

  registerExtensionPrefix: function FR_registerExtensionPrefix(ns, prefix) {
    throw Cr.NS_ERROR_NOT_IMPLEMENTED;
  },

  QueryInterface: function FR_QI(iid) {
    if (iid.equals(Ci.nsIFeedResult) ||
        iid.equals(Ci.nsISupports))
      return this;

    throw Cr.NS_ERROR_NOINTERFACE;
  },
}  

function Feed() {
  this.subtitle = null;
  this.title = null;
  this.items = Cc[ARRAY_CONTRACTID].createInstance(Ci.nsIMutableArray);
  this.link = null;
  this.id = null;
  this.generator = null;
  this.authors = Cc[ARRAY_CONTRACTID].createInstance(Ci.nsIMutableArray);
  this.contributors = Cc[ARRAY_CONTRACTID].createInstance(Ci.nsIMutableArray);
  this.baseURI = null;
}

Feed.prototype = {
  searchLists: {
    subtitle: ["description","dc:description","rss1:description",
               "atom03:tagline","atom:subtitle"],
    items: ["items","atom03_entries","entries"],
    id: ["atom:id","rdf:about"],
    generator: ["generator"],
    authors : ["authors"],
    contributors: ["contributors"],
    title: ["title","rss1:title", "atom03:title","atom:title"],
    link:  [["link",strToURI],["rss1:link",strToURI]],
    categories: ["categories", "dc:subject"],
    rights: ["atom03:rights","atom:rights"],
    cloud: ["cloud"],
    image: ["image", "rss1:image"],
    textInput: ["textInput", "rss1:textinput"],
    skipDays: ["skipDays"],
    skipHours: ["skipHours"],
    updated: ["pubDate", "lastBuildDate", "atom03:modified", "dc:date",
              "dcterms:modified", "atom:updated"]
  },

  normalize: function Feed_normalize() {
    fieldsToObj(this, this.searchLists);
    if (this.skipDays)
      this.skipDays = this.skipDays.getProperty("days");
    if (this.skipHours)
      this.skipHours = this.skipHours.getProperty("hours");

    if (this.updated)
      this.updated = dateParse(this.updated);

    // Assign Atom link if needed
    if (bagHasKey(this.fields, "links"))
      this._atomLinksToURI();

    this._resetBagMembersToRawText([this.searchLists.subtitle, 
                                    this.searchLists.title]);
  },

  _atomLinksToURI: function Feed_linkToURI() {
    var links = this.fields.getPropertyAsInterface("links", Ci.nsIArray);
    var alternates = findAtomLinks("alternate", links);
    if (alternates.length > 0) {
      try {
        var href = alternates[0].getPropertyAsAString("href");
        var base;
        if (bagHasKey(alternates[0], "xml:base"))
          base = strToURI(alternates[0].getPropertyAsAString("xml:base"),
                          this.baseURI);
        else
          base = this.baseURI;
        this.link = strToURI(alternates[0].getPropertyAsAString("href"), base);
      }
      catch(e) {
        LOG(e);
      }
    }
  },

  // reset the bag to raw contents, not text constructs
  _resetBagMembersToRawText: function Feed_resetBagMembers(fieldLists) {
    for (var i=0; i<fieldLists.length; i++) {      
      for (var j=0; j<fieldLists[i].length; j++) {
        if (bagHasKey(this.fields, fieldLists[i][j])) {
          var textConstruct = this.fields.getProperty(fieldLists[i][j]);
          this.fields.setPropertyAsAString(fieldLists[i][j],
                                           textConstruct.text);
        }
      }
    }
  },
   
  QueryInterface: function Feed_QI(iid) {
    if (iid.equals(Ci.nsIFeed) ||
        iid.equals(Ci.nsIFeedContainer) ||
        iid.equals(Ci.nsISupports))
    return this;
    throw Cr.NS_ERROR_NOINTERFACE;
  }
}

function Entry() {
  this.summary = null;
  this.content = null;
  this.title = null;
  this.fields = Cc["@mozilla.org/hash-property-bag;1"].
    createInstance(Ci.nsIWritablePropertyBag2);
  this.link = null;
  this.id = null;
  this.baseURI = null;
  this.updated = null;
  this.published = null;
  this.authors = Cc[ARRAY_CONTRACTID].createInstance(Ci.nsIMutableArray);
  this.contributors = Cc[ARRAY_CONTRACTID].createInstance(Ci.nsIMutableArray);
}
  
Entry.prototype = {
  fields: null,
  enclosures: null,
  mediaContent: null,
  
  searchLists: {
    title: ["title", "rss1:title", "atom03:title", "atom:title"],
    link: [["link",strToURI],["rss1:link",strToURI]],
    id: [["guid", makePropGetter("guid")], "rdf:about",
         "atom03:id", "atom:id"],
    authors : ["authors"],
    contributors: ["contributors"],
    summary: ["description", "rss1:description", "dc:description",
              "atom03:summary", "atom:summary"],
    content: ["content:encoded","atom03:content","atom:content"],
    rights: ["atom03:rights","atom:rights"],
    published: ["atom03:issued", "dcterms:issued", "atom:published"],
    updated: ["pubDate", "atom03:modified", "dc:date", "dcterms:modified",
              "atom:updated"]
  },
  
  normalize: function Entry_normalize() {
    fieldsToObj(this, this.searchLists);
 
    // Assign Atom link if needed
    if (bagHasKey(this.fields, "links"))
      this._atomLinksToURI();
 
    // The link might be a guid w/ permalink=true
    if (!this.link && bagHasKey(this.fields, "guid")) {
      var guid = this.fields.getProperty("guid");
      var isPermaLink = true;