rss2sql.php

RSS to SQL to store in Database

<?php
//
// rss2sql.php Store RSS feeds into SQL
//
// Copyright 2006-2007 NotePage, Inc.
// http://www.feedforall.com
//
// NotePage, Inc. grants registerd users of our FeedForAll and/or
// FeedForAll Mac product(s) the right to install and use the
// rss2sql.php script free of charge.
// Please refer to the EULA included in the download for full license
// terms and conditions.
//
// $Id: rss2sql.php,v 3.0 2007/04/16 14:23:04 housley Exp $
//
// $Log: rss2sql.php,v $
// Revision 3.0  2007/04/16 14:23:04  housley
// Release version 3.0 of the scripts
//
// Revision 1.24  2007/04/13 18:30:10  housley
// * Atom:content might need whole string so always make it available
// * atom:content of type xhtml is in a div that needs to be stripped and
//   then used as is.
//
// Revision 1.23  2007/04/11 18:36:57  housley
// Set user agent
//
// Revision 1.22  2007/04/09 01:25:50  housley
// Fix the commented out code for enabling the extensions
//
// Revision 1.21  2007/04/06 11:08:58  housley
// Add support for the Dublin Core (dc) namespace
//
// Revision 1.20  2007/04/04 20:55:46  housley
// Add the ability to set CURLOPT_CONNECTTIMEOUT
//
// Revision 1.19  2007/03/24 16:32:52  housley
// Set both character set values
//
// Revision 1.18  2007/03/24 15:52:24  housley
// * Talk the database in UTF-8
// * Include the Extensions before the parser
// * Move the extensions to before the parser
//
// Revision 1.17  2007/03/19 14:13:24  housley
// Fix some small bugs in the new code, and test
//
// Revision 1.16  2007/03/19 13:48:30  housley
// Allow for easier automated testing
//
// Revision 1.15  2007/03/07 03:01:22  housley
// Add the flag to not parse TrackBack
//
// Revision 1.14  2007/03/05 18:23:58  housley
// Don't abort processing on XML parse error, just don't do anything else
//
// Revision 1.13  2007/03/05 17:29:46  housley
// Add a simple way to block parsing extensions
//
// Revision 1.12  2007/03/05 16:47:19  housley
// * Add support to actually store the pubDate as a time value
// * Fix adding "category" to the database
//
// Revision 1.11  2007/03/05 01:33:45  housley
// Use a command convert and readFile routines
//
// Revision 1.10  2007/03/04 02:10:08  housley
// Move the parser used by the paid scripts into its own file.
//
// Revision 1.9  2007/03/03 21:10:09  housley
// * Make the item a full class object
// * Support parsing the iTunes(R) extension
//
// Revision 1.8  2007/02/26 15:37:31  housley
// Log changes
//
//


//
// Destination Encoding:  By default rss2html.php converts all feeds to UTF-8
// and then produces webpages in UTF-8 because UTF-8 is capable of displaying
// all possible characters.
$destinationEncoding = "UTF-8";

//
// Missing Encoding Default:  Some feeds do not specify the character set
// they are encoded in.  The XML specification states that if there is no
// encoding specified the XML file, all RSS feeds are XML, must be encoded
// in UTF-8, but experience has show differently.  This specifies the
// encoding that will be used for feeds that don't specify the encoding.
//$missingEncodingDefault = "UTF-8";
$missingEncodingDefault = "ISO-8859-1";

//
// The maximum number of errors allowed before rss2sql automatically
// marks the feed as inactive.
$maxErrors = 10;

//
// $connectTimeoutLimit allows the limiting the amount of time cURL will
// wait to successfully connect to a remote server.  Use with caution,
// a value too small will cause all connections to fail.
//$connectTimeoutLimit = 30;

//
// To prevent parsing of the iTunes(R) XML namespace extension, uncomment the
// following line.
//$Dont_Parse_iTunes == TRUE;
//
// To prevent parsing of the TrackBack XML namespace extension, uncomment the
// following line.
//$Dont_Parse_TrackBack == TRUE;
//
// To prevent parsing of the DublinCore XML namespace extension, uncomment the
// following line.
//$Dont_Parse_DublinCore == TRUE;

//
// Set the following variable useFopenURL to one if you want/need to use
// fopen() instead of CURL
$useFopenURL = 0;


// ==========================================================================
// Below this point of the file there are no user editable options.  Your
// are welcome to make any modifications that you wish to any of the code
// below, but that is not necessary for normal use.
// ==========================================================================

//
// If using cURL, make sure it exists
if (($useFopenURL == 0) && !function_exists("curl_init")) {
  $useFopenURL = 1;
}

if ($useFopenURL) {
  ini_set("allow_url_fopen", "1");
  ini_set("user_agent", 'FeedForAll rss2sql.php v3');
}

@include("FeedForAll_parse_Extensions.inc.php");
if (function_exists("FeedForAll_parseExtensions") === FALSE) {
  Function FeedForAll_parseExtensions() {
    return FALSE;
  }
}

@include("FeedForAll_XMLParser.inc.php");

$ConfigFilename = "rss2sql_SampleConfig.php";
if (isset($_REQUEST["CONFIGFILE"])) {
  if (stristr($_REQUEST["CONFIGFILE"], "file"."://")) {
    // Not allowed
    ;
  }
  elseif (stristr($_REQUEST["CONFIGFILE"], "://")) {
    // Not allowed
    ;
  } else {
    // It is local and must be in the same directory
    $ConfigFilename = basename($_REQUEST["CONFIGFILE"]);
  }
}

Function rss2sql_CreateUniqueLink($title, $description, $link, $guid) {
  if (($title == "") && ($description == "") && ($guid == "")) {
    // There is on information that a link can be made of
    $result = "";
  }
  elseif ($guid != "") {
    // We have a GUID, good feed creator
    $result = $guid;
  } else {
    // No GUID, bad feed creator, use a hash of the title and description
    $result = md5($title.$description);
  }

  return $result;
}

//
// Read in the configuration file
$config = Array();

if (isset($_REQUEST["buildConfig"])) {
  ob_start();
  @include_once($ConfigFilename);
  ob_end_clean();

  // If this is not a post, then use values from the file
  if (($_SERVER["REQUEST_METHOD"] != "POST") || (isset($_REQUEST["TEST"]) && ($_REQUEST["TEST"] == "Testing"))) {
    $_POST["DBconfigFile"] = "";
    if (isset($config["DBconfigFile"])) {
      $_POST["DBconfigFile"] = $config["DBconfigFile"];
    }
    if ($_POST["DBconfigFile"] == "") {
      $_POST["DBconfigFile"] = "rss2sql_DBconfig.inc.php";
    }
    $_POST["feedInfo"] = "feedInfo";
    if (isset($config["feedInfo"])) {
      $_POST["feedInfo"] = $config["feedInfo"];
    }
    $_POST["feedID"] = "feedID";
    if (isset($config["feedID"])) {
      $_POST["feedID"] = $config["feedID"];
    }
    $_POST["feedActive"] = "feedActive";
    if (isset($config["feedActive"])) {
      $_POST["feedActive"] = $config["feedActive"];
    }
    $_POST["feedURL"] = "feedURL";
    if (isset($config["feedURL"])) {
      $_POST["feedURL"] = $config["feedURL"];
    }
    $_POST["updateInterval"] = "updateInterval";
    if (isset($config["updateInterval"])) {
      $_POST["updateInterval"] = $config["updateInterval"];
    }
    $_POST["lastUpdate"] = "lastUpdate";
    if (isset($config["lastUpdate"])) {
      $_POST["lastUpdate"] = $config["lastUpdate"];
    }
    $_POST["nextUpdate"] = "nextUpdate";
    if (isset($config["nextUpdate"])) {
      $_POST["nextUpdate"] = $config["nextUpdate"];
    }
    $_POST["updateStatus"] = "updateStatus";
    if (isset($config["updateStatus"])) {
      $_POST["updateStatus"] = $config["updateStatus"];
    }
    $_POST["updateStatusCount"] = "updateStatusCount";
    if (isset($config["updateStatusCount"])) {
      $_POST["updateStatusCount"] = $config["updateStatusCount"];
    }

    $_POST["ItemTable"] = "feedItems";
    if (isset($config["ItemTable"])) {
      $_POST["ItemTable"] = $config["ItemTable"];
    }
    $_POST["itemID"] = "itemID";
    if (isset($config["itemID"])) {
      $_POST["itemID"] = $config["itemID"];
    }
    $_POST["UniqueItemID"] = "CreatedUniqueID";
    if (isset($config["UniqueItemID"])) {
      $_POST["UniqueItemID"] = $config["UniqueItemID"];
    }
    $_POST["ItemAddedTime"] = "ItemAddedTime";
    if (isset($config["ItemAddedTime"])) {
      $_POST["ItemAddedTime"] = $config["ItemAddedTime"];
    }
    $_POST["ItemTitle"] = "ItemTitle";
    if (isset($config["ItemTitle"])) {
      $_POST["ItemTitle"] = $config["ItemTitle"];
    }
    $_POST["ItemDescription"] = "ItemDescription";
    if (isset($config["ItemDescription"])) {
      $_POST["ItemDescription"] = $config["ItemDescription"];
    }
    $_POST["ItemContentEncoded"] = "ItemContentEncoded";
    if (isset($config["ItemContentEncoded"])) {
      $_POST["ItemContentEncoded"] = $config["ItemContentEncoded"];
    }
    $_POST["ItemLink"] = "ItemLink";
    if (isset($config["ItemLink"])) {
      $_POST["ItemLink"] = $config["ItemLink"];
    }
    $_POST["ItemPubDate"] = "ItemPubDate";
    if (isset($config["ItemPubDate"])) {
      $_POST["ItemPubDate"] = $config["ItemPubDate"];
    }
    $_POST["ItemPubDate_t"] = "ItemPubDate_t";
    if (isset($config["ItemPubDate_t"])) {
      $_POST["ItemPubDate_t"] = $config["ItemPubDate_t"];
    }
    $_POST["ItemEnclosureUrl"] = "ItemEnclosureUrl";
    if (isset($config["ItemEnclosureUrl"])) {
      $_POST["ItemEnclosureUrl"] = $config["ItemEnclosureUrl"];
    }
    $_POST["ItemEnclosureType"] = "ItemEnclosureType";
    if (isset($config["ItemEnclosureType"])) {
      $_POST["ItemEnclosureType"] = $config["ItemEnclosureType"];
    }
    $_POST["ItemEnclosureLength"] = "ItemEnclosureLength";
    if (isset($config["ItemEnclosureLength"])) {
      $_POST["ItemEnclosureLength"] = $config["ItemEnclosureLength"];
    }
    $_POST["ItemGuid"] = "ItemGuid";
    if (isset($config["ItemGuid"])) {