feedapi.module

<?php

/**
 * @file
 * Handle the submodules (for feed and item processing)
 * Provide a basic management of feeds
 */

define('FEEDAPI_NEVER_DELETE_OLD', 0);
define('FEEDAPI_TIMEOUT', 1);
// Number of feeds to process for each step in cron.
define('FEEDAPI_CRON_FEEDS', 100);
// Default time that should elapse before a feed can be refreshed again on cron.
define('FEEDAPI_CRON_DEFAULT_REFRESH_TIME', 1800);
// Denotes that a feed should never be refreshed.
define('FEEDAPI_CRON_NEVER_REFRESH', -1);
// Denotes that a feed should be refreshed as often as possible.
define('FEEDAPI_CRON_ALWAYS_REFRESH', 0);
// Prune FeedAPI stats 4 weeks
define('FEEDAPI_CRON_STAT_LIFETIME', 28*24*3600);

/**
 * Implementation of hook_help().
 */
function feedapi_help($path, $arg) {
  switch ($path) {
    case 'admin/help#feedapi':
      $output = '<p>'. t('Provides feed management interface and handles underlying processors and parsers for any type of feeds.') .'</p>';
      $output .= '<p>'. t('Feeds are based on content types. Default content types are created on install. You can create new content types on the <a href="@content-types">add content types</a> page. To do that, enable the "Is a feed content type" checkbox under the Feed API group on the content type edit form. Then choose the processors and parsers that you would like to use. At least one parser and one processor must be enabled.', array('@content-types' => url('admin/content/types/add'))) .'</p>';
      return $output;
    case 'admin/content/feed':
      return '<p>'. t('Current feeds are listed below. For each FeedAPI-enabled content type, the <em>Quick create</em> block may be enabled at the <a href="@block">blocks administration page</a>.', array('@block' => url('admin/build/block'))) .'</p>';
    case 'admin/content/feed/import_opml':
      return '<p>'. t('Feeds can be imported from a valid OPML file. You can check your OPML file at <a href="@validator">OPML Validator</a>.', array('@validator' => url('http://validator.opml.org/'))) .'</p>';
    case 'admin/settings/feedapi':
      return '<p>'. t('You can find more configuration options on the content type edit form of FeedAPI-enabled <a href="@content-types">content types</a>.', array('@content-types' => url('admin/content/types'))) .'</p>';
  }
}

/**
 * Implementation of hook_theme().
 */
function feedapi_theme() {
  return array(
    'feedapi_export_opml' => array(
      'arguments' => array('feeds' => NULL),
    ),
  );
}

/**
 * Implementation of hook_menu().
 */
function feedapi_menu() {
  $items = array();
  $items['admin/content/feed'] = array(
    'title' => 'Feeds',
    'description' => 'Overview which content your site aggregates from other sites and see detailed statistics about the feeds.',
    'page callback' => 'feedapi_admin_overview',
    'access arguments' => array('administer feedapi'),
    'file' => 'feedapi.admin.inc',
  );
  $items['admin/content/feed/list'] = array(
    'title' => 'List',
    'type' => MENU_DEFAULT_LOCAL_TASK,
    'access arguments' => array('administer feedapi'),
    'weight' => -15,
  );
  $items['admin/content/feed/import_opml'] = array(
    'title' => 'Import OPML',
    'access arguments' => array('administer feedapi'),
    'page callback' => 'drupal_get_form',
    'page arguments' => array('feedapi_import_opml'),
    'file' => 'feedapi.opml.inc',
  );
  $items['admin/content/feed/export_opml'] = array(
    'title' => 'Export all feeds as OPML',
    'access arguments' => array('administer feedapi'),
    'page callback' => 'feedapi_export_opml',
    'file' => 'feedapi.opml.inc',
  );
  $items['admin/settings/feedapi'] = array(
    'title' => 'FeedAPI',
    'description' => 'Configure advanced options for FeedAPI module.',
    'page callback' => 'drupal_get_form',
    'page arguments' => array('feedapi_admin_settings'),
    'access arguments' => array('administer feedapi'),
    'file' => 'feedapi.admin.inc',
  );

  $items['node/%node/refresh'] = array(
    'title' => 'Refresh',
    'page callback' => 'feedapi_refresh',
    'page arguments' => array(1),
    'type' => MENU_LOCAL_TASK,
    'access callback' => '_feedapi_op_access',
    'access arguments' => array(1),
  );
  $items['node/%node/purge'] = array(
    'title' => 'Remove items',
    'page callback' => 'feedapi_invoke',
    'page arguments' => array("purge", 1, 'items'),
    'type' => MENU_LOCAL_TASK,
    'access callback' => '_feedapi_op_access',
    'access arguments' => array(1),
  );
  return $items;
}

function _feedapi_op_access($node) {
  if (!feedapi_enabled_type($node->type)) {
    return FALSE;
  }
  global $user;
  $own_feed = $node->uid == $user->uid && user_access('edit own '. $node->type .' content') ? TRUE : FALSE;
  return user_access('administer feedapi') || $own_feed;
}

/**
 * Implementation of hook_nodeapi().
 */
function feedapi_nodeapi(&$node, $op, $teaser, $page) {
  if (isset($node->feed) || feedapi_enabled_type($node->type)) {
    switch ($op) {
      case 'validate':
        $node->feed->settings = feedapi_get_settings($node->type);
        $node->feed->parsers = _feedapi_format_settings($node->feed->settings, 'parsers');
        $node->feed->processors = _feedapi_format_settings($node->feed->settings, 'processors');
        if (count($node->feed->parsers) < 1) {
          if (user_access('administer content types')) {
            form_set_error('', t('There are no enabled parsers for this content type. In order to import feed items, you need to select a feed parser from the <a href="@url">content type settings</a>.', array('@url' => url("admin/content/node-type/$node->type"))));
          }
          else {
            form_set_error('', t('There is no parser enabled for this content-type. Contact your site administrator for help.'));
          }
        }
        if (count($node->feed->processors) < 1) {
          if (user_access('administer content types')) {
            form_set_error('', t('There are no enabled processors for this content type. In order to import feed items, you need to select a processor from the <a href="@url">content type settings</a>.', array('@url' => url("admin/content/node-type/$node->type"))));
          }
          else {
            form_set_error('', t('There is no processor enabled for this content-type. Contact your site administrator for help.'));
          }
        }
        break;
      case 'insert':
        _feedapi_insert($node);
        break;
      case 'update':
        _feedapi_update($node);
        break;
      case 'load':
        if ($feed = db_fetch_object(db_query('SELECT * FROM {feedapi} WHERE vid = %d', $node->vid))) {
          $node->feed = $feed;
          $node->feed->vid = $node->vid;
          $node->feed->nid = $node->nid;
          $node->feed->settings = feedapi_get_settings($node->type, $node->vid);
          // Load parsers and processors from content type
          $node_type_settings = feedapi_get_settings($node->type);
          $node->feed->parsers = _feedapi_format_settings($node_type_settings, 'parsers');
          $node->feed->processors = _feedapi_format_settings($node_type_settings, 'processors');
        }
        break;
      case 'delete':
        // Could be a performance problem - think of thousands of node feed items.
        //  This is a temporary status. See: http://drupal.org/node/195723
        // feedapi_invoke('purge', $node->feed);
        db_query("DELETE FROM {feedapi_stat} WHERE id = %d", $node->nid);
        db_query("DELETE FROM {feedapi} WHERE nid = %d", $node->nid);
        break;
      case 'presave':
        if (is_array($node->feedapi) || isset($node->feedapi_object)) {
          $node->feed = isset($node->feedapi_object) ? $node->feedapi_object : _feedapi_build_feed_object($node->type, $node->feedapi['feedapi_url']);
        }
        break;
      case 'delete revision':
        db_query("DELETE FROM {feedapi} WHERE nid = %d AND vid = %d", $node->nid, $node->vid);
        break;
    }
  }
}

/**
 * Implementation of hook_node_type().
 */
function feedapi_node_type($op, $info) {
  switch ($op) {
    case 'delete':
      variable_del('feedapi_settings_'. $info->type);
      variable_del('feedapi_'. $info->type);
      break;
    case 'update':
      if (!empty($info->old_type) && $info->old_type != $info->type) {
        $setting = variable_get('feedapi_settings_'. $info->old_type, array());
        variable_del('feedapi_settings_'. $info->old_type);
        variable_set('feedapi_settings_'. $info->type, $setting);
      }
      break;
  }
}

/**
 * Implementation of hook_block().
 */
function feedapi_block($op = 'list', $delta = 0) {
  $blocks = array();
  $names = feedapi_get_types();
  switch ($op) {
    case 'list':
      foreach ($names as $type => $name) {
        $blocks[$type]['info'] = t('FeedAPI: Quick create !preset', array('!preset' => $name));
        $blocks[$type]['cache'] = BLOCK_CACHE_GLOBAL;
      }
      break;
    case 'view':
      if (node_access('create', $delta)) {
        $blocks['subject'] = t('Create !preset', array('!preset' => $names[$delta]));
        $blocks['content'] = drupal_get_form('feedapi_simplified_form', $delta);
      }
      break;
  }
  return $blocks;
}

/**
 * Implementation of hook_perm().
 */
function feedapi_perm() {
  return array('administer feedapi', 'advanced feedapi options', 'use local files as feeds');
}

/**
 * Implementation of hook_link().
 */
function feedapi_link($type, $node = NULL) {
  if ($type == 'node' && isset($node->feed)) {
    if (strlen($node->feed->link) > 0) {
      $links['feedapi_original'] = array(
        'title' => t('Link to site'),
        'href' => $node->feed->link,
      );
      return $links;
    }
  }
}

/**
 * Implementation of hook_node_views().
 */
function feedapi_views_api() {
  return array(
    'api' => 2,
    'path' => drupal_get_path('module', 'feedapi') .'/views',
  );
}

/**
 * Invoke feedapi API callback functions.
 *
 * @param $op
 *  "load"    Load the feed items basic data into the $feed->items[]
 *  "refresh"  Re-download the feed and process newly arrived item
 *  "purge"    Delete all the feed items
 *
 * @param $feed
 *  A feed object. If only the ID is known, you should pass something like this: $feed->nid = X
 * @param $param
 *  Depends on the $op value.
 */
function feedapi_invoke($op, &$feed, $param = NULL) {
  if (!is_object($feed)) {
    return FALSE;
  }
  // The node is passed.
  if (isset($feed->feed) && is_object($feed->feed)) {
    $feed = $feed->feed;
  }
  if (!isset($feed->processors)) {
    $node = node_load($feed->nid);
    if (!isset($node->feed)) {
      return FALSE;
    }
    $feed = $node->feed;
  }
  _feedapi_sanitize_processors($feed);

  switch ($op) {
    case 'refresh':
      return _feedapi_invoke_refresh($feed, $param);
    case 'purge':
      return _feedapi_invoke_purge($feed, $param);
    default: // Other operations
      return _feedapi_invoke($op, $feed, $param);
  }
}

/**
 * Ask for confirmation before deleting all the items
 */
function feedapi_purge_confirm($form_state, $node) {
  $output = confirm_form(
    array('nid' => array('#type' => 'hidden', '#value' => $node->nid)),
    t('Delete all the feed items from !name', array('!name' => $node->title)),
    isset($_GET['destination']) ? $_GET['destination'] : 'node/'. $node->nid,
    t("Are you sure you want to delete all the feed items from !name?", array('!name' => $node->title)),
    t('Yes'), t('No'),
    'feedapi_purge_confirm'
  );
  return $output;
}

/**
 * Submitted items purging form. Drop all the items.
 */
function feedapi_purge_confirm_submit($form, &$form_state) {
  $feed->nid = $form_state['values']['nid'];
  feedapi_invoke('purge', $feed);
  $form_state['redirect'] = 'node/'.  $form_state['values']['nid'];
}

/**
 * Delete expired items and return informations about the feed refreshing
 *
 * @param $feed
 *  The feed object
 * @param $settings
 *  Optional feed settings
 * @return
 *  FALSE if the feed don't have to be refreshed. (forbidden if the $force is TRUE)
 */
function feedapi_expire($feed, $settings = NULL) {
  // Backwards compatibility, get settings if not passed
  $settings = is_null($settings) ? feedapi_get_settings(NULL, $feed->vid) : $settings;
  // Each processor can have its own expiration criteria ?
  $expired = _feedapi_invoke('expire', $feed, $settings);
  // Return the number of expired items
  return $expired ? array_sum($expired) : 0;
}

/**
 * Callback for expired items. Does the actual deleting
 */
function feedapi_expire_item($feed, $item) {
  foreach ($feed->processors as $processor) {
    module_invoke($processor, 'feedapi_item', 'delete', $item, $feed->nid);
  }
}

/**
 * Implementation of hook_form_alter().
 */
function feedapi_form_alter(&$form, $form_state, $form_id) {
  // Content type form.
  if ($form_id == 'node_type_form' && isset($form['identity']['type'])) {
    $node_type_settings = feedapi_get_settings($form['#node_type']->type);

    $form['#validate'][] = 'feedapi_content_type_validate';

    // Don't blow away existing form elements.
    if (!isset($form['feedapi'])) {
      $form['feedapi'] = array();
    }
    $form['feedapi'] += array(
      '#type' => 'fieldset',
      '#title' => t('Feed API'),
      '#collapsible' => TRUE,
      '#collapsed' => isset($node_type_settings['enabled']) ? !($node_type_settings['enabled']) : TRUE,
      '#tree' => TRUE,
    );
    $form['feedapi']['enabled'] = array(
      '#type' => 'checkbox',
      '#title' => t('Is a feed content type'),
      '#description' => t('Check if you want to use this content type for downloading feeds to your site.'),
      '#default_value' => isset($node_type_settings['enabled']) ? $node_type_settings['enabled'] : FALSE,
      '#weight' => -15,
    );
    $form['feedapi']['upload_method'] = array(
      '#type' => 'radios',
      '#title' => t('Supply feed as'),
      '#description' => t('Select how a user will supply a feed. Choose URL if the user will paste a URL to a textfield, choose File upload if the user will upload a feed from the local disk.'),
      '#options' => array('url' => t('URL'), 'upload' => t('File upload')),
      '#default_value' => isset($node_type_settings['upload_method']) ? $node_type_settings['upload_method'] : 'url',
      '#weight' => -14,
    );
    $modules = module_implements('feedapi_settings_form');
    foreach ($modules as $module) {
      $form['feedapi']['defaults'] = array('#type' => 'markup', '#value' => '<strong>'. t('Default settings') .'</strong><hr/>');
      if ($feedapi_form = module_invoke($module, 'feedapi_settings_form', 'general')) {
        $form['feedapi'] = array_merge_recursive($form['feedapi'], $feedapi_form);
      }
    }
    $form['feedapi']['parsers'] = array(
      '#type' => 'fieldset',
      '#title' => t('Parser settings'),
      '#description' => t('Parsers turn a feed into an object ready for processing. Choose at least one.'),
      '#collapsible' => FALSE,
      '#tree' => TRUE,
    );
    $parsers = module_implements('feedapi_feed', TRUE);
    rsort($parsers);
    foreach ($parsers as $parser) {
      $form['feedapi']['parsers'][$parser] = array(
        '#type' => 'fieldset',
        '#title' => feedapi_get_natural_name($parser),
        '#collapsible' => TRUE,
        '#collapsed' => isset($node_type_settings['parsers'][$parser]['enabled']) ? !($node_type_settings['parsers'][$parser]['enabled']) : TRUE,
        '#tree' => TRUE,
        '#weight' => isset($node_type_settings['parsers'][$parser]['weight']) ? $node_type_settings['parsers'][$parser]['weight']: 0,
      );
      $form['feedapi']['parsers'][$parser]['enabled'] = array(
        '#type' => 'checkbox',
        '#title' => t('Enable'),
        '#description' => t('Check this box if you want to enable the @name parser on this feed.', array('@name' => $parser)),
        '#default_value' => isset($node_type_settings['parsers'][$parser]['enabled']) ? $node_type_settings['parsers'][$parser]['enabled'] : FALSE,
        '#weight' => -15,
      );
      $form['feedapi']['parsers'][$parser]['weight'] = array(
        '#type' => 'weight',
        '#delta' => 15,
        '#title' => t('Weight'),
        '#description' => t('Control the execution order. Parsers with lower weights are called before parsers with higher weights.'),
        '#default_value' => isset($node_type_settings['parsers'][$parser]['weight']) ? $node_type_settings['parsers'][$parser]['weight'] : 0,
        '#weight' => -14,
      );
      if ($parser_form = module_invoke($parser, 'feedapi_settings_form', 'parsers')) {
        $form['feedapi']['parsers'][$parser]['defaults'] = array('#type' => 'markup', '#value' => '<strong>'. t('Default settings') .'</strong><hr/>');
        $form['feedapi']['parsers'][$parser] = array_merge_recursive($form['feedapi']['parsers'][$parser], $parser_form);
      }
    }
    $form['feedapi']['processors'] = array(
      '#type' => 'fieldset',
      '#title' => t('Processor settings'),
      '#description' => t('Processors are any kind of add on modules that hook into the feed handling process on download time - you can decide here what should happen to feed items once they are downloaded and parsed.'),
      '#collapsible' => FALSE,
      '#tree' => TRUE,
    );
    $processors = module_implements('feedapi_item', TRUE);
    rsort($processors);
    foreach ($processors as $processor) {
      $form['feedapi']['processors'][$processor] = array(
        '#type' => 'fieldset',
        '#title' => feedapi_get_natural_name($processor),
        '#collapsible' => TRUE,
        '#collapsed' => isset($node_type_settings['processors'][$processor]['enabled']) ? !($node_type_settings['processors'][$processor]['enabled']): TRUE,
        '#tree' => TRUE,
        '#weight' => isset($node_type_settings['processors'][$processor]['weight']) ? $node_type_settings['processors'][$processor]['weight'] : 0,
      );
      $form['feedapi']['processors'][$processor]['enabled'] = array(
        '#type' => 'checkbox',
        '#title' => t('Enable'),
        '#description' => t('Check this box if you want to enable the @name processor on this feed.', array('@name' => $processor)),
        '#default_value' => isset($node_type_settings['processors'][$processor]['enabled']) ? $node_type_settings['processors'][$processor]['enabled'] : FALSE,
        '#weight' => -15,
      );
      $form['feedapi']['processors'][$processor]['weight'] = array(
        '#type' => 'weight',
        '#delta' => 15,
        '#title' => t('Weight'),
        '#description' => t('Control the execution order. Processors with lower weights are called before processors with higher weights.'),
        '#default_value' => isset($node_type_settings['processors'][$processor]['weight']) ? $node_type_settings['processors'][$processor]['weight'] : 0,
        '#weight' => -14,
      );
      if ($processor_form = module_invoke($processor, 'feedapi_settings_form', 'processors')) {
        $form['feedapi']['processors'][$processor]['defaults'] = array('#type' => 'markup', '#value' => '<strong>'. t('Default settings') .'</strong><hr/>');
        $form['feedapi']['processors'][$processor] = array_merge_recursive($form['feedapi']['processors'][$processor], $processor_form);
      }
    }
    // Populate form with node type settings if available.
    if ($node_type_settings) {
      $form['feedapi'] = _feedapi_populate($form['feedapi'], $node_type_settings);
    }
    $form['#submit'][] = 'feedapi_content_type_submit';
  }
  elseif (isset($form['type']) && isset($form['#node']) && $form['type']['#value'] .'_node_form' == $form_id && feedapi_enabled_type($form['type']['#value'])) {
    // Get settings for corresponding content type
    // Which parsers / processors are enabled is a per content-type setting.
    $node_type_settings = feedapi_get_settings($form['type']['#value']);

    // FeedAPI-enabled node form.
    $form['title']['#required'] = FALSE;
    $form['title']['#description'] = t('This field will be populated with the feed title. You can override by filling in this field.');
    $form['body_field']['body']['#description'] = t('This field will be populated with the feed description. You can override by filling in this field.');
    $form['body_field']['body']['#rows'] = 2;

    // Don't blow away existing form elements.
    if (!isset($form['feedapi'])) {
      $form['feedapi'] = array();
    }
    $form['feedapi'] += array(
      '#type' => 'fieldset',
      '#title' => t('Feed'),
      '#collapsible' => TRUE,
      '#collapsed' => FALSE,
      '#tree' => TRUE,
    );
    $feedapi_url_default = '';
    if (isset($form['#node']->feed->url)) {
      $feedapi_url_default = $form['#node']->feed->url;
    }
    elseif (isset($form_state['values']['feedapi']['feedapi_url'])) {
      $feedapi_url_default = $form_state['values']['feedapi']['feedapi_url'];
    }
    if (isset($node_type_settings['upload_method']) && $node_type_settings['upload_method'] == 'upload') {
      // Makes possible to upload file via this form.
      $form['#attributes']['enctype'] = 'multipart/form-data';
      $form['feedapi']['feedapi_file'] = array(
        '#type' => 'file',
        '#title' => t('Upload a feed'),
        '#description' => $feedapi_url_default ? '<div class="feed-url">'. $feedapi_url_default .'</div>' : '',
        '#size' => 40,
      );
      $form['feedapi']['feedapi_url'] = array(
        '#type' => 'value',
        '#value' => $feedapi_url_default,
      );
    }
    else {
      $form['feedapi']['feedapi_url'] = array(
        '#type' => 'textfield',
        '#title' => t('Feed URL'),
        '#description' => t('Enter feed URL. The set of supported schemas (e.g. ftp://, http://) depends on the parser that you use.'),
        '#default_value' => $feedapi_url_default,
        '#maxlength' => 2048,
      );
    }
    // Show per-node-type feedapi, parser options only for users with permissions.
    if (user_access('advanced feedapi options')) {
      // retrieve forms.
      $modules = module_implements('feedapi_settings_form');
      foreach ($modules as $module) {
        if ($feedapi_form = module_invoke($module, 'feedapi_settings_form', 'general')) {
          $form['feedapi'] = array_merge_recursive($form['feedapi'], $feedapi_form);
        }
      }

      $submodules_names = array(
        'parsers' => t('Parsers'),
        'processors' => t('Processors'),
      );
      foreach (array("parsers" => "feedapi_feed", "processors" => "feedapi_item") as $type => $requirement) {
        $suitable_handlers = module_implements($requirement, TRUE);
        foreach ($suitable_handlers as $module) {
          if (isset($node_type_settings[$type][$module]) && $node_type_settings[$type][$module]['enabled']) {
            $result = array();
            $result = module_invoke($module, 'feedapi_settings_form', $type);
            if (is_array($result)) {
              $result['#weight'] = $node_type_settings[$type][$module]['weight'];
              $form['feedapi'][$type][$module] = $result;
              $form['feedapi'][$type][$module]['#type'] = 'fieldset';
              $form['feedapi'][$type][$module]['#title'] = feedapi_get_natural_name($module);
              $form['feedapi'][$type][$module]['#collapsible'] = TRUE;
              $form['feedapi'][$type][$module]['#collapsed'] = FALSE;
              $form['feedapi'][$type][$module]['#tree'] = TRUE;
            }
          }
        }
        if (isset($form['feedapi'][$type])) {
          $form['feedapi'][$type]['#type'] = 'fieldset';
          $form['feedapi'][$type]['#title'] = $submodules_names[$type];
          $form['feedapi'][$type]['#collapsible'] = TRUE;
          $form['feedapi'][$type]['#collapsed'] = TRUE;
          $form['feedapi'][$type]['#tree'] = TRUE;
        }
      }
    }
    // If we are on a node form, get per node settings and populate form.
    if (isset($form['#node']->nid)) {
      $settings = feedapi_get_settings($form['type']['#value'], $form['#node']->vid);
    }
    elseif (isset($node_type_settings)) {
      $settings = $node_type_settings;
    }
    if (isset($settings)) {
      $form['feedapi'] = _feedapi_populate($form['feedapi'], $settings);
    }
    $form['#validate'][] = 'feedapi_node_validate';
  }
}

/**
 * Build feed object on validate and submit.
 * See feedapi_form_alter on finding out how it is called (via FormAPI)
 */
function feedapi_node_validate($form, &$form_state) {
  // Don't validate when deleting.
  if ($form_state['values']['op'] == t('Delete')) {
    return TRUE;
  }
  // Upload file.
  $feed_dir = file_directory_path() .'/feeds';
  file_check_directory($feed_dir, TRUE);
  $file = file_save_upload('feedapi', array(), $feed_dir);
  $has_upload = is_object($file);

  // Validate and transform settings for submission.
  if (empty($form_state['values']['feedapi']['feedapi_url']) && !$has_upload) {
    form_set_error('source', t('The Feed URL or uploading a file is required.'));
  }
  else if (!empty($form_state['values']['feedapi']['feedapi_url']) && !$has_upload && (strpos($form_state['values']['feedapi']['feedapi_url'], 'file://') === 0) && !user_access('use local files as feeds')) {
    form_set_error('source', t('You do not have sufficient permissions to use local files as feeds.'));
  }
  else if (strpos($form_state['values']['feedapi']['feedapi_url'], 'file://') === 0 && !file_check_location(substr($form_state['values']['feedapi']['feedapi_url'], 7), file_directory_path())) {
    drupal_set_message(file_check_location(substr($form_state['values']['feedapi']['feedapi_url'], 7), file_directory_path()));
    form_set_error('source', t('file:// is only allowed for files under the files directory.'));
  }
  else {
    if ($has_upload) {
      $form_state['values']['feedapi']['feedapi_url'] = file_create_url($file->filepath);
    }
    $feed = _feedapi_build_feed_object($form_state['values']['type'], $form_state['values']['feedapi']['feedapi_url']);
    if (!isset($feed->title) && $has_upload) {
      $form_state['values']['feedapi']['feedapi_url'] = NULL;
    }
    // Stick feed object into feedapi form snippet - store it in submit.
    $form_state['values']['feedapi_object'] = $feed;
    if ($has_upload) {
      $form_state['values']['feedapi_object']->url = str_replace($GLOBALS['base_url'], '', $form_state['values']['feedapi_object']->url);
    }
    if (empty($form_state['values']['title']) && isset($feed->title)) {
      form_set_value($form['title'], $feed->title, $form_state);
    }
    if (isset($form['body_field']) && empty($form_state['values']['body']) && isset($feed->description)) {
      form_set_value($form['body_field']['body'], $feed->description, $form_state);
    }
    if (empty($form_state['values']['title'])) {
      if (!$has_upload) {
        form_set_error('title', t('Title could not be retrieved from feed.'));
      }
      else {
        form_set_error('title', t('Title could not be detected. Make sure that the uploaded file is a valid feed.'));
      }
    }
    elseif ($has_upload) {
      file_set_status($file, FILE_STATUS_PERMANENT);
    }
  }
}

/**
 * Store per-content-type settings
 */
function feedapi_content_type_submit($form, &$form_state) {
  // TODO: Drupal automatically stores mutilated 'feedapi_'. $form['#node_type']->type - remove.
  $type = !empty($form['#node_type']->type) ? $form['#node_type']->type : $form['#post']['type'];
  _feedapi_store_settings(array('node_type' => $type), $form_state['values']['feedapi']);
}

/**
 * Implementation of hook_feedapi_settings_form().
 */
function feedapi_feedapi_settings_form($type) {
  if ($type == 'general') {
    $form['refresh_on_create'] = array(
      '#type' => 'checkbox',
      '#title' => t('Refresh feed on creation'),
      '#description' => t('If checked, feed items will be processed immediately after a feed is created.'),
      '#default_value' => 0,
    );
    $form['update_existing'] = array(
      '#type' => 'checkbox',
      '#title' => t('Update existing feed items'),
      '#description' => t('If checked, existing feed items will be updated when feed is refreshed.'),
      '#default_value' => 1,
    );
    $period = array();
    $period[FEEDAPI_CRON_ALWAYS_REFRESH] = t('As often as possible');
    $period += drupal_map_assoc(array(900, 1800, 3600, 10800, 21600, 32400, 43200, 86400, 172800, 259200, 604800, 1209600, 2419200, 3628800, 4838400, 7257600, 15724800, 31536000), 'format_interval');
    $period[FEEDAPI_CRON_NEVER_REFRESH] = t('Never refresh');
    $form['refresh_time'] = array(
      '#type' => 'select',
      '#title' => t('Minimum refresh period'),
      '#description' => t('Select the minimum time that should elapse between two refreshes of the same feed. For news feeds, don\'t go under 30 minutes. Note that FeedAPI cannot guarantee that a feed will be refreshed at the rate of the selected time. The actual refresh rate depends on many factors such as number of feeds in system and your hardware.'),
      '#options' => $period,
      '#default_value' => FEEDAPI_CRON_DEFAULT_REFRESH_TIME,
    );
    $period = drupal_map_assoc(array(3600, 10800, 21600, 32400, 43200, 86400, 172800, 259200, 604800, 1209600, 2419200, 3628800, 4838400, 7257600, 15724800, 31536000), 'format_interval');
    $period[FEEDAPI_NEVER_DELETE_OLD] = t('Never delete');
    $form['items_delete'] = array(
      '#type' => 'select',
      '#title' => t('Delete news items older than'),
      '#options' => $period,
      '#default_value' => FEEDAPI_NEVER_DELETE_OLD,
    );
  }
  return $form;
}

/**
 * Implementation of hook_cron().
 */
function feedapi_cron() {
  global $user;

  // Saves the currently logged in user and start safe impersonating
  $original_user = $user;
  session_save_session(FALSE);

  db_query('DELETE FROM {feedapi_stat} WHERE timestamp < %d', variable_get('cron_semaphore', FALSE) - FEEDAPI_CRON_STAT_LIFETIME);

  // Initialize counters
  $count = array(
    '%feeds' => 0,
    '%expired' => 0,
    '%new' => 0,
    '%updated' => 0,
  );

  // We get feeds in small lots, this will save memory and have the process adjusting to the
  // time limit even when we have many thousands of them.
  $now = time();
  $process = 0;
  // The counter process will be > 0 if we've selected less feeds
  while (!$process && feedapi_cron_time()) {
    $process = FEEDAPI_CRON_FEEDS;
    $result = db_query_range("SELECT f.nid, n.uid FROM {feedapi} f JOIN {node} n ON n.vid = f.vid WHERE next_refresh_time <= %d AND next_refresh_time <> %d ORDER BY next_refresh_time ASC", $now, FEEDAPI_CRON_NEVER_REFRESH, 0, FEEDAPI_CRON_FEEDS);

    while (feedapi_cron_time() && $feed = db_fetch_object($result)) {
      $user = user_load(array('uid' => $feed->uid));
      // Call the refresh process for each feed and store counters
      $counter = feedapi_invoke('refresh', $feed, TRUE);
      if ($counter) {
        foreach ($counter as $name => $value) {
          $count['%'. $name] += $value;
        }
      }
      $count['%feeds']++;
      $process--;
    }
  }

  // Loads back the logged in user
  $user = $original_user;
  session_save_session(TRUE);
}

/**
 * Check for time limits in cron processing.
 *
 * @return
 *   Number of seconds left, zero if none.
 */
function feedapi_cron_time() {
  static $time_limit;

  if (!$time_limit) {
    $max_exec_time = ini_get('max_execution_time') == 0 ? 120 : ini_get('max_execution_time');
    $time_limit = time() + (variable_get('feedapi_cron_percentage', 15) / 100) * $max_exec_time;
    // However, check for left time, maybe some other cron processing already occured
    $cron_semaphore = variable_get('cron_semaphore', 0);
    if ($cron_semaphore) {
      $time_limit = min($time_limit, $cron_semaphore + $max_exec_time);
    }
    timer_start('feedapi_cron');
  }
  return max($time_limit - time(), 0);
}

/**
 * This is shown instead of normal node form when the simplified form is chosen at the settings
 */
function feedapi_simplified_form($form_state, $type) {
  $form['node']['#tree'] = TRUE;
  $form['node']['type'] = array(
    '#type' => 'hidden',
    '#value' => $type
  );
  $form['url'] = array(
    '#title' => t('Feed URL'),
    '#type' => 'textfield',
    '#size' => 25,
    '#required' => TRUE,
    '#maxlength' => 2048,
  );
  $form['add'] = array(
    '#type' => 'submit',
    '#value' => t('Add'),
  );
  return $form;
}

/**
 * Validates simplified form.
 */
function feedapi_simplified_form_validate($form, &$form_state) {
  if (!empty($form_state['values']['url']) && (strpos($form_state['values']['url'], 'file://') === 0) && !user_access('use local files as feeds')) {
    form_set_error('url', t('You do not have sufficient permissions to use local files as feeds.'));
  }
  else if (strpos($form_state['values']['url'], 'file://') === 0 && !file_check_location(substr($form_state['values']['feedapi']['feedapi_url'], 7), file_directory_path())) {
    form_set_error('url', t('file:// is only allowed for files under the files directory.'));
  }
}

/**
 * Create the node object and save
 */
function feedapi_simplified_form_submit($form, &$form_state) {
  $node_template = (object)$form_state['values']['node'];
  $feed_type = (string)$_POST['node']['type'];
  $valid_types = array_keys(feedapi_get_types());
  foreach ($valid_types as $type) {
    if ($type === $feed_type) {
      $node_template->type = $type;
    }
  }
  if ($node = feedapi_create_node($node_template, $form_state['values']['url'])) {
    drupal_set_message(t('Feed successfully created.'));
    $form_state['redirect'] = 'node/'. $node->nid;
  }
  else {
    drupal_set_message(t('Could not retrieve title from feed.'), 'error');
    $form_state['redirect'] = array('node/add/'. $node_template->type, 'feedapi_url='. urlencode($form_state['values']['url']));
  }
}

/**
 * Get the module-defined natural name of FeedAPI parser or processor
 * Define this name in hook_help():
 *
 *  function hook_help($section) {
 *    switch ($section) {
 *      case 'feedapi/full_name':
 *        return t('Natural name');
 *        break;
 *      }
 *    }
 */
function feedapi_get_natural_name($module) {
  $help = $module .'_help';
  $module_natural = function_exists($help) ? $help('feedapi/full_name', '') : $module;
  return empty($module_natural) ? $module : $module_natural;
}

/**
 * Create a feedapi node programatically.
 *
 * @param $param
 *  Either a feedapi - enabled node type or a $node object with at least valid $node->type.
 * @param $url
 *  URI of feed.
 */
function feedapi_create_node($param, $url) {
  if (is_object($param)) {
    $node = $param;
  }
  else {
    $node = new stdClass();
    $node->type = $param;
  }
  if (!feedapi_enabled_type($node->type)) {
    return FALSE;
  }
  $feed = _feedapi_build_feed_object($node->type, $url);
  if (!$feed->title && !$node->title) {
    return FALSE;
  }
  module_load_include('inc', 'node', 'node.pages');
  $node->title = $node->title ? $node->title : $feed->title;
  $node->body = $node->body ? $node->body : $feed->description;
  $node->feedapi_object = $feed;
  // Get the content-type settings as default
  $node->feedapi = feedapi_get_settings($node->type);
  node_object_prepare($node);
  global $user;
  $node->uid = $user->uid;
  node_save($node);
  return $node;
}

/**
 * Load node by URL.
 * @param $args
 *  Currently only supported $args['url] - URL string.
 * @return
 *  Node object if successful, FALSE if not.
 */
function feedapi_load_node($args) {
  if ($nid = db_result(db_query("SELECT nid FROM {feedapi} WHERE url = '%s'", $args['url']))) {
    return node_load($nid);
  }
  return FALSE;
}

/**
 * Refresh a feed node (= run enabled processors on it).
 * @param $node
 *  A node object with a $node->feed object.
 * @param $destination_path
 *  If a destination path is given, function redirects to this destination.
 */
function feedapi_refresh($node, $destination_path = NULL) {
  feedapi_invoke('refresh', $node->feed, FALSE);
  if ($destination_path) {
    drupal_goto($destination_path);
  }
  else {
    drupal_goto('node/'. $node->nid);
  }
}

/**
 * Insert feedapi data to the DB when it's a new for for FeedAPI
 */
function _feedapi_insert(&$node) {

  if (isset($node->feed->url) && isset($node->feed->feed_type)) {
    if (isset($node->feedapi)) {
      $values = $node->feedapi;
    }
    else {
      // On revert revision, settings are on $node->feed->settings
      // @todo: verify, settings shouldn't be NOT an array here anyway.
      $values = (array) $node->feed->settings;
    }
    db_query("INSERT INTO {feedapi} (
            nid, vid, url, link, feed_type, processors,
            parsers, next_refresh_time, settings) VALUES
            (%d, %d, '%s', '%s', '%s', '%s', '%s', %d, '%s')",
      $node->nid,
      $node->vid,
      $node->feed->url,
      isset($node->feed->options->link) ? $node->feed->options->link : '',
      $node->feed->feed_type,
      serialize($node->feed->processors),
      serialize($node->feed->parsers),
      $values['refresh_time'] == FEEDAPI_CRON_NEVER_REFRESH ? $values['refresh_time'] : time() + $values['refresh_time'],
      serialize(array())
    );
    // Store add on module's settings if user has permission to do so.
    if (user_access('advanced feedapi options')) {
      _feedapi_store_settings(array('vid' => $node->vid), $values);
    }
    // Refresh feed if the user would like to do that
    $settings = feedapi_get_settings($node->type, $node->vid);
    if (isset($settings['refresh_on_create'])) {
      if ($settings['refresh_on_create'] == TRUE) {
        $node->feed->nid = $node->nid;
        $node->feed->vid = $node->vid;
        $node->feed->settings = $settings;
        feedapi_invoke('refresh', $node->feed);
      }
    }
  }
}

/**
 * Update feed data of an existing feed
 */
function _feedapi_update(&$node) {
  if (isset($node->feed)) {
    $old_config = node_load($node->nid);

    // In that case this feed has never have feed data. Should be created then, this is not really an update
    if (!is_numeric($old_config->feed->nid)) {
      $url = isset($node->feed->url) ? $node->feed->url : $node->feedapi['feedapi_url'];
      $node->feed = _feedapi_build_feed_object($node->type, $url);
      _feedapi_insert($node);
      return;
    }
    $old_vid = db_result(db_query_range("SELECT vid FROM {feedapi} WHERE nid = %d ORDER BY vid DESC", $node->nid, 0, 1));
    if ($old_vid !== $node->vid) {
      _feedapi_insert($node);
      return;
    }
    // Only change next_refresh_time if refresh_time changed
    // or if $next_refresh_time is FEEDAPI_CRON_NEVER_REFRESH.
    $next_refresh_time = $old_config->feed->next_refresh_time;
    if (isset($node->feedapi['refresh_time'])) {
      if ($node->feedapi['refresh_time'] == FEEDAPI_CRON_NEVER_REFRESH) {
        $next_refresh_time = FEEDAPI_CRON_NEVER_REFRESH;
      }
      elseif ($old_config->feed->settings['refresh_time'] != $node->feedapi['refresh_time'] || $next_refresh_time == FEEDAPI_CRON_NEVER_REFRESH) {
        $next_refresh_time = time() + $node->feedapi['refresh_time'];
      }
    }

    db_query("UPDATE {feedapi} SET
            url = '%s',
            feed_type = '%s',
            processors = '%s',
            parsers = '%s',
            link = '%s',
            next_refresh_time = %d
            WHERE vid = %d",
      isset($node->feed->url) ? $node->feed->url : $node->feedapi['feedapi_url'],
      isset($node->feed->feed_type) ? $node->feed->feed_type : '',
      isset($node->feed->processors) ? serialize($node->feed->processors) : serialize($old_config->feed->processors),
      isset($node->feed->parsers) ? serialize($node->feed->parsers) : serialize($old_config->feed->parsers),
      isset($node->feed->link) ? $node->feed->link : '',
      $next_refresh_time,
      $node->vid
    );
    // Store add on module's settings if user has permission to do so.
    if (user_access('advanced feedapi options')) {
      _feedapi_store_settings(array('vid' => $node->vid), $node->feedapi);
    }
  }
}

/**
 * Execute the enabled parsers and create an unified output
 *
 * @param $feed
 *  Feed object
 * @param $parsers
 *  Structure: array(
 *    "primary" => "parser_primary",
 *    "secondary" => array("parser1", "parser2", "parserN")
 *  );
 * @return
 *  The object of the parser data
 */
function _feedapi_call_parsers($feed, $parsers, $settings) {
  $nid = isset($feed->nid) ? $feed->nid : '';
  $parser_primary = array_shift($parsers);
  $parsers_secondary = $parsers;
  // Normalize relative URLs according to the base URL (mostly for uploaded feeds), but allow other schemes too
  if (!valid_url($feed->url, TRUE) && valid_url($feed->url) && !strpos($feed->url, '://')) {
    $feed->url = $GLOBALS['base_url'] . $feed->url;
  }
  if (module_exists($parser_primary)) {
    $settings_primary = isset($settings[$parser_primary]) ? $settings[$parser_primary] : array();
    $feed->feed_type =  module_invoke($parser_primary, 'feedapi_feed', 'compatible', $feed, $settings_primary);
    $parser_output =  module_invoke($parser_primary, 'feedapi_feed', 'parse', $feed, $settings_primary);
    if ($parser_output === FALSE) {
      return $feed;
    }
    $feed = (object) array_merge((array) $feed, (array) $parser_output);
  }
  // Call the turned on parsers, create a union of returned options
  $parsers_secondary = is_array($parsers_secondary) ? $parsers_secondary : array();
  foreach ($parsers_secondary as $parser) {
    $settings_secondary = isset($settings[$parser]) ? $settings[$parser] : array();
    $feed_ext = module_invoke($parser, 'feedapi_feed', 'parse', $feed, $settings_secondary);
    $feed->options = (object) ((array) $feed->options + (array) $feed_ext->options);
    // Merge items' options
    if (is_array($feed_ext->items)) {
      foreach ($feed_ext->items as $key => $item) {
        $src = isset($feed->items[$key]) ? $feed->items[$key]->options : array();
        $feed->items[$key]->options = (object) ((array) $src + (array) $item->options);
      }
    }
  }
  $feed->nid = $nid;
  foreach (module_implements('feedapi_after_parse') as $module) {
    $func = $module .'_feedapi_after_parse';
    $func($feed);
  }
  // Filter bad or not allowed tags, sanitize data (currently timestamp checking)
  if (!variable_get('feedapi_allow_html_all', FALSE)) {
    $allowed = preg_split('/\s+|<|>/', variable_get('feedapi_allowed_html_tags', '<a> <b> <br> <dd> <dl> <dt> <em> <i> <li> <ol> <p> <strong> <u> <ul>'), -1, PREG_SPLIT_NO_EMPTY);
  }
  else {
    $allowed = TRUE;
  }
  foreach (array('title', 'description') as $property) {
    if (isset($feed->{$property})) {
      if (is_string($feed->{$property})) {
        $feed->{$property} = _feedapi_process_text($feed->{$property}, $allowed);
      }
    }
  }
  if (isset($feed->options)) {
    $props = array_keys(get_object_vars($feed->options));
    foreach ($props as $property) {
      if (isset($feed->options->{$property})) {
        if (is_string($feed->options->{$property})) {
          $feed->options->{$property} = _feedapi_process_text($feed->options->{$property}, $allowed);
        }
      }
    }
  }

  if (isset($feed->items)) {
    foreach (array_keys($feed->items) as $i) {
      $feed->items[$i]->title = _feedapi_process_text($feed->items[$i]->title, array());
      $feed->items[$i]->description = _feedapi_process_text($feed->items[$i]->description, $allowed);
      if ($feed->items[$i]->options->timestamp == 0) {
        $feed->items[$i]->options->timestamp = time();
      }
    }
  }

  return $feed;
}

/**
 * Filter texts from parsers
 *
 * @param $text
 *  The text to be processed
 * @param $allowed
 *  Allowed tags in that text
 * @return
 *  The safe string
 */
function _feedapi_process_text($text, $allowed) {
  if (is_array($allowed)) {
    $text = filter_xss($text, $allowed);
  }
  if (version_compare(PHP_VERSION, '5.0.0', '<')) {
    return trim(html_entity_decode($text, ENT_QUOTES));
  }
  else {
    return trim(html_entity_decode($text, ENT_QUOTES, 'UTF-8'));
  }
}

/**
 * Stores settings per content type or per node.
 *
 * @param $args
 *  Associative array which is $args['vid'] = N or $args['node_type'] = "content_type". Depends on what to store for
 * @param $settings
 *  The settings data itself
 */
function _feedapi_store_settings($args, $settings) {
  if (isset($args['vid'])) {
    db_query("UPDATE {feedapi} SET settings = '%s' WHERE vid = %d", serialize($settings), $args['vid']);
    module_invoke_all('feedapi_after_settings', $args['vid'], $settings);
    // This ensures that next time, not the cached, but the updated value will be used.
    feedapi_get_settings(NULL, $args['vid'], TRUE);
  }
  elseif (isset($args['node_type'])) {
    variable_set('feedapi_settings_'. $args['node_type'], $settings);
  }
}

/**
 * Determines wether feedapi is enabled for given node type.
 * If parser or processor is passed in, this function determines wether given
 * parser or processor is enabled for given node type.
 * @param $node_type
 *  A Drupal node type.
 * @param $parser_or_processor
 *  A parser or processor - pass in by module name.
 * @return TRUE if enabled, FALSE if not.
 */
function feedapi_enabled_type($node_type, $parser_or_processor = '') {
  $settings = feedapi_get_settings($node_type);
  if (empty($parser_or_processor)) {
    if (isset($settings['enabled'])) {
      return $settings['enabled'] ? TRUE : FALSE;
    }
    else {
      return FALSE;
    }
  }
  foreach (array('parsers', 'processors') as $stage) {
    if (isset($settings[$stage][$parser_or_processor]['enabled'])) {
      if ($settings[$stage][$parser_or_processor]['enabled'] == TRUE) {
        return TRUE;
      }
    }
  }
  return FALSE;
}

/**
 * Helper function for feedapi_invoke().
 *
 * Generic operations, collects results and returns array
 */
function _feedapi_invoke($op, &$feed, $param) {
  $output = array();
  foreach ($feed->processors as $processor) {
    $result = module_invoke($processor, 'feedapi_item', $op, $feed, $param);
    // Result may be a list of items or single values (count)
    if ($result) {
      if (is_array($result)) {
        $output = array_merge($output, $result);
      }
      else {
        $output[] = $result;
      }
    }
  }
  return $output;
}

/**
 * Helper function for feedapi_invoke().
 * Refresh the feed, call the proper parsers and processors' hooks.
 * Don't call this function directly, use feedapi_refresh() instead.
 *
 * @ TODO Fix: This may loop forever when a feed has no processors
 */
function _feedapi_invoke_refresh(&$feed, $param) {
  $timestamp = variable_get('cron_semaphore', FALSE) !== FALSE ? variable_get('cron_semaphore', FALSE) : time();

  $counter = array();
  timer_start('feedapi_'. $feed->nid);
  $memory_usage_before = function_exists('memory_get_usage') ? memory_get_usage() : 0;
  $cron = $param;

  // Step 0: Check processors and grab settings
  if (!is_array($feed->processors) || count($feed->processors) == 0) {
    if (!$cron) {
      drupal_set_message(t("No processors specified for URL %url. Could not refresh.", array('%url' => $feed->url)), "error");
      drupal_goto('node/'. $feed->nid);
    }
    return 0;
  }
  $settings = feedapi_get_settings(NULL, $feed->vid);

  // Step 1: Force processors to delete old items and determine the max. create elements.
  $counter['expired'] = feedapi_expire($feed, $settings);

  // Step 2: Get feed.
  $nid = $feed->nid;
  $hash_old = isset($feed->hash) ? $feed->hash : '';
  $feed = _feedapi_call_parsers($feed, $feed->parsers, $settings['parsers']);
  if (is_object($feed)) {
    $feed->hash = md5(serialize($feed->items));
  }

  // Step 3: See, whether feed has been modified.
  if (!isset($feed->items) || $hash_old == $feed->hash) {
    // Updated the next_refresh_time field in any case.
    db_query("UPDATE {feedapi} SET next_refresh_time = %d, half_done = %d WHERE nid = %d", time() + $settings['refresh_time'], FALSE, $nid);
    if (!$cron) {
      if (is_object($feed) && $hash_old == $feed->hash) {
        drupal_set_message(t('There are no new items in the feed.'), 'status');
      }
      else {
        drupal_set_message(t('Could not refresh feed.'), 'error');
      }
    }
    return $counter;
  }

  // Step 4: Walk through the items and check duplicates, then save or update
  $items = $feed->items;
  $updated = 0;
  $new = 0;
  $half_done = FALSE;

  // We check for time-out after each item
  foreach ($items as $index => $item) {
    // Call each item parser.
    $item->is_updated = FALSE;
    $item->is_new = FALSE;
    foreach ($feed->processors as $processor) {
      $unique = module_invoke($processor, 'feedapi_item', 'unique', $item, $feed->nid, $settings['processors'][$processor]);
      if ($unique === FALSE || is_numeric($unique)) {
        if ($settings['update_existing'] == TRUE) {
          module_invoke($processor, 'feedapi_item', 'update', $item, $feed->nid, $settings['processors'][$processor], $unique);
          $item->is_updated = TRUE;
        }
      }
      else {
        // We have checked before for expired items, so just save it.
        // if the item is already expired then do nothing
        $items_delete = $settings['items_delete'];
        $diff = abs(time() - (isset($item->options->timestamp) ? $item->options->timestamp : time()));
        if ($diff > $items_delete && ($items_delete > FEEDAPI_NEVER_DELETE_OLD)) {
          break;
        }
        $result = module_invoke($processor, 'feedapi_item', 'save', $item, $feed->nid, $settings['processors'][$processor]);
        if ($result !== FALSE) {
          $item->is_new = TRUE;
        }
      }
    }
    $new = $item->is_new ? $new + 1 : $new;
    $updated = ($item->is_updated && !$item->is_new) ? $updated + 1 : $updated;

    // Decision on time. If the exec time is greather than the user-set percentage of php max execution time
    if ($cron && !feedapi_cron_time()) {
      $half_done = ($new + $updated) == count($items) ? FALSE : TRUE;
      break;
    }
    // Save the item status for further processing
    $feed->items[$index] = $item;
  }

  // Closing step: Call after refresh and update feed statistics
  foreach (module_implements('feedapi_after_refresh') as $module) {
    $func = $module .'_feedapi_after_refresh';
    $func($feed);
  }

  // Set next_refresh_time to FEEDAPI_CRON_NEVER_REFRESH if refresh_time is FEEDAPI_CRON_NEVER_REFRESH.
  $next_refresh_time = $settings['refresh_time'] == FEEDAPI_CRON_NEVER_REFRESH ? $settings['refresh_time'] : (time() + $settings['refresh_time']);
  db_query("UPDATE {feedapi} SET next_refresh_time = %d, half_done = %d, hash = '%s' WHERE nid = %d", $next_refresh_time, $half_done, $feed->hash, $feed->nid);

  // Log statistics.
  $memory_usage_after = function_exists('memory_get_usage') ? memory_get_usage() : 0;
  _feedapi_store_stat($nid, 'update_times', time(), $timestamp);
  _feedapi_store_stat($nid, 'new', $new, $timestamp);
  _feedapi_store_stat($nid, 'download_num', count($items), $timestamp);
  _feedapi_store_stat($nid, 'process_time', timer_read('feedapi_'. $feed->nid), $timestamp);
  _feedapi_store_stat($nid, 'memory_increase', $memory_usage_after - $memory_usage_before, $timestamp);
  _feedapi_store_stat($nid, 'next_refresh_time', $next_refresh_time, $timestamp);

  if (!$cron) {
    if ($new == 0 && $updated == 0) {
      drupal_set_message(t('There are no new items in the feed.'), 'status');
    }
    else {
      drupal_set_message(t("%new new item(s) were saved. %updated existing item(s) were updated.", array("%new" => $new, "%updated" => $updated)));
    }
    // @ TODO what value to return here?
  }
  else {
    // Update and return counter
    $counter['new'] = $new;
    $counter['updated'] = $updated;
    return $counter;
  }
}

/**
 * Helper function for feedapi_invoke().
 * Delete all feed items of a feed.
 */
function _feedapi_invoke_purge(&$feed, $param) {
  $node = node_load($feed->nid);

  if ($param == 'items') {
    return drupal_get_form('feedapi_purge_confirm', $node);
  }

  // Delete items from the processors
  foreach ($feed->processors as $processor) {
    // FIXME: it's possible now to accidentally delete an item from another processor
    module_invoke($processor, 'feedapi_item', 'purge', $feed);
  }

  // Closing step: Call after purge hook
  foreach (module_implements('feedapi_after_purge') as $module) {
    $func = $module .'_feedapi_after_purge';
    $func($feed);
  }

  // Reset hash.
  db_query("UPDATE {feedapi} SET hash = 0 WHERE nid = %d", $feed->nid);
}

/**
 * Builds feed object ready to be sticked onto node.
 */
function _feedapi_build_feed_object($node_type, $url) {
  $feed = new stdClass();
  $feed->url = $url;
  $node_type_settings = feedapi_get_settings($node_type);
  $feed->processors = _feedapi_format_settings($node_type_settings, 'processors');
  $feed->parsers = _feedapi_format_settings($node_type_settings, 'parsers');
  if (isset($feed->url)) {
    $feed = _feedapi_call_parsers($feed, $feed->parsers, $node_type_settings['parsers']);
  }
  $feed->link = isset($feed->options->link) ? $feed->options->link : '';
  return $feed;
}

/**
 * Returns per content type settings ordered by weight
 * and only those that are turned on.
 * @param $node_type_settings
 *  Content type settings retrieved with feedapi_get_settings().
 * @param $stage_type
 *  'parsers' or 'processors'
 */
function _feedapi_format_settings($node_type_settings, $stage_type) {
  $result = array();
  $settings = $node_type_settings[$stage_type];
  if (!is_array($settings)) {
    return $result;
  }
  foreach ($settings as $name => $properties) {
    if (isset($properties['enabled'])) {
      if ($properties['enabled'] == TRUE) {
        $result[$properties['weight']] = $name;
      }
    }
  }
  ksort($result);
  return $result;
}

/**
 * Retrieve settings per content type or per node.
 *
 * @param $node_type
 *  Content type name or NULL if per node
 * @param $vid
 *  Node vid or NULL if per content type
 * @param $reset
 *  If TRUE, the data is returned from the database.
 * @return
 *  The associative array of feedapi settings
 *
 * @todo: Use node type settings for pulling on/off and weight of
 *       parsers/processors, use per node settings to override their
 *       configuration, this allows us a more predictable
 *       presets/settings behaviour. See d. o. #191692
 * Watch out: cache permutations of node_type or node_type+nid or nid.
 * Watch out: changes within page load likely.
 */
function feedapi_get_settings($node_type, $vid = FALSE, $reset = FALSE) {
  static $node_settings;

  if (is_numeric($vid)) {
    if (!isset($node_settings[$vid]) || $reset) {
      if ($settings = db_fetch_object(db_query('SELECT settings FROM {feedapi} WHERE vid = %d', $vid))) {
        $settings = unserialize($settings->settings);
        // If parsers don't have any settings, create an empty array
        if (!isset($settings['parsers'])) {
          $settings['parsers'] = array();
        }
        // If processors don't have any settings, create an empty array
        if (!isset($settings['processors'])) {
          $settings['processors'] = array();
        }
      }
      if (is_array($settings) && count($settings['processors']) == 0 && count($settings['parsers']) == 0) {
        $settings = NULL;
      }
      $node_settings[$vid] = !empty($settings) && is_array($settings) ? $settings : FALSE;
    }
    if (!is_array($node_settings[$vid])) {
      if (empty($node_type)) {
        // In normal case, this shouldn't happen. This is an emergency branch
        $node_type = db_result(db_query("SELECT type FROM {node} WHERE vid = %d", $vid));
      }
    }
    else {
      return $node_settings[$vid];
    }
  }

  // Fallback: node_type.
  if (isset($node_type) && is_string($node_type)) {
    if (($settings = variable_get('feedapi_settings_'. $node_type, FALSE)) && ($settings['enabled'] == 1)) {
      // Sanitize data right now, tricky users may turned off the module
      foreach (array('parsers', 'processors') as $type) {
        if (isset($settings[$type]) && is_array($settings[$type])) {
          $modules = array_keys($settings[$type]);
          foreach ($modules as $module) {
            if (!module_exists($module)) {
              unset($settings['parsers'][$module]);
            }
          }
        }
        else {
          // Missing parser or processor, set error message.
          if (user_access('administer content types')) {
            drupal_set_message(t('There are no !type defined for this content type. Go to !edit_page and enable at least one.', array('!type' => $type, '!edit_page' => l('admin/content/node-type/'. $node_type, 'admin/content/node-type/'. $node_type))), 'warning', FALSE);
          }
          else {
            drupal_set_message(t('There are no !type defined for this content type. Contact your site administrator.', array('!type' => $type)), 'warning', FALSE);
          }
        }
      }
      return $settings;
    }
  }
  return FALSE;
}

/**
 * Set default value of $form elements if present in $settings.
 */
function _feedapi_populate($form, $settings) {
  foreach ($form as $k => $v) {
    if (is_array($v)) {
      if (array_key_exists('#default_value', $v)) {
        // Don't prepopulate feedapi_url slot, not stored in settings
        // Might be overwritten otherwise by users without advanced feedapi options permissions.
        // Todo: stick all settings form elements that are not in 'parsers' or 'processors' in 'general' -
        // This is kind of tricky though without breaking sites out there.
        if ($k != 'feedapi_url') {
          if (isset($form[$k]['#parents']) && is_array($form[$k]['#parents'])) {
            // respect #parents if set
            $form[$k]['#default_value'] = _feedapi_populate_get_setting($form[$k]['#parents'], $settings);
          }
          elseif (isset($settings[$k])) {
            $form[$k]['#default_value'] = $settings[$k];
          }
        }
      }
      elseif (isset($settings[$k])) {
        $form[$k] = _feedapi_populate($form[$k], $settings[$k]);
      }
    }
  }
  return $form;
}

/**
 * Gets the setting for '#parent'
 * (there must be a more efficent way)
 */
function _feedapi_populate_get_setting($parents, $settings) {
  if (is_array($parents) && count($parents)) {
    $this_parent = array_shift($parents);
    return _feedapi_populate_get_setting($parents, $settings[$this_parent]);
  }
  else {
    return $settings[$parents];
  }
}

/**
 * Calculate the average between-update time
 */
function _feedapi_update_rate($update_times) {
  $between = array();
  for ($i = 0; $i < count($update_times) - 1; $i++) {
    $between[] = abs($update_times[$i] - $update_times[$i + 1]);
  }
  return (count($between) > 0) ? round(array_sum($between) / count($between), 2) : t('No data yet');
}

/**
 * Remove non-existing processors from the processors arrays
 */
function _feedapi_sanitize_processors(&$feed) {
  if (is_array($feed->processors)) {
    foreach ($feed->processors as $key => $processor) {
      if (!module_exists($processor)) {
        unset($feed->processors[$key]);
      }
    }
  }
}

/**
 * Store statistics information
 *
 * @param $id
 *  A numerical id
 * @param $type
 *  A string which describes what we want to store. This is an identifier, think of as a variable name
 * @param $val
 *  This is the variable value
 * @param $timestamp
 *  Timestamp for the value
 * @param $time
 *  Optional, a string equivalent to the $timestamp
 * @param $update
 *  Boolean, TRUE if you'd like to modify an existing entry in the stat table
 */
function _feedapi_store_stat($id, $type, $val, $timestamp, $time = NULL, $update = FALSE) {
  if (!$time) {
    $time = date("Y-m-d H:i", $timestamp);
  }
  if ($update) {
    db_query("UPDATE {feedapi_stat} SET value = %d, timestamp = %d WHERE time = '%s' AND type = '%s' AND id = %d", $val, $timestamp, $time, $type, $id);
  }
  if (!$update || !db_affected_rows()) {
    db_query("INSERT INTO {feedapi_stat} (id, value, time, timestamp, type) VALUES (%d, %d, '%s', %d, '%s')", $id, $val, $time, $timestamp, $type);
  }
}

/**
 * Return the type-specific statistics data
 *
 * @param $id
 *  A numerical id
 * @param $type
 *  Name of the type (variable)
 * @name $only_val
 *  If TRUE, only the values are returned, no more.
 * @return
 *  $only_val = FALSE -> array("timestamp" => array(), "time" => array(), "value" => array());
 */
function _feedapi_get_stat($id, $type, $only_val = FALSE) {
  $stat = array();
  $result = db_query("SELECT timestamp, time, value FROM {feedapi_stat} WHERE type = '%s' AND id = %d", $type, $id);
  while ($row = db_fetch_array($result)) {
    if ($only_val) {
      $stat[] = $row['value'];
    }
    else {
      foreach (array('timestamp', 'time', 'value') as $member) {
        $stat[$member][] = $row[$member];
      }
    }
  }
  return $stat;
}

/**
 * Return a list of FeedAPI-enabled content-types list, ready-to-use for #options at FormsAPI
 */
function feedapi_get_types() {
  $names = node_get_types('names');
  foreach ($names as $type => $name) {
    if (!feedapi_enabled_type($type)) {
      unset($names[$type]);
    }
  }
  return $names;
}

/**
 * Prevent users to use the same weight for two or more parsers and processors
 * because FeedAPI cannot handle this. And this is not neccessary too.
 */
function feedapi_content_type_validate($form, &$form_state) {
  if ($form_state['values']['feedapi']['enabled'] == FALSE) {
    return;
  }
  $parsers = module_implements('feedapi_feed', TRUE);
  rsort($parsers);
  $processors = module_implements('feedapi_item', TRUE);
  rsort($processors);
  $count_enabled_per_type = array();
  $count_enabled_per_type['parsers'] = 0;
  $count_enabled_per_type['processors'] = 0;
  foreach (array('processors', 'parsers') as $type) {
    $proc_weight = array();
    foreach (${$type} as $stuff) {
      if (isset($form_state['values']['feedapi'][$type][$stuff]) && $form_state['values']['feedapi'][$type][$stuff]['enabled'] == TRUE) {
        $count_enabled_per_type[$type]++;
        $weight = $form_state['values']['feedapi'][$type][$stuff]['weight'];
        if (!isset($proc_weight[$weight])) {
          $proc_weight[$weight] = 0;
        }
        if (++$proc_weight[$weight] > 1) {
          form_error($form, t('Two enabled processors or parsers cannot have the same weight.'), 'error');
        }
      }
    }
  }
  if ($count_enabled_per_type['parsers'] == 0) {
    form_error($form, t('Using FeedAPI for this content-type requires at least one enabled parser.'));
  }
  if ($count_enabled_per_type['processors'] == 0) {
    form_error($form, t('Using FeedAPI for this content-type requires at least one enabled processor.'));
  }
}