New prepareIndexdata method for the CatalogSearch Helper in Magento (to integrate with Sphinx)

Helper_Data.php

<?php

// Helper/Data.php

...

public function prepareIndexdata($index, $separator = ' ', $entity_id = NULL)
{
	$_attributes = array();

	$_index = array();
	foreach ($index as $key => $value) {

		// As long as this isn't a standard attribute use it in our 
		// concatenated column.
		if ( ! in_array($key, array('sku', 'name', 'description', 'short_description', 'meta_keywords', 'meta_title')))
		{
			$_attributes[$key] = $value;
		}

		if (!is_array($value)) {
			$_index[] = $value;
		}
		else {
			$_index = array_merge($_index, $value);
		}
	}

	// Get the product name.
	if (is_array($index['name']))
	{
		$name = $index['name'][0]; // Use the configurable product's name
	}
	else
	{
		$name = $index['name']; // Use the simple product's name
	}

	// Combine the name with each non-standard attribute
	$name_attributes = array();
	foreach ($_attributes as $code => $value)
	{
		if ( ! is_array($value))
		{
			$value = array($value);
		}

		// Loop through each simple product's attribute values and assign to 
		// product name.
		foreach ($value as $key => $item_value)
		{
			if (isset($name_attributes[$key]))
			{
				$name_attributes[$key] .= ' '.$item_value;
			}
			else
			{
				// The first time we see this add the name to start.
				$name_attributes[] = $name.' '.$item_value;
			}
		}
	}

	$category = '';
	if ($entity_id)
	{
		$entity_id = (int) $entity_id;
		$read	  = Mage::getSingleton('core/resource')->getConnection('core/read');

		// Get categories
		$data = $read->fetchRow("
			SELECT value FROM `catalog_category_entity_varchar` `ccev`
				JOIN `catalog_category_entity` `cce` ON `cce`.`entity_id` = `ccev`.`entity_id`
				JOIN `catalog_category_product` `ccp` on `ccp`.`category_id` = `cce`.`entity_id`
			WHERE `ccp`.`product_id` = {$entity_id}
			ORDER BY `cce`.`level` DESC 
			LIMIT 1"
		);

		$category = $data['value'];
	}

	$data = array(
		'name'			=> $name,
		'name_attributes' => join('. ', $name_attributes),
		'data_index'	  => join($separator, $_index),
		'category'		=> $category,
	);

	return $data;
}

Model_Mysql4_Fulltext.php

<?php

	// Model/Mysql4/Fulltext.php
	
	// ...

	/**
	 * Prepare results for query
	 *
	 * @param Mage_CatalogSearch_Model_Fulltext $object
	 * @param string $queryText
	 * @param Mage_CatalogSearch_Model_Query $query
	 * @return Mage_CatalogSearch_Model_Mysql4_Fulltext
	 */
	public function prepareResult($object, $queryText, $query)
	{
		if ($query->getIsProcessed())
		{
			// return $this;
		}

		$searchType = $object->getSearchType($query->getStoreId());

		$stringHelper = Mage::helper('core/string');
		/* @var $stringHelper Mage_Core_Helper_String */

		$bind = array(
			':query'	 => $queryText
		);
		$like = array();

		$fulltextCond   = '';
		$likeCond       = '';
		$separateCond   = '';

		if ($searchType == Mage_CatalogSearch_Model_Fulltext::SEARCH_TYPE_LIKE || $searchType == Mage_CatalogSearch_Model_Fulltext::SEARCH_TYPE_COMBINE) {
			$words = $stringHelper->splitWords($queryText, true, $query->getMaxQueryWords());
			$likeI = 0;
			foreach ($words as $word) {
				$like[] = '`s`.`data_index` LIKE :likew' . $likeI;
				$bind[':likew' . $likeI] = '%' . $word . '%';
				$likeI ++;
			}
			if ($like) {
				$likeCond = '(' . join(' AND ', $like) . ')';
			}
		}
		if ($searchType == Mage_CatalogSearch_Model_Fulltext::SEARCH_TYPE_FULLTEXT
			|| $searchType == Mage_CatalogSearch_Model_Fulltext::SEARCH_TYPE_COMBINE) {
			$fulltextCond = 'MATCH (`s`.`data_index`) AGAINST (:query IN BOOLEAN MODE)';
		}
		if ($searchType == Mage_CatalogSearch_Model_Fulltext::SEARCH_TYPE_COMBINE && $likeCond) {
			$separateCond = ' OR ';
		}

		define('SPH_RANK_SPH04', 7);
		define('SPH_RANK_WORDCOUNT', 3);

		// Connect to our Sphinx Search Engine and run our queries
		$sphinx = new SphinxClient();
		$sphinx->SetServer('192.168.100.88', 9312);
		$sphinx->SetMatchMode(SPH_MATCH_EXTENDED);
		$sphinx->setFieldWeights(array(
			'name' => 7,
			'category' => 1,
			'name_attributes' => 3,
			'data_index' => 1
		));
		$sphinx->setLimits(0, 200, 1000, 5000);

		// $sphinx->SetRankingMode(SPH_RANK_SPH04, 7);
		$sphinx->SetRankingMode(SPH_RANK_PROXIMITY_BM25);
		$sphinx->AddQuery($queryText, "fulltext");
		$results = $sphinx->RunQueries();

		// Loop through our Sphinx results
		foreach ($results as $item)
		{
			if (empty($item['matches']))
				continue;

			foreach ($item['matches'] as $doc => $docinfo)
			{
				// Ensure we log query results into the Magento table.
				$sql = sprintf("INSERT INTO `{$this->getTable('catalogsearch/result')}` "
						. " (`query_id`, `product_id`, `relevance`) VALUES "
						. " (%d, %d, %f) "
						. " ON DUPLICATE KEY UPDATE `relevance` = %f",
					$query->getId(),
					$doc,
					$docinfo['weight']/1000,
					$docinfo['weight']/1000
				);
				$this->_getWriteAdapter()->query($sql, $bind);
			}
		}

		$query->setIsProcessed(1);
		return $this;
	}
	
	// ...

	/**
	 * Prepare Fulltext index value for product
	 *
	 * @param array $indexData
	 * @param array $productData
	 * @return string
	 */
	protected function _prepareProductIndex($indexData, $productData, $storeId)
	{
		$index = array();
		foreach ($this->_getSearchableAttributes('static') as $attribute) {
			if (isset($productData[$attribute->getAttributeCode()])) {
				if ($value = $this->_getAttributeValue($attribute->getId(), $productData[$attribute->getAttributeCode()], $storeId)) {

					//For grouped products
					if (isset($index[$attribute->getAttributeCode()])) {
						if (!is_array($index[$attribute->getAttributeCode()])) {
							$index[$attribute->getAttributeCode()] = array($index[$attribute->getAttributeCode()]);
						}
						$index[$attribute->getAttributeCode()][] = $value;
					}
					//For other types of products
					else {
						$index[$attribute->getAttributeCode()] = $value;
					}
				}
			}
		}
		foreach ($indexData as $attributeData) {
			foreach ($attributeData as $attributeId => $attributeValue) {
				if ($value = $this->_getAttributeValue($attributeId, $attributeValue, $storeId)) {
					$code = $this->_getSearchableAttribute($attributeId)->getAttributeCode();

					//For grouped products
					if (isset($index[$code])) {
						if (!is_array($index[$code])) {
							$index[$code] = array($index[$code]);
						}
						$index[$code][] = $value;
					}
					//For other types of products
					else {
						$index[$code] = $value;
					}
				}
			}
		}

		$product = $this->_getProductEmulator()
			->setId($productData['entity_id'])
			->setTypeId($productData['type_id'])
			->setStoreId($storeId);
		$typeInstance = $this->_getProductTypeInstance($productData['type_id']);
		if ($data = $typeInstance->getSearchableData($product)) {
			$index['options'] = $data;
		}

		if (isset($productData['in_stock'])) {
			$index['in_stock'] = $productData['in_stock'];
		}

		if ($this->_engine) {
			return $this->_engine->prepareEntityIndex($index, $this->_separator, $productData['entity_id']);
		}
		return Mage::helper('catalogsearch')->prepareIndexdata($index, $this->_separator, $productData['entity_id']);
	}

Model_Mysql4_Fulltext_Engine.php

<?php

	// Model_Mysql4_Fulltext_Engine.php

	// ...

	/**
	 * Multi add entities data to fulltext search table
	 *
	 * @param int $storeId
	 * @param array $entityIndexes
	 * @param string $entity 'product'|'cms'
	 * @return Mage_CatalogSearch_Model_Mysql4_Fulltext_Engine
	 */
	public function saveEntityIndexes($storeId, $entityIndexes, $entity = 'product')
	{
		$adapter = $this->_getWriteAdapter();
		$data	= array();
		$storeId = (int)$storeId;
		foreach ($entityIndexes as $entityId => &$index) {
			$data[] = array(
				'product_id'      => (int)$entityId,
				'store_id'        => $storeId,
				'data_index'      => $index['data_index'],
				'name'            => $index['name'],
				'name_attributes' => $index['name_attributes'],
				'category'        => $index['category'],
			);
		}

		if ($data) {
			$adapter->insertOnDuplicate('sphinx_catalogsearch_fulltext', $data, array('data_index', 'name', 'name_attributes', 'category'));
		}

		return $this;
	}

	// ...
	
	/**
	 * Prepare index array as a string glued by separator
	 *
	 * @param array $index
	 * @param string $separator
	 * @return string
	 */
	public function prepareEntityIndex($index, $separator = ' ', $entity_id = NULL)
	{
		return Mage::helper('catalogsearch')->prepareIndexdata($index, $separator, $entity_id);
	}
	

Since using this code, I am getting much more relevant search results in magento. The only problem I'm having now is that if you search by a specific SKU, that product is not returned first. Do you have any suggestions on how to put exact search terms as a priority? Thanks SR