Use unbuffered SQL queries with Magento to reduce memory usage on large data sets.

unbuffered_shell_example.php

<?php

require_once 'abstract.php';

class Cmtickle_Demo_Shell_Tool extends Mage_Shell_Abstract
{
    private $_readConnection = null;

    protected function _getReadConnection()
    {
        if(null === $this->_readConnection) {
            $this->_readConnection = Mage::getSingleton('core/resource')->getConnection('core_read');
            $this->_readConnection->getConnection()->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false);
        }

        return $this->_readConnection;
    }

    protected function _getStatement($query, $conn = null)
    {
        if ($query instanceof Zend_Db_Statement_Interface) {
            return $query;
        }

        if ($query instanceof Zend_Db_Select) {
            return $query->query();
        }

        if (is_string($query)) {
            if (!$conn instanceof Zend_Db_Adapter_Abstract) {
                Mage::throwException(Mage::helper('core')->__('Invalid connection'));
            }
            return $conn->query($query);
        }

        Mage::throwException(Mage::helper('core')->__('Invalid query'));
    }

    private function _doProcessing()
    {
        $modelClass = Mage::getModel('sales/order_address');
        $modelCollection = $modelClass->getCollection()->addAttributeToSelect('*');
        $identityField = $modelClass->getResource()->getIdFieldName();
        $query = $modelCollection->getSelect();
      
        Mage::getSingleton('core/resource_iterator')->walk(
            $query,
            array(array($this, 'collectionCallback')),
            array('id_field' => $identityField),
            $this->_getReadConnection()
        );

        // This needs to happen because we're using unbuffered queries.
        $this->_getStatement($query, $this->_getReadConnection())->closeCursor();
    }

    public function collectionCallback($args)
    {
        $rowId = $args['row'][$args['id_field']];
        $row = $args['row'];
      
        //
        // Do your required task with the data set here.
        //
    }

    public function run()
    {
        $this->_doProcessing();
    }
}

$tool = new Cmtickle_Demo_Shell_Tool();
$tool->run();

Line 13 and line 54 are the key parts to using unbuffered queries.

Caveat (Credit to Iain Hubbard for pointing this out) :
https://dev.mysql.com/doc/refman/5.7/en/mysql-use-result.html
"... you should not use [unbuffered query] for locking reads if you are doing a lot of processing for each row on the client side, or if the output is sent to a screen on which the user may type a ^S (stop scroll). This ties up the server and prevent other threads from updating any tables from which the data is being fetched."