Take a look @ Magento Maximum Allowed Order Amount, you would have to create a custom module to add this feature.
Create an observer for sales_quote_save_before
<config>
<frontend>
<events>
<sales_quote_save_before>
<observers>
<inchoo_maxorderamount_enforceSingleOrderLimit>
<class>inchoo_maxorderamount/observer</class>
<method>enforceSingleOrderLimit</method>
</inchoo_maxorderamount_enforceSingleOrderLimit>
</observers>
</sales_quote_save_before>
</events>
</frontend>
</config>
In your observer
class Inchoo_MaxOrderAmount_Model_Observer
{
private $_helper;
public function __construct()
{
$this->_helper = Mage::helper('inchoo_maxorderamount');
}
/**
* No single order can be placed over the amount of X
*/
public function enforceSingleOrderLimit($observer)
{
if (!$this->_helper->isModuleEnabled()) {
return;
}
$quote = $observer->getEvent()->getQuote();
if ($quote->getCart()->getItemsCount() == 1) {
Mage::getSingleton('checkout/session')->addError('limit only one product per order');
Mage::app()->getFrontController()->getResponse()->setRedirect(Mage::getUrl('checkout/cart'));
Mage::app()->getResponse()->sendResponse();
exit;
}
}
}