sqlFiddle
SELECT category_id,picture_id,ranking FROM
(
select c.category_id,(select p.picture_id
from pictures p, picture_votes pv
where p.picture_id = pv.picture_id
and p.category_id = c.category_id
group by p.picture_id
order by sum(pv.vote) desc
limit 0,1)as picture_id,1 as ranking
from categories c
union
select c.category_id,(select p.picture_id
from pictures p, picture_votes pv
where p.picture_id = pv.picture_id
and p.category_id = c.category_id
group by p.picture_id
order by sum(pv.vote) desc
limit 1,1)as picture_id,2 as ranking
from categories c
union
select c.category_id,(select p.picture_id
from pictures p, picture_votes pv
where p.picture_id = pv.picture_id
and p.category_id = c.category_id
group by p.picture_id
order by sum(pv.vote) desc
limit 2,1)as picture_id,3 as ranking
from categories c
)result
WHERE picture_id is not null
order by category_id asc,ranking asc
or this sqlFiddle
SELECT picture_id,category_id,sumvotes,voteOrder
FROM
(SELECT picture_id,category_id,sumvotes,
IF(@prevCat <> category_id,@voteOrder:=1,@voteOrder:=@voteOrder+1)
as voteOrder,
@prevCat:=category_id
FROM(SELECT p.picture_id,
p.category_id,
SUM(pv.vote) as sumvotes
FROM pictures p
JOIN picture_votes pv
ON p.picture_id = pv.picture_id
GROUP BY p.picture_id,
p.category_id
ORDER BY p.category_id, sumvotes DESC
)as ppv,
(SELECT @prevCat:=0,@voteOrder:=0)pc
)finalTable
WHERE voteOrder BETWEEN 1 AND 3
ORDER BY category_id ASC, voteOrder ASC