answers = Answer.objects.filter(<something here>) .annotate(score=Coalesce(Sum('vote__type'), 0)) .order_by('-score')