Tek yazma sorununu yeni okudum şunu deneyin;

<?php

	include("ayar.php"); 

   mysql_select_db($dbname) or die("Could not select database"); 
    
   function altsayi($ustid){ 

    $rs3 = mysql_query("SELECT * FROM kat WHERE ustid = $ustid ");          
    $num_rows2 = mysql_num_rows($rs3); 
    if($num_rows2 == 0){ 
             $sonuc = 0; 
        }else{ 
            $sonuc = $num_rows2; 
        } 
		return $sonuc;      
   } 
    
   $rs = mysql_query("SELECT * FROM kategorilerim WHERE ustid = $ustid order by sira asc LIMIT $start,$end "); 
   $rs2 = mysql_query("SELECT * FROM kategorilerim WHERE ustid = $ustid "); 
    while($obj = mysql_fetch_array($rs, MYSQL_ASSOC)){ 
   $arr = array( 
   "id" => $obj['id'], 
   "basligim" => $obj['baslik'], 
   "icerik" => $obj['icerik'], 
   "alt" => altsayi($obj['id']) 
   ); 
   
   echo json_encode($arr); 
   $num_rows = mysql_num_rows($rs2); 
   } 
   echo "',\"total\" : '".$num_rows."' }';"; 
   
?>

yada istenen tam olarak bu değilse şuna da bi bakın;

<?php

	include("ayar.php"); 

   mysql_select_db($dbname) or die("Could not select database"); 
    
   function altsayi($ustid){ 

    $rs3 = mysql_query("SELECT * FROM kat WHERE ustid = $ustid ");          
    $num_rows2 = mysql_num_rows($rs3); 
    if($num_rows2 == 0){ 
             $sonuc = 0; 
        }else{ 
            $sonuc = $num_rows2; 
        } 
		return $sonuc;      
   } 
    
   $rs = mysql_query("SELECT * FROM kategorilerim WHERE ustid = $ustid order by sira asc LIMIT $start,$end "); 
   $rs2 = mysql_query("SELECT * FROM kategorilerim WHERE ustid = $ustid "); 
	echo '{"success":true,"sorgu":\'';
   while($obj = mysql_fetch_array($rs, MYSQL_ASSOC)){ 
   $arr = array( 
   "id" => $obj['id'], 
   "basligim" => $obj['baslik'], 
   "icerik" => $obj['icerik'], 
   "alt" => altsayi($obj['id']) 
   ); 
   $num_rows = mysql_num_rows($rs2); 
    echo '{"success":true,"sorgu":'.json_encode($arr).',"total":'.$num_rows.'}';
   } 
   
?>

Bu arada $start ve $end in değerleri gözükmüyor ? LIMIT ifadesinde de sorun olabilir.