They were using transform method and setting the line item manually, that is what causing the error, i commented out that and code works. If you know the reason behind this then please share it. And if you want i can share the code as well.
var returnRec = nlapiTransformRecord('salesorder', salesOrderId, 'returnauthorization');
returnRec.setFieldValue('customform', '110');
var locationId = ProcessRTNLoc(returnRec.getFieldValue('location'));
returnRec.setFieldValue('location', locationId);
//returnRec.setFieldValue('custbody_ready_to_ship', "F");
returnRec.setFieldValue('custbody_tile_rtn_type_of_return', 2);
returnRec.setFieldValue('custbody1', refundRMACat);
returnRec.setFieldValue('custbody_dsg_rma_subcategory', refundRMAList);
returnRec.setFieldValue('custbody_dsg_rma_instructions', refundRMAInstr);
returnRec.setFieldValue('custbody_orig_so_no', salesOrderNumber);
returnRec.setFieldValue('custbody_tile_created_by_agent', email_cc_post);
/*
var count = returnRec.getLineItemCount('item');
var newItem = [];
var newQuantity = [];
var newRate = [];
for (var j = 0; j < count; j++) {
var oldItemType = returnRec.getLineItemValue('item', 'itemtype', j + 1);
oldItemType = MapItemId(oldItemType);
var oldItem = returnRec.getLineItemValue('item', 'item', j + 1);
newItem[j] = oldItem;
newQuantity[j] = returnRec.getLineItemValue('item', 'quantity', j + 1);
newRate[j] = returnRec.getLineItemValue('item', 'rate', j + 1);
}
for (var k = 0; k < newItem.length; k++) {
if (newItem[k]) {
returnRec.setLineItemValue('item', 'item', k + 1, newItem[k]);
returnRec.setLineItemValue('item', 'quantity', k + 1, newQuantity[k]);
returnRec.setLineItemValue('item', 'rate', k + 1, newRate[k] || 0);
} else {
var msg = "Return Item is defined for items on sales order!";
nlapiLogExecution('DEBUG', 'msg:', msg);
throw msg;
}
}
*/