View Holder pattern not working

I have implemented a custom listview in which I have two textviews and checkbox. These are encapsulated into a linear layout on whoms click I check/uncheck the checkbox. The problem I am facing is that when I click the linear layout, it checks the checkbox but it also checks other checkboxes on a fixed pattern i.e every 7-8 rows. I read many problems regarding it and found out that implementing a view holder would solve the problem but that's also not working for me. Following is my code:

CustomListAdapter:

public class ContactsListAdapter extends BaseAdapter{

    private Context context;
    private List<ContactInfo> contacts;
    private List<ContactInfo> selectedContacts;

    public ContactsListAdapter(Context context, List<ContactInfo> contacts) {
        this.context = context;
        this.contacts = contacts;
        selectedContacts = new ArrayList<ContactInfo>();
    }

    public List<ContactInfo> getSelectedContacts() {
        return selectedContacts;
    }

    @Override
    public int getCount() {
        // TODO Auto-generated method stub
        return contacts.size();
    }

    @Override
    public Object getItem(int position) {
        // TODO Auto-generated method stub
        return contacts.get(position);
    }

    @Override
    public long getItemId(int position) {
        // TODO Auto-generated method stub
        return Long.parseLong(contacts.get(position).id);
    }

    @Override
    public View getView(final int position, View convertView, ViewGroup parent) {
        LayoutInflater inflater = (LayoutInflater) context.getSystemService(Context.LAYOUT_INFLATER_SERVICE);

        final ViewHolder holder;

        if(convertView == null) {
            convertView = inflater.inflate(R.layout.contacts_list_row_item, null);
            holder = new ViewHolder();
            holder.view = (LinearLayout) convertView.findViewById(R.id.llContactRow);
            holder.name = (TextView) convertView.findViewById(R.id.tvContactName);
            holder.number = (TextView) convertView.findViewById(R.id.tvContactNumber);
            holder.checkbox = (CheckBox) convertView.findViewById(R.id.cbMultipleSelect);
            convertView.setTag(holder);
        } else {
            holder = (ViewHolder) convertView.getTag();
        }

        holder.name.setText(contacts.get(position).name);
        holder.number.setText(contacts.get(position).number);
        holder.view.setOnClickListener(new OnClickListener() {

            @Override
            public void onClick(View v) {
                ContactInfo contact = new ContactInfo();
                contact.id = contacts.get(position).id;
                contact.name = contacts.get(position).name;
                contact.number = contacts.get(position).number;
                holder.checkbox.setChecked(!holder.checkbox.isChecked());
                if(holder.checkbox.isChecked()) {
                    selectedContacts.add(contact);
                } else {
                    selectedContacts.remove(contact);
                }
            }
        });

        return convertView;
    }

    private static class ViewHolder {
        LinearLayout view;
        CheckBox checkbox;
        TextView name;
        TextView number;
    }
}

What am I doing wrong? Please help.