Android checkboxes dont work when fragment first loads

I am implementing a layout that has checkboxes nested in a listview. When the fragment loads for the first time, when i click a checkbox the visual does not stay checked.

Here is the onCheckedChangedListener

public OnCheckedChangeListener checkboxContactsCallback = new OnCheckedChangeListener() {
    @Override
    public void onCheckedChanged(CompoundButton buttonView, boolean isChecked) {
        if(isChecked) {
            System.out.println("Checkbox checked");
            objects.getContactList().getContactList().get((Integer)buttonView.getTag(R.string.position)).setCheckbox(true);
            if(keyStoreContacts.getPublicKeyStore().size() == 0) {
                LinearLayout layout = (LinearLayout) rootView.findViewById(R.id.contactsWrapperLayout);
                layout.addView(helpers.getInterfaceHelper().createDynamicButton("Delete Contact", checkboxButtonListener));
            }
            keyStoreContacts.addPublicKey((String)buttonView.getTag(R.string.publickey));
        }
        else if(!isChecked) {
            System.out.println("Checkbox unchecked");
            objects.getContactList().getContactList().get((Integer)buttonView.getTag(R.string.position)).setCheckbox(false);
            if(keyStoreContacts.getPublicKeyStore().size() <= 1) {
                LinearLayout layout = (LinearLayout) rootView.findViewById(R.id.contactsWrapperLayout);
                LinearLayout layout2 = (LinearLayout)layout.findViewById(1);
                layout.removeView(layout2);
            }
            keyStoreContacts.removePublicKey((String)buttonView.getTag(R.string.publickey));
        }
    }
};

Here is the getView method in my array adapter

public View getView(int position, View convertView, ViewGroup parent) {
    LayoutInflater inflater = (LayoutInflater) act.getBaseContext().getSystemService(Context.LAYOUT_INFLATER_SERVICE);
    View rowView = inflater.inflate(R.layout.inflatable_contact_view, parent, false);
    TextView textView = (TextView) rowView.findViewById(R.id.inflate_textview);
    CheckBox checkbox = (CheckBox)rowView.findViewById(R.id.checkbox);
    System.out.println("Setting the checkbox " + objects.getContactList().getContactList().get(position).getIsChecked());
    checkbox.setChecked(objects.getContactList().getContactList().get(position).getIsChecked());
    checkbox.setOnCheckedChangeListener(callback.getCheckboxListener());
    checkbox.setTag(R.string.publickey, objects.getContactList().getContactList().get(position).getPublicKey());
    checkbox.setTag(R.string.position, position);
    textView.setText(objects.getContactList().getContactList().get(position).getDisplayName());
    PublicKeyStore keyStore = new PublicKeyStore();
    keyStore.addPublicKey(objects.getContactList().getContactList().get(position).getPublicKey());
    contact.setTag(keyStore);
    return rowView;
}

Again the issue is when the fragment loads for the first time, when you click a checkbox, it doesn't update the visual to be clicked. ONLY THE FIRST TIME IT LOADS. If you continue to use the app and go back to this fragment it works correctly.