object is not getting removed from list onCheckedChange listener of check box

I have a list of contacts. I have a check box to select the contacts. These selected contacts I want to add in another list called invitation array list. I have created a method to add all contacts in invitation list if check box is selected called as toogleContactsSelection. This I am using in an activity.

I have created another method to add contacts in invitation arraylist if check box is selected.

Now I want to remove the object from an invitation array list if check box is unchecked i.e onCheckChangeListener of check box.

But object is not getting removed from invitationArrayList.

Adapter:

 public class InviteAdapter extends RecyclerView.Adapter<InviteAdapter.MyViewHolder> {

    private ArrayList<Contact> contactArrayList;
    private Context mContext;
    public ArrayList<Invitation>  invitationArrayList = new ArrayList<>();

    public class MyViewHolder extends RecyclerView.ViewHolder {
        public TextView name;
        private CheckBox checkBox;

        public MyViewHolder(View view) {
            super(view);
            name = (TextView) view.findViewById(R.id.textContactName);
            checkBox = (CheckBox) view.findViewById(R.id.checkBox);

        }
    }

    public InviteAdapter(Context context, ArrayList<Contact> contactArrayList) {
        this.contactArrayList = contactArrayList;
        this.mContext = context;

    }

    @Override
    public MyViewHolder onCreateViewHolder(ViewGroup parent, int viewType) {
        View itemView = LayoutInflater.from(parent.getContext())
                .inflate(R.layout.invite_contact_item, parent, false);

        return new MyViewHolder(itemView);
    }

    @Override
    public void onBindViewHolder(MyViewHolder holder, final int position) {
        final Contact contact = contactArrayList.get(position);
        holder.name.setText(contact.getmFullName());

        holder.checkBox.setChecked(contact.getSelected());

        holder.checkBox.setOnCheckedChangeListener(new CompoundButton.OnCheckedChangeListener() {
            @Override
            public void onCheckedChanged(CompoundButton compoundButton, boolean b) {

                if(b)
                {
                    invite(contact);

                    Log.e("inviteList",String.valueOf(invitationArrayList.size()));
                }
                else {
                    invitationArrayList.remove(contact);

                    Log.e("inviteList",String.valueOf(invitationArrayList.size()));
                }
            }
        });
    }

    @Override
    public int getItemCount() {
        return contactArrayList.size();

    }

    public void toggleContactsSelection( boolean isSelected ) {
        for( Contact contact : contactArrayList ) {
            contact.setSelected(isSelected);

                invite(contact);

        }
        notifyDataSetChanged(); // OR you can use notifyItemRangeChanged - which ever suits your needs
    }

    public void invite(Contact contact)
    {

        SharedPreferences sharedpreferences = mContext.getSharedPreferences("UserId", Context.MODE_PRIVATE);

        String mUserId = sharedpreferences.getString("userId","");

        DateFormat df = new SimpleDateFormat("EEE, d MMM yyyy, HH:mm", Locale.ENGLISH);
        String date = df.format(Calendar.getInstance().getTime());

        Invitation invitation = new Invitation();

        invitation.setSender_id(mUserId);
        invitation.setDate(date);
        invitation.setInvitee_no(contact.getmMobileNo());
        invitation.setStatus("0");
        invitation.setUser_name(contact.getmUserName());

        invitationArrayList.add(invitation);

    }

    public void removeInvite(int position)
    {
        invitationArrayList.remove(position);
    }

    public ArrayList<Invitation> getArrayList(){
        return invitationArrayList;
    }

}

What is going wrong?

Please help. thank you..