Conditional removal of dropdown options based on checkbox selections

I've created a jsfiddle for this question, here: http://ift.tt/1lcYLN6

HTML

<span title="Fruit" class="ms-RadioText">
    <input id="groupType_0" type="checkbox" checked="checked"/>
    <label>Fruit</label>
</span><br>
<span title="Pies" class="ms-RadioText">
    <input id="groupType_1" type="checkbox" />  
    <label>Pies</label>
</span>
<br><br>
<select id="Use_d8ffe43">
   <option value="Apples">Apples</option>
   <option value="Grapes">Grapes</option>
   <option value="Both">Both</option>
</select><BR><BR>

JavaScript

$(document).ready(function () {
    var whatSelected = new Array();
    var piesOnly = false;
    whatSelected.push('fruit'); // initial preset

    var groupTypeField = $('input[id*="groupType"]');
    groupTypeField.on('change', function () {
        //alert(this.id);
        var thisId = this.id.split('groupType_')[1];
        //alert(thisId);
        var thisChecked = this.checked;
        var key = "";
        if (thisId == 0)
            key = "fruit";
        else
            key = "pies";
        //alert(key);

        if (whatSelected.indexOf(key) == -1 && thisChecked) {
            whatSelected.push(key);
            //alert('push: ' + key);
        }
        if (whatSelected.indexOf(key) != -1 && !thisChecked) {
            //alert('pull: ' + key);
            var index = whatSelected.indexOf(key);
            //alert('index: ' + index);
            if (index != -1) {
                whatSelected.splice(index, 1);
                //alert('pulled ' + key);
            }
        }
        alert('after alteration: ' + whatSelected);

        if (whatSelected.indexOf('pies') != -1) { // Pies checked   
            if (whatSelected.indexOf('fruit') != -1) { // Fruit checked, so not Pies only
                piesOnly = false;
                // Add "Grapes" and "Both" back, if gone
                var exists = false;
                $('select[id*="Use"] option').each(function () {
                    if (this.value == "Grapes") {
                        exists = true;
                    }
                });
                if (!exists) {
                    alert('added');
                    $('select[id*="Use"]').append('<option val="Grapes">Grapes</option>');
                    $('select[id*="Use"]').append('<option val="Both">Both</option>');
                }
            } else { // Fruit not checked
                piesOnly = true;
                // Remove Grapes and Both from dropdown
                $('select[id*="Use"]').find('option[value="Grapes"]').remove(); $('select[id*="Use"]').find('option[value="Both"]').remove();
                alert('removed');
            }
        } else { // Pies not checked
            if (whatSelected.indexOf('fruit') != -1) { // Fruit checked, so not pies only
                piesOnly = false;

            } else {// nothing selected, revert to Fruit
                alert('Must select an option.');
                $('span[title*="Fruit"]').children(0).prop('checked', true);
                $('span[title*="Pies"]').children(0).prop('checked', false);
                piesOnly = false;
                whatSelected.push('fruit'); // add it back
            }

            // Add "Grapes" and "Both" back to dropdown, if gone
            var exists = false;
            $('select[id*="Use"] option').each(function () {
                if (this.value == "Grapes") {
                    exists = true;
                }
            });
            if (!exists) {
                alert('added');
                $('select[id*="Use"]').append('<option value="Grapes">Grapes</option>');
                $('select[id*="Use"]').append('<option value="Both">Both</option>');
            }

        }
    });
});

Say I have 2 checkboxes, Fruit and Pies.
Say I have a dropdown, Apples, Grapes, Both

So if "Fruit" is selected, I expect all options to appear in the dropdown. If "Pies" is selected, I only want "Apples", since who ever heard of a "Grape" pie? (or pie with "Both"?)

Fruit is the default selection, and at least one checkbox must be selected at all times. I have an array, "whatSelected", that tracks what checkboxes have been selected.

My code works to remove "Grapes" and "Both" from the dropdown if I select "Pies" and unselect "Fruit". (leaves "Apples" in the dropdown)

If I then re-select "Fruit", the options are all added back, correctly, through code I use to re-append the options using jQuery.

My issue is then, if I unselect "Fruit", it then DOES NOT remove "Grapes" and "Both", which are my appended options, on the dropdown. How can I remove these appended options? Thanks.