Have combobox a required field only if checkbox is clicked

I Have a checkbox, which displays a hidden combobox when checked. When I submit the form and the checkbox is unpressed (combobox is still hidden) it still has the hidden combobox as a 'required' field. Here is my code:

$(function() {
  var checkbox = $("#hascustpo");
  var hidden = $("#custpo");
  hidden.hide();
  checkbox.change(function() {
    if (checkbox.is(':checked')) {
      checkbox.attr("required", true);
      hidden.show();
    } else {
      hidden.hide();
      checkbox.attr("required", false);
    }
  });
});

<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>
<label>
   <input type="checkbox" name="hascustpo" value="True" id="hascustpo"/>
   Has Customer PO
 </label><br>
<div id="custpo">
  <label>Customer PO Number <b style="color:red;">*</b></label>
  <select id="txtfield" style="width: 200px;" name="custpo" class="inputvalues" required>
    <option disabled selected>-- Customer PO Number</option>
    <option>AUTO PO NUM | DESCRIPTION</option>
  </select>
  <br>
</div>