How to persist prev.data in localStorage when adding new data?

I added "select all" checkbox to my table, and I'm saving the data in localStorage to use it in other components. I'm able to save data and get it from localstorage when box is checked, also if user unchecks box, data gets removed from localstorage. The problem is that after I uncheck "select all" checkbox I need to remove the data that was stored in localStorage when user clicked "select all" checkbox but all data from local storage gets deleted.

import React, { useState, useEffect } from "react";

const peopleInfo = [
  {
    id: "1",
    name: "Jane",
    lastName: "Doe",
    age: "25"
  },
  {
    id: "2",
    name: "James",
    lastName: "Doe",
    age: "40"
  },
  {
    id: "3",
    name: "Alexa",
    lastName: "Doe",
    age: "27"
  },
  {
    id: "4",
    name: "Jane",
    lastName: "Brown",
    age: "40"
  }
];

export default function App() {
  const [peopleInfoValue, setPeopleInfoValue] = useState([]);

  useEffect(() => {
    const sp = localStorage.getItem("selectedPeople");
    if (sp) setPeopleInfoValue(JSON.parse(sp));
  }, []);

  useEffect(() => {
    localStorage.setItem("selectedPeople", JSON.stringify(peopleInfoValue));
  }, [peopleInfoValue]);

  const handleSelect = (e, id) => {
    let checked = e.target.checked;

    if (checked) {
      setPeopleInfoValue(
        peopleInfo?.map((d) => {
          return {
            select: true,
            id: d.id,
            first: d.first,
            last: d.last,
            age: d.age
          };
        })
      );
    } else {
      setPeopleInfoValue([]);
    }
  };
  function toggle(e, id) {
    let checked = e.target.checked;
    if (checked) {
      setPeopleInfoValue([
        ...peopleInfoValue,
        {
          ...peopleInfo.find((pi) => pi.id === id),
          select: true
        }
      ]);
    } else {
      // to remove from localstorage
      setPeopleInfoValue(peopleInfoValue.filter((people) => people.id !== id));
    }
  }

  return (
    <div className="App">
      <table>
        <thead>
          <input
            style=
            onChange={(e) => handleSelect(e)}
            checked={peopleInfoValue.length == `${peopleInfo?.length}`}
            type="checkbox"
          />
          <th>Name</th>
          <th>Last Name</th>
          <th>Age</th>
        </thead>
        <tbody>
          {peopleInfo?.map((d) => {
            const match = peopleInfoValue.find((piv) => piv.id === d.id);
            return (
              <tr key={d.id}>
                <td>
                  <input
                    style=
                    onChange={(e) => toggle(e, d.id)}
                    checked={!!match?.select}
                    type="checkbox"
                  />
                </td>
                <td style=>{d.name}</td>
                <td style=>{d.lastName}</td>
                <td style=>{d.age}</td>
              </tr>
            );
          })}
        </tbody>
      </table>
    </div>
  );
}

and codeSandbox

In my code I use setPeopleInfoValue([]) to delete data from localstorage when user clicks uncheck "select all" checkbox, and because of that all previously saved data in localstorage gets removed too. I can't figure out how to save/remove data received only from "select all" checkbox in localstorage and still keep previously saved data, for ex.if user selected 1st row and went to another page, and that 1st row will be displayed there(from localStorage), but if user comes back, checks and then unchecks "select all" box, previously saved 1st row will be removed also. How can I prevent that? I also need to keep displaying checked if boxes were checked on page refresh. Any suggestions and help are greatly appreciated.