diff --git a/package.json b/package.json
index b038cdb..11cc3bb 100644
--- a/package.json
+++ b/package.json
@@ -18,6 +18,7 @@
     "dexie": "^3.0.3",
     "err-code": "^2.0.3",
     "fake-indexeddb": "^3.1.2",
+    "fast-deep-equal": "^3.1.3",
     "fuse.js": "^6.4.1",
     "interactjs": "^1.9.7",
     "konva": "^7.1.8",
diff --git a/src/components/map/MapInteraction.js b/src/components/map/MapInteraction.js
index 1e6b749..a55e3c1 100644
--- a/src/components/map/MapInteraction.js
+++ b/src/components/map/MapInteraction.js
@@ -18,6 +18,8 @@ import MapStageContext, {
 import AuthContext from "../../contexts/AuthContext";
 import SettingsContext from "../../contexts/SettingsContext";
 import KeyboardContext from "../../contexts/KeyboardContext";
+import { PlayerUpdaterContext } from "../../contexts/PlayerContext";
+import PartyContext from "../../contexts/PartyContext";
 
 function MapInteraction({
   map,
@@ -35,6 +37,7 @@ function MapInteraction({
   useEffect(() => {
     if (
       !map ||
+      !mapState ||
       (map.type === "file" && !map.file && !map.resolutions) ||
       mapState.mapId !== map.id
     ) {
@@ -177,6 +180,8 @@ function MapInteraction({
 
   const auth = useContext(AuthContext);
   const settings = useContext(SettingsContext);
+  const player = useContext(PlayerUpdaterContext);
+  const party = useContext(PartyContext);
 
   const mapInteraction = {
     stageScale,
@@ -218,15 +223,19 @@ function MapInteraction({
             />
             {/* Forward auth context to konva elements */}
             <AuthContext.Provider value={auth}>
-              <SettingsContext.Provider value={settings}>
-                <KeyboardContext.Provider value={keyboardValue}>
-                  <MapInteractionProvider value={mapInteraction}>
-                    <MapStageProvider value={mapStageRef}>
-                      {mapLoaded && children}
-                    </MapStageProvider>
-                  </MapInteractionProvider>
-                </KeyboardContext.Provider>
-              </SettingsContext.Provider>
+              <PlayerUpdaterContext.Provider value={player}>
+                <PartyContext.Provider value={party}>
+                  <SettingsContext.Provider value={settings}>
+                    <KeyboardContext.Provider value={keyboardValue}>
+                      <MapInteractionProvider value={mapInteraction}>
+                        <MapStageProvider value={mapStageRef}>
+                          {mapLoaded && children}
+                        </MapStageProvider>
+                      </MapInteractionProvider>
+                    </KeyboardContext.Provider>
+                  </SettingsContext.Provider>
+                </PartyContext.Provider>
+              </PlayerUpdaterContext.Provider>
             </AuthContext.Provider>
           </Layer>
         </Stage>
diff --git a/src/components/party/Party.js b/src/components/party/Party.js
index 2acb24b..45eee01 100644
--- a/src/components/party/Party.js
+++ b/src/components/party/Party.js
@@ -14,10 +14,16 @@ import DiceTrayButton from "./DiceTrayButton";
 
 import useSetting from "../../helpers/useSetting";
 
-import PlayerContext from "../../contexts/PlayerContext";
+import PartyContext from "../../contexts/PartyContext";
+import {
+  PlayerUpdaterContext,
+  PlayerStateWithoutPointerContext,
+} from "../../contexts/PlayerContext";
 
 function Party({ gameId, stream, partyStreams, onStreamStart, onStreamEnd }) {
-  const { playerState, setPlayerState, partyState } = useContext(PlayerContext);
+  const setPlayerState = useContext(PlayerUpdaterContext);
+  const playerState = useContext(PlayerStateWithoutPointerContext);
+  const partyState = useContext(PartyContext);
 
   const [fullScreen] = useSetting("map.fullScreen");
   const [shareDice, setShareDice] = useSetting("dice.shareDice");
diff --git a/src/contexts/PartyContext.js b/src/contexts/PartyContext.js
new file mode 100644
index 0000000..37c3e32
--- /dev/null
+++ b/src/contexts/PartyContext.js
@@ -0,0 +1,34 @@
+import React, { useState, useEffect } from "react";
+
+const PartyContext = React.createContext();
+
+export function PartyProvider({ session, children }) {
+  const [partyState, setPartyState] = useState({});
+
+  useEffect(() => {
+    function handleSocketPartyState(partyState) {
+      if (partyState) {
+        const { [session.id]: _, ...otherMembersState } = partyState;
+        setPartyState(otherMembersState);
+      } else {
+        setPartyState({});
+      }
+    }
+
+    if (session.socket) {
+      session.socket.on("party_state", handleSocketPartyState);
+    }
+
+    return () => {
+      if (session.socket) {
+        session.socket.off("party_state", handleSocketPartyState);
+      }
+    };
+  });
+
+  return (
+    <PartyContext.Provider value={partyState}>{children}</PartyContext.Provider>
+  );
+}
+
+export default PartyContext;
diff --git a/src/contexts/PlayerContext.js b/src/contexts/PlayerContext.js
index e70a163..4a756da 100644
--- a/src/contexts/PlayerContext.js
+++ b/src/contexts/PlayerContext.js
@@ -1,4 +1,5 @@
-import React, { useState, useEffect, useContext } from "react";
+import React, { useEffect, useContext, useState } from "react";
+import compare from "fast-deep-equal";
 
 import useNetworkedState from "../helpers/useNetworkedState";
 import DatabaseContext from "./DatabaseContext";
@@ -6,7 +7,12 @@ import AuthContext from "./AuthContext";
 
 import { getRandomMonster } from "../helpers/monsters";
 
-const PlayerContext = React.createContext();
+export const PlayerStateContext = React.createContext();
+export const PlayerUpdaterContext = React.createContext(() => {});
+/**
+ * Store the player state without the pointer data to prevent unnecessary updates
+ */
+export const PlayerStateWithoutPointerContext = React.createContext();
 
 export function PlayerProvider({ session, children }) {
   const { userId } = useContext(AuthContext);
@@ -17,14 +23,13 @@ export function PlayerProvider({ session, children }) {
       nickname: "",
       timer: null,
       dice: { share: false, rolls: [] },
-      pointer: {},
+      pointer: { position: { x: 0, y: 0 }, visible: false },
       sessionId: null,
       userId,
     },
     session,
     "player_state"
   );
-  const [partyState, setPartyState] = useState({});
 
   useEffect(() => {
     if (!database || databaseStatus === "loading") {
@@ -45,7 +50,7 @@ export function PlayerProvider({ session, children }) {
     }
 
     loadNickname();
-  }, [database, databaseStatus]);
+  }, [database, databaseStatus, setPlayerState]);
 
   useEffect(() => {
     if (
@@ -64,7 +69,7 @@ export function PlayerProvider({ session, children }) {
       ...prevState,
       userId,
     }));
-  }, [userId]);
+  }, [userId, setPlayerState]);
 
   useEffect(() => {
     function handleSocketConnect() {
@@ -72,21 +77,11 @@ export function PlayerProvider({ session, children }) {
       setPlayerState({ ...playerState, sessionId: session.id });
     }
 
-    function handleSocketPartyState(partyState) {
-      if (partyState) {
-        const { [session.id]: _, ...otherMembersState } = partyState;
-        setPartyState(otherMembersState);
-      } else {
-        setPartyState({});
-      }
-    }
-
     session.on("connected", handleSocketConnect);
 
     if (session.socket) {
       session.socket.on("connect", handleSocketConnect);
       session.socket.on("reconnect", handleSocketConnect);
-      session.socket.on("party_state", handleSocketPartyState);
     }
 
     return () => {
@@ -95,19 +90,32 @@ export function PlayerProvider({ session, children }) {
       if (session.socket) {
         session.socket.off("connect", handleSocketConnect);
         session.socket.off("reconnect", handleSocketConnect);
-        session.socket.off("party_state", handleSocketPartyState);
       }
     };
   });
 
-  const value = {
-    playerState,
-    setPlayerState,
-    partyState,
-  };
+  const [playerStateWithoutPointer, setPlayerStateWithoutPointer] = useState(
+    playerState
+  );
+  useEffect(() => {
+    const { pointer, ...state } = playerState;
+    if (
+      !playerStateWithoutPointer ||
+      !compare(playerStateWithoutPointer, state)
+    ) {
+      setPlayerStateWithoutPointer(state);
+    }
+  }, [playerState, playerStateWithoutPointer]);
+
   return (
-    <PlayerContext.Provider value={value}>{children}</PlayerContext.Provider>
+    <PlayerStateContext.Provider value={playerState}>
+      <PlayerUpdaterContext.Provider value={setPlayerState}>
+        <PlayerStateWithoutPointerContext.Provider
+          value={playerStateWithoutPointer}
+        >
+          {children}
+        </PlayerStateWithoutPointerContext.Provider>
+      </PlayerUpdaterContext.Provider>
+    </PlayerStateContext.Provider>
   );
 }
-
-export default PlayerContext;
diff --git a/src/helpers/useNetworkedState.js b/src/helpers/useNetworkedState.js
index b421a5e..25741ce 100644
--- a/src/helpers/useNetworkedState.js
+++ b/src/helpers/useNetworkedState.js
@@ -1,4 +1,4 @@
-import { useEffect, useState } from "react";
+import { useEffect, useState, useCallback } from "react";
 
 function useNetworkedState(defaultState, session, eventName) {
   const [state, _setState] = useState(defaultState);
@@ -6,10 +6,10 @@ function useNetworkedState(defaultState, session, eventName) {
   const [dirty, setDirty] = useState(false);
 
   // Update dirty at the same time as state
-  function setState(update, sync = true) {
+  const setState = useCallback((update, sync = true) => {
     _setState(update);
     setDirty(sync);
-  }
+  }, []);
 
   useEffect(() => {
     if (session.socket && dirty) {
@@ -31,7 +31,7 @@ function useNetworkedState(defaultState, session, eventName) {
         session.socket.off(eventName, handleSocketEvent);
       }
     };
-  }, [session.socket]);
+  }, [session.socket, eventName]);
 
   return [state, setState];
 }
diff --git a/src/network/NetworkedMapAndTokens.js b/src/network/NetworkedMapAndTokens.js
index db521aa..b5c61c1 100644
--- a/src/network/NetworkedMapAndTokens.js
+++ b/src/network/NetworkedMapAndTokens.js
@@ -5,7 +5,7 @@ import MapDataContext from "../contexts/MapDataContext";
 import MapLoadingContext from "../contexts/MapLoadingContext";
 import AuthContext from "../contexts/AuthContext";
 import DatabaseContext from "../contexts/DatabaseContext";
-import PlayerContext from "../contexts/PlayerContext";
+import PartyContext from "../contexts/PartyContext";
 
 import { omit } from "../helpers/shared";
 import useDebounce from "../helpers/useDebounce";
@@ -27,7 +27,7 @@ import Tokens from "../components/token/Tokens";
  */
 function NetworkedMapAndTokens({ session }) {
   const { userId } = useContext(AuthContext);
-  const { partyState } = useContext(PlayerContext);
+  const partyState = useContext(PartyContext);
   const {
     assetLoadStart,
     assetLoadFinish,
diff --git a/src/network/NetworkedMapPointer.js b/src/network/NetworkedMapPointer.js
index 4347f19..afa4374 100644
--- a/src/network/NetworkedMapPointer.js
+++ b/src/network/NetworkedMapPointer.js
@@ -2,29 +2,31 @@ import React, { useState, useContext, useEffect, useRef } from "react";
 import { Group } from "react-konva";
 
 import AuthContext from "../contexts/AuthContext";
+import PartyContext from "../contexts/PartyContext";
+import { PlayerUpdaterContext } from "../contexts/PlayerContext";
 
 import MapPointer from "../components/map/MapPointer";
 import { isEmpty } from "../helpers/shared";
-import { lerp } from "../helpers/vector2";
+import { lerp, compare } from "../helpers/vector2";
 
 // Send pointer updates every 33ms
-const sendTickRate = 33;
+const sendTickRate = 100;
 
-function NetworkedMapPointer({ session, active, gridSize }) {
+let t = 0;
+
+function NetworkedMapPointer({ active, gridSize }) {
   const { userId } = useContext(AuthContext);
-  const [pointerState, setPointerState] = useState({});
+  const setPlayerState = useContext(PlayerUpdaterContext);
+  const partyState = useContext(PartyContext);
+  const [localPointerState, setLocalPointerState] = useState({});
+
   useEffect(() => {
-    if (userId && !(userId in pointerState)) {
-      setPointerState({
+    if (userId && !(userId in localPointerState)) {
+      setLocalPointerState({
         [userId]: { position: { x: 0, y: 0 }, visible: false, id: userId },
       });
     }
-  }, [userId, pointerState]);
-
-  const sessionRef = useRef(session);
-  useEffect(() => {
-    sessionRef.current = session;
-  }, [session]);
+  }, [userId, localPointerState]);
 
   // Send pointer updates every sendTickRate to peers to save on bandwidth
   // We use requestAnimationFrame as setInterval was being blocked during
@@ -42,8 +44,14 @@ function NetworkedMapPointer({ session, active, gridSize }) {
 
       if (counter > sendTickRate) {
         counter -= sendTickRate;
-        if (ownPointerUpdateRef.current && sessionRef.current) {
-          sessionRef.current.send("pointer", ownPointerUpdateRef.current);
+        if (ownPointerUpdateRef.current) {
+          const { position, visible } = ownPointerUpdateRef.current;
+          console.log("send time", performance.now() - t);
+          t = performance.now();
+          setPlayerState((prev) => ({
+            ...prev,
+            pointer: { position, visible },
+          }));
           ownPointerUpdateRef.current = null;
         }
       }
@@ -55,7 +63,7 @@ function NetworkedMapPointer({ session, active, gridSize }) {
   }, []);
 
   function updateOwnPointerState(position, visible) {
-    setPointerState((prev) => ({
+    setLocalPointerState((prev) => ({
       ...prev,
       [userId]: { position, visible, id: userId },
     }));
@@ -75,39 +83,43 @@ function NetworkedMapPointer({ session, active, gridSize }) {
   }
 
   // Handle pointer data receive
-  const syncedPointerStateRef = useRef({});
+  const interpolationsRef = useRef({});
   useEffect(() => {
-    function handlePeerData({ id, data }) {
-      if (id === "pointer") {
-        // Setup an interpolation to the current pointer data when receiving a pointer event
-        if (syncedPointerStateRef.current[data.id]) {
-          const from = syncedPointerStateRef.current[data.id].to;
-          syncedPointerStateRef.current[data.id] = {
-            id: data.id,
-            from: {
-              ...from,
-              time: performance.now(),
-            },
-            to: {
-              ...data,
-              time: performance.now() + sendTickRate,
-            },
-          };
-        } else {
-          syncedPointerStateRef.current[data.id] = {
-            from: null,
-            to: { ...data, time: performance.now() + sendTickRate },
-          };
-        }
+    // TODO: Handle player disconnect while pointer visible
+    const interpolations = interpolationsRef.current;
+    for (let player of Object.values(partyState)) {
+      const id = player.userId;
+      const pointer = player.pointer;
+      if (!id) {
+        continue;
+      }
+      if (!(id in interpolations)) {
+        interpolations[id] = {
+          id,
+          from: null,
+          to: { ...pointer, time: performance.now() + sendTickRate },
+        };
+      } else if (
+        !compare(interpolations[id].to.position, pointer.position, 0.0001) ||
+        interpolations[id].to.visible !== pointer.visible
+      ) {
+        console.log("receive time", performance.now() - t, pointer.position);
+        t = performance.now();
+        const from = interpolations[id].to;
+        interpolations[id] = {
+          id,
+          from: {
+            ...from,
+            time: performance.now(),
+          },
+          to: {
+            ...pointer,
+            time: performance.now() + sendTickRate,
+          },
+        };
       }
     }
-
-    session.on("peerData", handlePeerData);
-
-    return () => {
-      session.off("peerData", handlePeerData);
-    };
-  });
+  }, [partyState]);
 
   // Animate to the peer pointer positions
   useEffect(() => {
@@ -116,36 +128,32 @@ function NetworkedMapPointer({ session, active, gridSize }) {
     function animate(time) {
       request = requestAnimationFrame(animate);
       let interpolatedPointerState = {};
-      for (let syncState of Object.values(syncedPointerStateRef.current)) {
-        if (!syncState.from || !syncState.to) {
+      for (let interp of Object.values(interpolationsRef.current)) {
+        if (!interp.from || !interp.to) {
           continue;
         }
-        const totalInterpTime = syncState.to.time - syncState.from.time;
-        const currentInterpTime = time - syncState.from.time;
+        const totalInterpTime = interp.to.time - interp.from.time;
+        const currentInterpTime = time - interp.from.time;
         const alpha = currentInterpTime / totalInterpTime;
 
         if (alpha >= 0 && alpha <= 1) {
-          interpolatedPointerState[syncState.id] = {
-            id: syncState.to.id,
-            visible: syncState.from.visible,
-            position: lerp(
-              syncState.from.position,
-              syncState.to.position,
-              alpha
-            ),
+          interpolatedPointerState[interp.id] = {
+            id: interp.id,
+            visible: interp.from.visible,
+            position: lerp(interp.from.position, interp.to.position, alpha),
           };
         }
-        if (alpha > 1 && !syncState.to.visible) {
-          interpolatedPointerState[syncState.id] = {
-            id: syncState.id,
-            visible: syncState.to.visible,
-            position: syncState.to.position,
+        if (alpha > 1 && !interp.to.visible) {
+          interpolatedPointerState[interp.id] = {
+            id: interp.id,
+            visible: interp.to.visible,
+            position: interp.to.position,
           };
-          delete syncedPointerStateRef.current[syncState.to.id];
+          delete interpolationsRef.current[interp.id];
         }
       }
       if (!isEmpty(interpolatedPointerState)) {
-        setPointerState((prev) => ({
+        setLocalPointerState((prev) => ({
           ...prev,
           ...interpolatedPointerState,
         }));
@@ -159,7 +167,7 @@ function NetworkedMapPointer({ session, active, gridSize }) {
 
   return (
     <Group>
-      {Object.values(pointerState).map((pointer) => (
+      {Object.values(localPointerState).map((pointer) => (
         <MapPointer
           key={pointer.id}
           gridSize={gridSize}
diff --git a/src/network/NetworkedParty.js b/src/network/NetworkedParty.js
index 809b671..40daf1e 100644
--- a/src/network/NetworkedParty.js
+++ b/src/network/NetworkedParty.js
@@ -5,7 +5,7 @@ import React, { useContext, useState, useEffect, useCallback } from "react";
 import Session from "./Session";
 import { isStreamStopped, omit } from "../helpers/shared";
 
-import PlayerContext from "../contexts/PlayerContext";
+import PartyContext from "../contexts/PartyContext";
 
 import Party from "../components/party/Party";
 
@@ -19,7 +19,7 @@ import Party from "../components/party/Party";
  * @param {NetworkedPartyProps} props
  */
 function NetworkedParty({ gameId, session }) {
-  const { partyState } = useContext(PlayerContext);
+  const partyState = useContext(PartyContext);
   const [stream, setStream] = useState(null);
   const [partyStreams, setPartyStreams] = useState({});
 
diff --git a/src/routes/Game.js b/src/routes/Game.js
index 7f505fc..702180e 100644
--- a/src/routes/Game.js
+++ b/src/routes/Game.js
@@ -13,6 +13,7 @@ import AuthContext from "../contexts/AuthContext";
 import { MapStageProvider } from "../contexts/MapStageContext";
 import DatabaseContext from "../contexts/DatabaseContext";
 import { PlayerProvider } from "../contexts/PlayerContext";
+import { PartyProvider } from "../contexts/PartyContext";
 
 import NetworkedMapAndTokens from "../network/NetworkedMapAndTokens";
 import NetworkedParty from "../network/NetworkedParty";
@@ -104,49 +105,54 @@ function Game() {
 
   return (
     <PlayerProvider session={session}>
-      <MapStageProvider value={mapStageRef}>
-        <Flex sx={{ flexDirection: "column", height: "100%" }}>
-          <Flex
-            sx={{
-              justifyContent: "space-between",
-              flexGrow: 1,
-              height: "100%",
-            }}
-          >
-            <NetworkedParty session={session} gameId={gameId} />
-            <NetworkedMapAndTokens session={session} />
+      <PartyProvider session={session}>
+        <MapStageProvider value={mapStageRef}>
+          <Flex sx={{ flexDirection: "column", height: "100%" }}>
+            <Flex
+              sx={{
+                justifyContent: "space-between",
+                flexGrow: 1,
+                height: "100%",
+              }}
+            >
+              <NetworkedParty session={session} gameId={gameId} />
+              <NetworkedMapAndTokens session={session} />
+            </Flex>
           </Flex>
-        </Flex>
-        <Banner isOpen={!!peerError} onRequestClose={() => setPeerError(null)}>
-          <Box p={1}>
-            <Text as="p" variant="body2">
-              {peerError} See <Link to="/faq#connection">FAQ</Link> for more
-              information.
-            </Text>
-          </Box>
-        </Banner>
-        <Banner isOpen={offline} onRequestClose={() => {}} allowClose={false}>
-          <Box p={1}>
-            <Text as="p" variant="body2">
-              Unable to connect to game, refresh to reconnect.
-            </Text>
-          </Box>
-        </Banner>
-        <Banner
-          isOpen={!connected && authenticationStatus === "authenticated"}
-          onRequestClose={() => {}}
-          allowClose={false}
-        >
-          <Box p={1}>
-            <Text as="p" variant="body2">
-              Disconnected. Attempting to reconnect...
-            </Text>
-          </Box>
-        </Banner>
-        <AuthModal isOpen={authenticationStatus === "unauthenticated"} />
-        {authenticationStatus === "unknown" && !offline && <LoadingOverlay />}
-        <MapLoadingOverlay />
-      </MapStageProvider>
+          <Banner
+            isOpen={!!peerError}
+            onRequestClose={() => setPeerError(null)}
+          >
+            <Box p={1}>
+              <Text as="p" variant="body2">
+                {peerError} See <Link to="/faq#connection">FAQ</Link> for more
+                information.
+              </Text>
+            </Box>
+          </Banner>
+          <Banner isOpen={offline} onRequestClose={() => {}} allowClose={false}>
+            <Box p={1}>
+              <Text as="p" variant="body2">
+                Unable to connect to game, refresh to reconnect.
+              </Text>
+            </Box>
+          </Banner>
+          <Banner
+            isOpen={!connected && authenticationStatus === "authenticated"}
+            onRequestClose={() => {}}
+            allowClose={false}
+          >
+            <Box p={1}>
+              <Text as="p" variant="body2">
+                Disconnected. Attempting to reconnect...
+              </Text>
+            </Box>
+          </Banner>
+          <AuthModal isOpen={authenticationStatus === "unauthenticated"} />
+          {authenticationStatus === "unknown" && !offline && <LoadingOverlay />}
+          <MapLoadingOverlay />
+        </MapStageProvider>
+      </PartyProvider>
     </PlayerProvider>
   );
 }
diff --git a/yarn.lock b/yarn.lock
index ffd0ea8..9347d08 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -5451,7 +5451,7 @@ fake-indexeddb@^3.1.2:
     realistic-structured-clone "^2.0.1"
     setimmediate "^1.0.5"
 
-fast-deep-equal@^3.1.1:
+fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3:
   version "3.1.3"
   resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525"
   integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==