diff --git a/src/components/map/MapDice.js b/src/components/map/MapDice.js
index 1f606e0..21934ef 100644
--- a/src/components/map/MapDice.js
+++ b/src/components/map/MapDice.js
@@ -18,6 +18,7 @@ function MapDice() {
         bottom: 0,
         flexDirection: "column",
         alignItems: "flex-start",
+        pointerEvents: "none",
       }}
       ml={1}
     >
@@ -29,6 +30,7 @@ function MapDice() {
           display: "block",
           backgroundColor: "overlay",
           borderRadius: "50%",
+          pointerEvents: "all",
         }}
         m={2}
       >
diff --git a/src/components/map/MapInteraction.js b/src/components/map/MapInteraction.js
index 3362f53..2fa5330 100644
--- a/src/components/map/MapInteraction.js
+++ b/src/components/map/MapInteraction.js
@@ -75,6 +75,7 @@ function MapInteraction({ map, children, controls, selectedToolId }) {
 
   const pinchPreviousDistanceRef = useRef();
   const pinchPreviousOriginRef = useRef();
+  const isDraggingCanvas = useRef(false);
 
   const bind = useGesture({
     onWheel: ({ delta }) => {
@@ -120,8 +121,12 @@ function MapInteraction({ map, children, controls, selectedToolId }) {
       pinchPreviousDistanceRef.current = distance;
       pinchPreviousOriginRef.current = { x: originX, y: originY };
     },
+    onDragStart: ({ event }) => {
+      isDraggingCanvas.current =
+        event.target === mapLayerRef.current.getCanvas()._canvas;
+    },
     onDrag: ({ delta, xy, first, last, pinching }) => {
-      if (preventMapInteraction || pinching) {
+      if (preventMapInteraction || pinching || !isDraggingCanvas.current) {
         return;
       }
 
diff --git a/src/components/map/MapMenu.js b/src/components/map/MapMenu.js
index f710585..2c79ddb 100644
--- a/src/components/map/MapMenu.js
+++ b/src/components/map/MapMenu.js
@@ -1,11 +1,7 @@
-import React, { useEffect, useState, useContext } from "react";
+import React, { useEffect, useState } from "react";
 import Modal from "react-modal";
 import { useThemeUI } from "theme-ui";
 
-import MapInteractionContext from "../../contexts/MapInteractionContext";
-
-import usePrevious from "../../helpers/usePrevious";
-
 function MapMenu({
   isOpen,
   onRequestClose,
@@ -24,17 +20,6 @@ function MapMenu({
   // callback
   const [modalContentNode, setModalContentNode] = useState(null);
 
-  // Toggle map interaction when menu is opened
-  const wasOpen = usePrevious(isOpen);
-  const { setPreventMapInteraction } = useContext(MapInteractionContext);
-  useEffect(() => {
-    if (isOpen && !wasOpen) {
-      setPreventMapInteraction(true);
-    } else if (wasOpen && !isOpen) {
-      setPreventMapInteraction(false);
-    }
-  }, [isOpen, setPreventMapInteraction, wasOpen]);
-
   useEffect(() => {
     // Close modal if interacting with any other element
     function handlePointerDown(event) {
diff --git a/src/components/map/SelectMapButton.js b/src/components/map/SelectMapButton.js
index f397a89..fb8c81f 100644
--- a/src/components/map/SelectMapButton.js
+++ b/src/components/map/SelectMapButton.js
@@ -5,7 +5,6 @@ import SelectMapModal from "../../modals/SelectMapModal";
 import SelectMapIcon from "../../icons/SelectMapIcon";
 
 import MapDataContext from "../../contexts/MapDataContext";
-import MapInteractionContext from "../../contexts/MapInteractionContext";
 
 function SelectMapButton({
   onMapChange,
@@ -15,16 +14,13 @@ function SelectMapButton({
 }) {
   const [isModalOpen, setIsModalOpen] = useState(false);
 
-  const { setPreventMapInteraction } = useContext(MapInteractionContext);
   const { updateMapState } = useContext(MapDataContext);
   function openModal() {
     currentMapState && updateMapState(currentMapState.mapId, currentMapState);
     setIsModalOpen(true);
-    setPreventMapInteraction(true);
   }
   function closeModal() {
     setIsModalOpen(false);
-    setPreventMapInteraction(false);
   }
 
   function handleDone() {
diff --git a/src/components/map/dice/DiceTrayOverlay.js b/src/components/map/dice/DiceTrayOverlay.js
index f47daf3..44ced35 100644
--- a/src/components/map/dice/DiceTrayOverlay.js
+++ b/src/components/map/dice/DiceTrayOverlay.js
@@ -17,7 +17,6 @@ import LoadingOverlay from "../../LoadingOverlay";
 
 import DiceTray from "../../../dice/diceTray/DiceTray";
 
-import MapInteractionContext from "../../../contexts/MapInteractionContext";
 import DiceLoadingContext from "../../../contexts/DiceLoadingContext";
 
 function DiceTrayOverlay({ isOpen }) {
@@ -212,8 +211,6 @@ function DiceTrayOverlay({ isOpen }) {
     assetLoadFinish();
   }
 
-  const { setPreventMapInteraction } = useContext(MapInteractionContext);
-
   return (
     <Box
       sx={{
@@ -227,6 +224,7 @@ function DiceTrayOverlay({ isOpen }) {
         display: isOpen ? "block" : "none",
         position: "relative",
         overflow: "hidden",
+        pointerEvents: "all",
       }}
       bg="background"
     >
@@ -234,11 +232,9 @@ function DiceTrayOverlay({ isOpen }) {
         onSceneMount={handleSceneMount}
         onPointerDown={() => {
           sceneInteractionRef.current = true;
-          setPreventMapInteraction(true);
         }}
         onPointerUp={() => {
           sceneInteractionRef.current = false;
-          setPreventMapInteraction(false);
         }}
       />
       <DiceControls
diff --git a/src/components/map/dice/SelectDiceButton.js b/src/components/map/dice/SelectDiceButton.js
index 7f470ec..65332bf 100644
--- a/src/components/map/dice/SelectDiceButton.js
+++ b/src/components/map/dice/SelectDiceButton.js
@@ -1,22 +1,17 @@
-import React, { useState, useContext } from "react";
+import React, { useState } from "react";
 import { IconButton } from "theme-ui";
 
 import SelectDiceIcon from "../../../icons/SelectDiceIcon";
 import SelectDiceModal from "../../../modals/SelectDiceModal";
 
-import MapInteractionContext from "../../../contexts/MapInteractionContext";
-
 function SelectDiceButton({ onDiceChange, currentDice }) {
   const [isModalOpen, setIsModalOpen] = useState(false);
-  const { setPreventMapInteraction } = useContext(MapInteractionContext);
 
   function openModal() {
     setIsModalOpen(true);
-    setPreventMapInteraction(true);
   }
   function closeModal() {
     setIsModalOpen(false);
-    setPreventMapInteraction(false);
   }
 
   function handleDone(dice) {