diff --git a/src/components/map/Map.tsx b/src/components/map/Map.tsx
index dcc5157..d480d0e 100644
--- a/src/components/map/Map.tsx
+++ b/src/components/map/Map.tsx
@@ -151,8 +151,7 @@ function Map({
     onMapNoteCreate,
     onMapNoteChange,
     onMapNoteRemove,
-    selectedToolId,
-    !!(map?.owner === userId || mapState?.editFlags.includes("notes"))
+    selectedToolId
   );
 
   const { selectionTool, selectionMenu, selectionDragOverlay } =
diff --git a/src/components/tools/SelectTool.tsx b/src/components/tools/SelectTool.tsx
index 04d4876..668406a 100644
--- a/src/components/tools/SelectTool.tsx
+++ b/src/components/tools/SelectTool.tsx
@@ -42,6 +42,8 @@ type MapSelectProps = {
   onSelectionMenuOpen: (open: boolean) => void;
   onSelectionDragStart: () => void;
   onSelectionDragEnd: () => void;
+  disabledTokens: Record<string, boolean>;
+  disabledNotes: Record<string, boolean>;
 };
 
 function SelectTool({
@@ -53,6 +55,8 @@ function SelectTool({
   onSelectionMenuOpen,
   onSelectionDragStart,
   onSelectionDragEnd,
+  disabledTokens,
+  disabledNotes,
 }: MapSelectProps) {
   const stageScale = useDebouncedStageScale();
   const mapWidth = useMapWidth();
@@ -192,7 +196,10 @@ function SelectTool({
           const tokens = tokensGroup.children;
           if (tokens) {
             for (let token of tokens) {
-              if (intersection.intersects(token.position())) {
+              if (
+                !(token.id() in disabledTokens) &&
+                intersection.intersects(token.position())
+              ) {
                 intersectingItems.push({ type: "token", id: token.id() });
               }
             }
@@ -200,7 +207,10 @@ function SelectTool({
           const notes = notesGroup.children;
           if (notes) {
             for (let note of notes) {
-              if (intersection.intersects(note.position())) {
+              if (
+                !(note.id() in disabledNotes) &&
+                intersection.intersects(note.position())
+              ) {
                 intersectingItems.push({ type: "note", id: note.id() });
               }
             }
diff --git a/src/hooks/useMapNotes.tsx b/src/hooks/useMapNotes.tsx
index 1d93c09..c3cfb54 100644
--- a/src/hooks/useMapNotes.tsx
+++ b/src/hooks/useMapNotes.tsx
@@ -4,6 +4,7 @@ import { useState } from "react";
 import NoteDragOverlay from "../components/note/NoteDragOverlay";
 import NoteMenu from "../components/note/NoteMenu";
 import NoteTool from "../components/tools/NoteTool";
+import { useUserId } from "../contexts/UserIdContext";
 import {
   NoteChangeEventHandler,
   NoteCreateEventHander,
@@ -19,13 +20,18 @@ function useMapNotes(
   onNoteCreate: NoteCreateEventHander,
   onNoteChange: NoteChangeEventHandler,
   onNoteRemove: NoteRemoveEventHander,
-  selectedToolId: MapToolId,
-  allowNoteEditing: boolean
+  selectedToolId: MapToolId
 ) {
+  const userId = useUserId();
+  const allowNoteEditing = !!(
+    map?.owner === userId || mapState?.editFlags.includes("notes")
+  );
+
   const [isNoteMenuOpen, setIsNoteMenuOpen] = useState<boolean>(false);
   const [noteMenuOptions, setNoteMenuOptions] = useState<NoteMenuOptions>();
   const [noteDraggingOptions, setNoteDraggingOptions] =
     useState<NoteDraggingOptions>();
+
   function handleNoteMenuOpen(noteId: string, noteNode: Konva.Node) {
     setNoteMenuOptions({ noteId, noteNode });
     setIsNoteMenuOpen(true);
diff --git a/src/hooks/useMapSelection.tsx b/src/hooks/useMapSelection.tsx
index ae9c9fe..d457383 100644
--- a/src/hooks/useMapSelection.tsx
+++ b/src/hooks/useMapSelection.tsx
@@ -2,6 +2,7 @@ import { useEffect, useState } from "react";
 import SelectionDragOverlay from "../components/selection/SelectionDragOverlay";
 import SelectionMenu from "../components/selection/SelectionMenu";
 import SelectTool from "../components/tools/SelectTool";
+import { useUserId } from "../contexts/UserIdContext";
 import {
   SelectionItemsChangeEventHandler,
   SelectionItemsCreateEventHandler,
@@ -21,6 +22,24 @@ function useMapSelection(
   selectedToolId: MapToolId,
   settings: SelectToolSettings
 ) {
+  const userId = useUserId();
+  const disabledTokens: Record<string, boolean> = {};
+  const disabledNotes: Record<string, boolean> = {};
+  if (mapState && map && map.owner !== userId) {
+    if (!mapState.editFlags.includes("tokens")) {
+      for (let token of Object.values(mapState.tokens)) {
+        if (token.owner !== userId) {
+          disabledTokens[token.id] = true;
+        }
+      }
+    }
+    if (!mapState.editFlags.includes("notes")) {
+      for (let note of Object.values(mapState.notes)) {
+        disabledNotes[note.id] = true;
+      }
+    }
+  }
+
   const [isSelectionMenuOpen, setIsSelectionMenuOpen] =
     useState<boolean>(false);
   const [isSelectionDragging, setIsSelectionDragging] = useState(false);
@@ -72,6 +91,8 @@ function useMapSelection(
       onSelectionMenuOpen={handleSelectionMenuOpen}
       onSelectionDragStart={handleSelectionDragStart}
       onSelectionDragEnd={handleSelectionDragEnd}
+      disabledTokens={disabledTokens}
+      disabledNotes={disabledNotes}
     />
   );
 
diff --git a/src/hooks/useMapTokens.tsx b/src/hooks/useMapTokens.tsx
index e9b02ac..12c9540 100644
--- a/src/hooks/useMapTokens.tsx
+++ b/src/hooks/useMapTokens.tsx
@@ -27,28 +27,23 @@ function useMapTokens(
   onTokenStateRemove: TokenStateRemoveHandler,
   selectedToolId: MapToolId
 ) {
+  const userId = useUserId();
+  const disabledTokens: Record<string, boolean> = {};
+  if (mapState && map) {
+    if (!mapState.editFlags.includes("tokens") && map.owner !== userId) {
+      for (let token of Object.values(mapState.tokens)) {
+        if (token.owner !== userId) {
+          disabledTokens[token.id] = true;
+        }
+      }
+    }
+  }
+
   const [isTokenMenuOpen, setIsTokenMenuOpen] = useState<boolean>(false);
   const [tokenMenuOptions, setTokenMenuOptions] = useState<TokenMenuOptions>();
   const [tokenDraggingOptions, setTokenDraggingOptions] =
     useState<TokenDraggingOptions>();
 
-  const userId = useUserId();
-
-  const disabledTokens: Record<string, boolean> = {};
-  // If we have a map and state and have the token permission disabled
-  // and are not the map owner
-  if (
-    mapState &&
-    !mapState.editFlags.includes("tokens") &&
-    map?.owner !== userId
-  ) {
-    for (let token of Object.values(mapState.tokens)) {
-      if (token.owner !== userId) {
-        disabledTokens[token.id] = true;
-      }
-    }
-  }
-
   function handleTokenMenuOpen(tokenStateId: string, tokenImage: Konva.Node) {
     setTokenMenuOptions({ tokenStateId, tokenImage });
     setIsTokenMenuOpen(true);