diff --git a/packages/web/components/patterns/LibraryCards/LibraryGridCard.tsx b/packages/web/components/patterns/LibraryCards/LibraryGridCard.tsx
index 6509c251c..dab543b99 100644
--- a/packages/web/components/patterns/LibraryCards/LibraryGridCard.tsx
+++ b/packages/web/components/patterns/LibraryCards/LibraryGridCard.tsx
@@ -18,6 +18,7 @@ import {
   timeAgo,
   TitleStyle,
 } from './LibraryCardStyles'
+import { sortedLabels } from '../../../lib/labelsSort'
 
 dayjs.extend(relativeTime)
 
@@ -178,11 +179,11 @@ export function LibraryGridCard(props: LinkedItemCardProps): JSX.Element {
                   minHeight: '35px',
                 }}
               >
-                {props.item.labels
-                  ?.sort((a, b) => a.name.localeCompare(b.name))
-                  .map(({ name, color }, index) => (
+                {sortedLabels(props.item.labels).map(
+                  ({ name, color }, index) => (
                     <LabelChip key={index} text={name || ''} color={color} />
-                  ))}
+                  )
+                )}
               </HStack>
               <VStack
                 css={{
diff --git a/packages/web/components/patterns/LibraryCards/LibraryListCard.tsx b/packages/web/components/patterns/LibraryCards/LibraryListCard.tsx
index b94aa4575..742114b5d 100644
--- a/packages/web/components/patterns/LibraryCards/LibraryListCard.tsx
+++ b/packages/web/components/patterns/LibraryCards/LibraryListCard.tsx
@@ -14,6 +14,7 @@ import {
   timeAgo,
   TitleStyle,
 } from './LibraryCardStyles'
+import { sortedLabels } from '../../../lib/labelsSort'
 
 export function LibraryListCard(props: LinkedItemCardProps): JSX.Element {
   const [isHovered, setIsHovered] = useState(false)
@@ -118,11 +119,11 @@ export function LibraryListCard(props: LinkedItemCardProps): JSX.Element {
                   display: 'block',
                 }}
               >
-                {props.item.labels
-                  ?.sort((a, b) => a.name.localeCompare(b.name))
-                  .map(({ name, color }, index) => (
+                {sortedLabels(props.item.labels).map(
+                  ({ name, color }, index) => (
                     <LabelChip key={index} text={name || ''} color={color} />
-                  ))}
+                  )
+                )}
               </HStack>
             </HStack>
           </VStack>
diff --git a/packages/web/lib/labelsSort.ts b/packages/web/lib/labelsSort.ts
new file mode 100644
index 000000000..189852e9e
--- /dev/null
+++ b/packages/web/lib/labelsSort.ts
@@ -0,0 +1,36 @@
+import { Label } from './networking/fragments/labelFragment'
+
+export const sortedLabels = (labels: Label[] | undefined): Label[] => {
+  if (!labels) {
+    return []
+  }
+
+  const colors = new Map<string, Label[]>()
+  for (const label of labels) {
+    let list = colors.get(label.color) ?? []
+    list.push(label)
+    colors.set(
+      label.color,
+      list.sort((a, b) => a.name.localeCompare(b.name))
+    )
+  }
+
+  const sortedColors = Array.from(colors.keys()).sort((a, b) => {
+    // Sort by the first element's name
+    const aname = colors.get(a)?.find(() => true)?.name ?? a
+    const bname = colors.get(b)?.find(() => true)?.name ?? b
+    return aname.localeCompare(bname)
+  })
+
+  const result: Label[] = []
+  for (const key of sortedColors) {
+    const items = colors.get(key)
+    if (!items) {
+      continue
+    }
+    const sorted = items.sort((a, b) => a.name.localeCompare(b.name))
+    result.push(...sorted)
+  }
+
+  return result
+}