diff --git a/src/decode.rs b/src/decode.rs
index 0049a18..0f4e660 100644
--- a/src/decode.rs
+++ b/src/decode.rs
@@ -76,7 +76,8 @@ where
             }
         }
 
-        index[px.hash_index() as usize] = px;
+        let px_rgba = px.as_rgba(0xff);
+        index[px_rgba.hash_index() as usize] = px;
         *px_out = px.into();
     }
 
diff --git a/src/encode.rs b/src/encode.rs
index 33cf9bb..604620e 100644
--- a/src/encode.rs
+++ b/src/encode.rs
@@ -52,9 +52,9 @@ where
                 }
                 run = 0;
             }
-            hash_prev = px.hash_index();
-            let index_px = &mut index[hash_prev as usize];
             let px_rgba = px.as_rgba(0xff);
+            hash_prev = px_rgba.hash_index();
+            let index_px = &mut index[hash_prev as usize];
             if *index_px == px_rgba {
                 buf = buf.write_one(QOI_OP_INDEX | hash_prev)?;
             } else {