OpenGL: Respect stencil-read allow register

src/video_core/renderer_opengl/gl_rasterizer.cpp

@@ -151,8 +151,8 @@ void RasterizerOpenGL::Reset() {
     SyncBlendFuncs();
     SyncBlendColor();
     SyncLogicOp();
-    SyncStencilTest();
     SyncDepthTest();
+    SyncStencilTest();
     SyncColorWriteMask();
     SyncStencilWriteMask();
     SyncDepthWriteMask();
@@ -291,6 +291,7 @@ void RasterizerOpenGL::NotifyPicaRegisterChanged(u32 id) {
     // (Pica depth test function register also contains a depth and color write mask)
     case PICA_REG_INDEX(output_merger.depth_test_enable):
         SyncDepthTest();
+        SyncStencilTest();
         SyncDepthWriteMask();
         SyncColorWriteMask();
         break;
@@ -317,6 +318,7 @@ void RasterizerOpenGL::NotifyPicaRegisterChanged(u32 id) {
     // Stencil and depth read mask
     case PICA_REG_INDEX(framebuffer.allow_stencil_read):
         SyncDepthTest();
+        SyncStencilTest();
         break;
 
     // Logic op
@@ -1024,17 +1026,117 @@ void RasterizerOpenGL::SyncDepthWriteMask() {
                            : GL_FALSE;
 }
 
+// Depends on the correct GL DepthTest state!
 void RasterizerOpenGL::SyncStencilTest() {
     const auto& regs = Pica::g_state.regs;
-    state.stencil.test_enabled = regs.output_merger.stencil_test.enable && regs.framebuffer.depth_format == Pica::Regs::DepthFormat::D24S8;
+    const auto& stencil_test = regs.output_merger.stencil_test;
-    state.stencil.test_func = PicaToGL::CompareFunc(regs.output_merger.stencil_test.func);
+
-    state.stencil.test_ref = regs.output_merger.stencil_test.reference_value;
+    state.stencil.test_enabled = stencil_test.enable && regs.framebuffer.depth_format == Pica::Regs::DepthFormat::D24S8;
-    state.stencil.test_mask = regs.output_merger.stencil_test.input_mask;
+
-    state.stencil.action_stencil_fail = PicaToGL::StencilOp(regs.output_merger.stencil_test.action_stencil_fail);
+    if (state.stencil.test_enabled) {
-    state.stencil.action_depth_fail = PicaToGL::StencilOp(regs.output_merger.stencil_test.action_depth_fail);
+        if (!regs.framebuffer.allow_stencil_read) {
-    state.stencil.action_depth_pass = PicaToGL::StencilOp(regs.output_merger.stencil_test.action_depth_pass);
+
+            // All stencil reads must be emulated as 0x00
+
+            u8 masked_ref = (stencil_test.reference_value & stencil_test.input_mask);
+
+            if (masked_ref == 0x00) {
+                // x = 0x00
+                state.stencil.test_func = PicaToGL::CompareXToXFunc(stencil_test.func);
+            } else {
+                // x = masked stencil ref
+                state.stencil.test_func = PicaToGL::CompareXToZeroFunc(stencil_test.func);
             }
 
+            //FIXME: check if writeback is possible, otherwise this is useless
+            if (true) {
+
+                // We need a stencil read if we can't decide the stencil test staticly
+                bool needs_stencil_read = (state.stencil.test_func != GL_NEVER &&
+                                           state.stencil.test_func != GL_ALWAYS);
+
+                //TODO: In rare cases the stencil test doesn't depend on the masked ref ('Never' and 'Always') and only 1 specific value for
+                //      the stencil op writes is necessary. In those cases one could set the ref and ref-mask to the required value.
+                //FIXME: Return value instead..?
+                auto StencilAllowedOp = [&](Pica::Regs::StencilAction action) -> GLenum {
+
+                    switch (action) {
+
+                    // FIXME: Decide for one of those:
+                    //a. Keeping the framebuffer value means reading zero, writing back zero
+                    //b. Keeping the framebuffer value means not touching it
+                    case Pica::Regs::StencilAction::Keep:
+                        return GL_ZERO; // a
+                        return GL_KEEP; // b
+
+                    // Always 0x01, requires 0x01 in masked_ref to work
+                    case Pica::Regs::StencilAction::Increment:
+                    case Pica::Regs::StencilAction::IncrementWrap:
+                        if (masked_ref != 0x01) {
+                            needs_stencil_read = true;
+                        }
+                        return GL_REPLACE;
+
+                    // Always 0xFF, requires 0xFF in masked_ref to work
+                    case Pica::Regs::StencilAction::Invert:
+                    case Pica::Regs::StencilAction::DecrementWrap:
+                        if (masked_ref != 0xFF) {
+                            needs_stencil_read = true;
+                        }
+                        return GL_REPLACE;
+
+                    // Always masked ref
+                    case Pica::Regs::StencilAction::Replace:
+                        return GL_REPLACE;
+
+                    // Always 0x00
+                    case Pica::Regs::StencilAction::Zero:
+                    case Pica::Regs::StencilAction::Decrement:
+                        return GL_ZERO;
+
+                    default:
+                        LOG_CRITICAL(Render_OpenGL, "Unknown stencil action %x", (int)action);
+                        UNIMPLEMENTED();
+                        return GL_KEEP;
+                    }
+
+                };
+
+                // If the stencil test can fail we have to check the stencil op
+                if (state.depth.test_func != GL_ALWAYS) {
+                    state.stencil.action_stencil_fail = StencilAllowedOp(stencil_test.action_stencil_fail);
+                }
+
+                // If the depth test can pass we have to check the stencil op
+                if (state.depth.test_func != GL_NEVER) {
+                    state.stencil.action_depth_fail = StencilAllowedOp(stencil_test.action_depth_pass);
+                }
+
+                // If the depth test can fail we have to check the stencil op
+                if (state.depth.test_func != GL_ALWAYS) {
+                    state.stencil.action_depth_pass = StencilAllowedOp(stencil_test.action_depth_fail);
+                }
+
+                // Now check if we support this mode
+                if (needs_stencil_read) {
+                    LOG_CRITICAL(Render_OpenGL, "Can't emulate disabled read from stencil yet");
+                    UNIMPLEMENTED();
+                }
+
+            }
+
+        } else {
+            state.stencil.test_func = PicaToGL::CompareFunc(stencil_test.func);
+            state.stencil.test_ref = stencil_test.reference_value;
+            state.stencil.test_mask = stencil_test.input_mask;
+            state.stencil.action_stencil_fail = PicaToGL::StencilOp(stencil_test.action_stencil_fail);
+            state.stencil.action_depth_fail = PicaToGL::StencilOp(stencil_test.action_depth_fail);
+            state.stencil.action_depth_pass = PicaToGL::StencilOp(stencil_test.action_depth_pass);
+        }
+    }
+}
+
+// Always call SyncStencilTest after this returns!
 void RasterizerOpenGL::SyncDepthTest() {
     const auto& regs = Pica::g_state.regs;
 

src/video_core/renderer_opengl/pica_to_gl.h

@@ -155,6 +155,30 @@ inline GLenum CompareFunc(Pica::Regs::CompareFunc func) {
     return compare_func_table[(unsigned)func];
 }
 
+/// Pretend we compare 'x FUNC x'
+inline GLenum CompareXToXFunc(Pica::Regs::CompareFunc func) {
+    static const GLenum compare_func_table[] = {
+        GL_NEVER,    // CompareFunc::Never
+        GL_ALWAYS,   // CompareFunc::Always
+        GL_ALWAYS,   // CompareFunc::Equal
+        GL_NEVER,    // CompareFunc::NotEqual
+        GL_NEVER,    // CompareFunc::LessThan
+        GL_ALWAYS,   // CompareFunc::LessThanOrEqual
+        GL_NEVER,    // CompareFunc::GreaterThan
+        GL_ALWAYS,   // CompareFunc::GreaterThanOrEqual
+    };
+
+    // Range check table for input
+    if (static_cast<size_t>(func) >= ARRAY_SIZE(compare_func_table)) {
+        LOG_CRITICAL(Render_OpenGL, "Unknown compare function %d", func);
+        UNREACHABLE();
+
+        return GL_ALWAYS;
+    }
+
+    return compare_func_table[(unsigned)func];
+}
+
 /// Pretend we compare 'x FUNC 0' (unsigned)
 inline GLenum CompareXToZeroFunc(Pica::Regs::CompareFunc func) {
     static const GLenum compare_func_table[] = {