diff --git a/src/core/hle/service/hid/controllers/npad.cpp b/src/core/hle/service/hid/controllers/npad.cpp
index e5c951e064..aa6cb34b70 100644
--- a/src/core/hle/service/hid/controllers/npad.cpp
+++ b/src/core/hle/service/hid/controllers/npad.cpp
@@ -262,11 +262,6 @@ void Controller_NPad::OnInit() {
             service_context.CreateEvent(fmt::format("npad:NpadStyleSetChanged_{}", i));
     }
 
-    if (hid_core.GetSupportedStyleTag().raw == Core::HID::NpadStyleSet::None) {
-        // We want to support all controllers
-        hid_core.SetSupportedStyleTag({Core::HID::NpadStyleSet::All});
-    }
-
     supported_npad_id_types.resize(npad_id_list.size());
     std::memcpy(supported_npad_id_types.data(), npad_id_list.data(),
                 npad_id_list.size() * sizeof(Core::HID::NpadIdType));
@@ -288,14 +283,6 @@ void Controller_NPad::OnInit() {
             WriteEmptyEntry(npad);
         }
     }
-
-    // Connect controllers
-    for (auto& controller : controller_data) {
-        const auto& device = controller.device;
-        if (device->IsConnected()) {
-            AddNewControllerAt(device->GetNpadStyleIndex(), device->GetNpadIdType());
-        }
-    }
 }
 
 void Controller_NPad::WriteEmptyEntry(NpadInternalState& npad) {
@@ -320,6 +307,7 @@ void Controller_NPad::WriteEmptyEntry(NpadInternalState& npad) {
 }
 
 void Controller_NPad::OnRelease() {
+    is_controller_initialized = false;
     for (std::size_t i = 0; i < controller_data.size(); ++i) {
         auto& controller = controller_data[i];
         service_context.CloseEvent(controller.styleset_changed_event);
@@ -651,9 +639,27 @@ void Controller_NPad::OnMotionUpdate(const Core::Timing::CoreTiming& core_timing
 
 void Controller_NPad::SetSupportedStyleSet(Core::HID::NpadStyleTag style_set) {
     hid_core.SetSupportedStyleTag(style_set);
+
+    if (is_controller_initialized) {
+        return;
+    }
+
+    // Once SetSupportedStyleSet is called controllers are fully initialized
+    is_controller_initialized = true;
+
+    // Connect all active controllers
+    for (auto& controller : controller_data) {
+        const auto& device = controller.device;
+        if (device->IsConnected()) {
+            AddNewControllerAt(device->GetNpadStyleIndex(), device->GetNpadIdType());
+        }
+    }
 }
 
 Core::HID::NpadStyleTag Controller_NPad::GetSupportedStyleSet() const {
+    if (!is_controller_initialized) {
+        return {Core::HID::NpadStyleSet::None};
+    }
     return hid_core.GetSupportedStyleTag();
 }
 
diff --git a/src/core/hle/service/hid/controllers/npad.h b/src/core/hle/service/hid/controllers/npad.h
index 3287cf4358..c417a36bb0 100644
--- a/src/core/hle/service/hid/controllers/npad.h
+++ b/src/core/hle/service/hid/controllers/npad.h
@@ -511,7 +511,8 @@ private:
     NpadHandheldActivationMode handheld_activation_mode{NpadHandheldActivationMode::Dual};
     NpadCommunicationMode communication_mode{NpadCommunicationMode::Default};
     bool permit_vibration_session_enabled{false};
-    bool analog_stick_use_center_clamp{};
+    bool analog_stick_use_center_clamp{false};
     bool is_in_lr_assignment_mode{false};
+    bool is_controller_initialized{false};
 };
 } // namespace Service::HID
diff --git a/src/core/hle/service/hid/hid.cpp b/src/core/hle/service/hid/hid.cpp
index d9202ea6ce..ec9fda248a 100644
--- a/src/core/hle/service/hid/hid.cpp
+++ b/src/core/hle/service/hid/hid.cpp
@@ -63,10 +63,6 @@ IAppletResource::IAppletResource(Core::System& system_,
     MakeController<Controller_Gesture>(HidController::Gesture);
     MakeController<Controller_ConsoleSixAxis>(HidController::ConsoleSixAxisSensor);
 
-    // Homebrew doesn't try to activate some controllers, so we activate them by default
-    GetController<Controller_NPad>(HidController::NPad).ActivateController();
-    GetController<Controller_Touchscreen>(HidController::Touchscreen).ActivateController();
-
     GetController<Controller_Stubbed>(HidController::HomeButton).SetCommonHeaderOffset(0x4C00);
     GetController<Controller_Stubbed>(HidController::SleepButton).SetCommonHeaderOffset(0x4E00);
     GetController<Controller_Stubbed>(HidController::CaptureButton).SetCommonHeaderOffset(0x5000);