diff --git a/packages/react-native-gesture-handler/android/build.gradle b/packages/react-native-gesture-handler/android/build.gradle
index 8e12a59ce8..372c0c2fa5 100644
--- a/packages/react-native-gesture-handler/android/build.gradle
+++ b/packages/react-native-gesture-handler/android/build.gradle
@@ -60,7 +60,22 @@ if (isNewArchitectureEnabled()) {
 }
 
 apply plugin: 'com.android.library'
-apply plugin: 'kotlin-android'
+
+def shouldEnableAgpFallback() {
+    def agpMajorVersion = com.android.Version.ANDROID_GRADLE_PLUGIN_VERSION.tokenize('.')[0].toInteger()
+    if (agpMajorVersion <= 8) {
+        return true
+    }
+
+    def propertyVal = providers.gradleProperty("android.builtInKotlin").orNull
+    def isBuiltInKotlinEnabled = propertyVal != null ? propertyVal.toBoolean() : true
+
+    return !isBuiltInKotlinEnabled
+}
+
+if (shouldEnableAgpFallback()) {
+    apply plugin: 'kotlin-android'
+}
 
 if (project == rootProject) {
     apply from: "spotless.gradle"
@@ -201,17 +216,17 @@ android {
     }
 
     sourceSets.main {
-        java {
+        kotlin {
             if (shouldUseCommonInterfaceFromReanimated()) {
-                srcDirs += 'reanimated/src/main/java'
+                directories.add('reanimated/src/main/java')
             } else {
-                srcDirs += 'noreanimated/src/main/java'
+                directories.add('noreanimated/src/main/java')
             }
 
             if (shouldUseCommonInterfaceFromRNSVG()) {
-                srcDirs += 'svg/src/main/java'
+                directories.add('svg/src/main/java')
             } else {
-                srcDirs += 'nosvg/src/main/java'
+                directories.add('nosvg/src/main/java')
             }
         }
     }