Abrarul8541 commited on
Commit
7af5b7c
·
verified ·
1 Parent(s): ba77a6b

Phase 4: opt-in accessibility service (screen text dump, tap)

Browse files
app/src/main/java/io/github/abrarul/modularai/service/AgentAccessibilityService.kt ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ package io.github.abrarul.modularai.service
2
+
3
+ import android.accessibilityservice.AccessibilityService
4
+ import android.accessibilityservice.GestureDescription
5
+ import android.graphics.Path
6
+ import android.view.accessibility.AccessibilityEvent
7
+ import android.view.accessibility.AccessibilityNodeInfo
8
+
9
+ /**
10
+ * Off until the user enables it in Android Settings > Accessibility. It reads nothing on its own;
11
+ * it only answers the explicit console commands `screen` and `tap`.
12
+ */
13
+ class AgentAccessibilityService : AccessibilityService() {
14
+
15
+ override fun onServiceConnected() { instance = this }
16
+ override fun onAccessibilityEvent(event: AccessibilityEvent?) {}
17
+ override fun onInterrupt() {}
18
+ override fun onDestroy() { if (instance === this) instance = null; super.onDestroy() }
19
+
20
+ fun dumpText(): String {
21
+ val sb = StringBuilder()
22
+ fun walk(n: AccessibilityNodeInfo?, depth: Int) {
23
+ if (n == null || depth > 40) return
24
+ val t = n.text ?: n.contentDescription
25
+ if (!t.isNullOrBlank()) sb.append(" ".repeat(depth)).append(t).append('\n')
26
+ for (i in 0 until n.childCount) walk(n.getChild(i), depth + 1)
27
+ }
28
+ walk(rootInActiveWindow, 0)
29
+ return sb.toString()
30
+ }
31
+
32
+ fun tap(x: Float, y: Float): Boolean {
33
+ val path = Path().apply { moveTo(x, y) }
34
+ val g = GestureDescription.Builder().addStroke(GestureDescription.StrokeDescription(path, 0, 50)).build()
35
+ return dispatchGesture(g, null, null)
36
+ }
37
+
38
+ companion object {
39
+ @Volatile var instance: AgentAccessibilityService? = null
40
+ }
41
+ }