feat(route): domain matcher + bundled geosite DB search
Suffix/exact/keyword/regex matcher, normalizeDomain, and a hand-rolled geosite.dat parser (MIT v2fly) with category/domain search. Unit-tested.
This commit is contained in:
parent
8ed667d49e
commit
e01b0c2a87
8 changed files with 24067 additions and 0 deletions
21
app/src/main/assets/geosite/LICENSE
Normal file
21
app/src/main/assets/geosite/LICENSE
Normal file
|
|
@ -0,0 +1,21 @@
|
||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2018-2019 V2Ray
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
23413
app/src/main/assets/geosite/geosite.dat
Normal file
23413
app/src/main/assets/geosite/geosite.dat
Normal file
File diff suppressed because one or more lines are too long
48
app/src/main/java/chat/vojo/proxy/core/route/Domains.kt
Normal file
48
app/src/main/java/chat/vojo/proxy/core/route/Domains.kt
Normal file
|
|
@ -0,0 +1,48 @@
|
||||||
|
package chat.vojo.proxy.core.route
|
||||||
|
|
||||||
|
import java.net.IDN
|
||||||
|
|
||||||
|
// A label is 1..63 chars of [a-z0-9-], not starting/ending with '-'. Punycode
|
||||||
|
// labels (xn--…) and digits are covered by this set, so IDN input survives
|
||||||
|
// IDN.toASCII and still validates.
|
||||||
|
private val LABEL = Regex("^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$")
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Normalises user-typed site input into a bare ASCII registrable host suitable for
|
||||||
|
* suffix matching, or null if it isn't a usable domain.
|
||||||
|
*
|
||||||
|
* Forgiving on purpose — a user may paste a full URL. We strip scheme, userinfo,
|
||||||
|
* path/query/fragment and port, lowercase, drop a trailing dot, punycode-encode IDNs
|
||||||
|
* (IDN.toASCII), then validate the label structure. A bare TLD or single label is
|
||||||
|
* rejected: a one-label suffix would match an entire TLD, which is never intended.
|
||||||
|
*/
|
||||||
|
fun normalizeDomain(input: String): String? {
|
||||||
|
var s = input.trim().lowercase()
|
||||||
|
if (s.isEmpty()) return null
|
||||||
|
|
||||||
|
val scheme = s.indexOf("://")
|
||||||
|
if (scheme >= 0) s = s.substring(scheme + 3)
|
||||||
|
// Authority only: cut everything from the first path/query/fragment separator.
|
||||||
|
s = s.substringBefore('/').substringBefore('?').substringBefore('#')
|
||||||
|
if ('@' in s) s = s.substringAfterLast('@') // drop user:pass@
|
||||||
|
s = s.substringBefore(':') // drop :port
|
||||||
|
s = s.trim('.')
|
||||||
|
if (s.isEmpty()) return null
|
||||||
|
|
||||||
|
val ascii = try { IDN.toASCII(s) } catch (e: IllegalArgumentException) { return null }
|
||||||
|
val host = ascii.lowercase()
|
||||||
|
if (!isValidDomain(host)) return null
|
||||||
|
return host
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A multi-label host (at least one dot) whose every label is a valid DNS label. */
|
||||||
|
private fun isValidDomain(host: String): Boolean {
|
||||||
|
if (host.length > 253) return false
|
||||||
|
val labels = host.split('.')
|
||||||
|
if (labels.size < 2) return false
|
||||||
|
// Reject IPv4 literals (and any all-numeric final label — no such TLD exists). The
|
||||||
|
// suffix matcher only ever sees domains; IP destinations bypass it (isDomain=false),
|
||||||
|
// so an IP entry would be a silently-dead rule. Rejecting it lets the UI flag it.
|
||||||
|
if (labels.last().all { it.isDigit() }) return false
|
||||||
|
return labels.all { LABEL.matches(it) }
|
||||||
|
}
|
||||||
180
app/src/main/java/chat/vojo/proxy/core/route/Geosite.kt
Normal file
180
app/src/main/java/chat/vojo/proxy/core/route/Geosite.kt
Normal file
|
|
@ -0,0 +1,180 @@
|
||||||
|
package chat.vojo.proxy.core.route
|
||||||
|
|
||||||
|
/** A geosite category surfaced by search: its code plus how many rules it carries. */
|
||||||
|
data class GeositeHit(val category: String, val ruleCount: Int, val matchedByName: Boolean)
|
||||||
|
|
||||||
|
/**
|
||||||
|
* In-memory index of the bundled geosite database (MIT v2fly `geosite.dat`, a
|
||||||
|
* `GeoSiteList` protobuf of category → domain rules). Built once from the asset bytes by
|
||||||
|
* [parse] — a tiny hand-rolled protobuf reader, no protobuf-java dependency — and shared
|
||||||
|
* process-wide (see data/GeositeRepository). Categories are keyed lowercased.
|
||||||
|
*
|
||||||
|
* geosite.dat schema (v2fly/v2ray-core routercommon):
|
||||||
|
* GeoSiteList { repeated GeoSite entry = 1 }
|
||||||
|
* GeoSite { string country_code = 1; repeated Domain domain = 2; ... }
|
||||||
|
* Domain { Type type = 1; string value = 2; ... } Type: 0 Plain(keyword),
|
||||||
|
* 1 Regex, 2 RootDomain(suffix), 3 Full(exact)
|
||||||
|
*/
|
||||||
|
class GeositeIndex private constructor(
|
||||||
|
private val byCategory: Map<String, List<DomainRule>>,
|
||||||
|
) : GeositeSource {
|
||||||
|
|
||||||
|
/** All category codes, lowercased, sorted — for browsing/suggestions. */
|
||||||
|
val categories: List<String> = byCategory.keys.sorted()
|
||||||
|
|
||||||
|
// Geosite regexp rules compiled ONCE (the DB has a few hundred) so a domain search
|
||||||
|
// doesn't recompile them on every keystroke; uncompilable patterns map to null.
|
||||||
|
private val regexCache: Map<String, Regex?> = buildMap {
|
||||||
|
for (rules in byCategory.values) for (r in rules) {
|
||||||
|
if (r is DomainRule.Regexp && r.value !in this) {
|
||||||
|
put(r.value, runCatching { Regex(r.value, RegexOption.IGNORE_CASE) }.getOrNull())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun rules(category: String): List<DomainRule> =
|
||||||
|
byCategory[category.trim().lowercase()] ?: emptyList()
|
||||||
|
|
||||||
|
private fun covers(rule: DomainRule, q: String): Boolean = when (rule) {
|
||||||
|
is DomainRule.Suffix -> q == rule.value || q.endsWith(".${rule.value}")
|
||||||
|
is DomainRule.Exact -> q == rule.value
|
||||||
|
is DomainRule.Keyword -> q.contains(rule.value)
|
||||||
|
is DomainRule.Regexp -> regexCache[rule.value]?.containsMatchIn(q) == true
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Suggestions for [query]: categories whose code contains the query (name match,
|
||||||
|
* ranked first), plus — when the query looks like a domain — categories that actually
|
||||||
|
* cover it (domain match). The costly per-rule domain scan is skipped once [limit] hits
|
||||||
|
* are found; name matches (O(categories)) stay cheap. Run off the main thread + debounced.
|
||||||
|
*/
|
||||||
|
fun search(query: String, limit: Int = 40): List<GeositeHit> {
|
||||||
|
val q = query.trim().lowercase()
|
||||||
|
if (q.isEmpty()) return emptyList()
|
||||||
|
val domainish = '.' in q
|
||||||
|
val byName = ArrayList<GeositeHit>()
|
||||||
|
val byDomain = ArrayList<GeositeHit>()
|
||||||
|
for ((cat, rules) in byCategory) {
|
||||||
|
if (cat.contains(q)) {
|
||||||
|
byName.add(GeositeHit(cat, rules.size, true))
|
||||||
|
} else if (domainish && byName.size + byDomain.size < limit && rules.any { covers(it, q) }) {
|
||||||
|
byDomain.add(GeositeHit(cat, rules.size, false))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
byName.sortBy { it.category }
|
||||||
|
byDomain.sortBy { it.category }
|
||||||
|
return (byName + byDomain).take(limit)
|
||||||
|
}
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
/** Empty index — returned when the asset is missing/corrupt so callers never crash. */
|
||||||
|
val EMPTY = GeositeIndex(emptyMap())
|
||||||
|
|
||||||
|
fun parse(bytes: ByteArray): GeositeIndex {
|
||||||
|
val r = ProtoReader(bytes)
|
||||||
|
val map = HashMap<String, List<DomainRule>>()
|
||||||
|
while (r.hasMore()) {
|
||||||
|
val tag = r.readTag()
|
||||||
|
if (tag.field == 1 && tag.wire == 2) parseSite(r.readLengthDelimited(), map)
|
||||||
|
else r.skip(tag)
|
||||||
|
}
|
||||||
|
return GeositeIndex(map)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun parseSite(bytes: ByteArray, out: MutableMap<String, List<DomainRule>>) {
|
||||||
|
val r = ProtoReader(bytes)
|
||||||
|
var name = ""
|
||||||
|
val rules = ArrayList<DomainRule>()
|
||||||
|
while (r.hasMore()) {
|
||||||
|
val tag = r.readTag()
|
||||||
|
when {
|
||||||
|
tag.field == 1 && tag.wire == 2 -> name = r.readString()
|
||||||
|
tag.field == 2 && tag.wire == 2 -> parseDomain(r.readLengthDelimited())?.let { rules.add(it) }
|
||||||
|
else -> r.skip(tag)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (name.isNotEmpty()) out[name.lowercase()] = rules
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun parseDomain(bytes: ByteArray): DomainRule? {
|
||||||
|
val r = ProtoReader(bytes)
|
||||||
|
var type = 2 // default RootDomain (suffix)
|
||||||
|
var value = ""
|
||||||
|
while (r.hasMore()) {
|
||||||
|
val tag = r.readTag()
|
||||||
|
when {
|
||||||
|
tag.field == 1 && tag.wire == 0 -> type = r.readVarint().toInt()
|
||||||
|
tag.field == 2 && tag.wire == 2 -> value = r.readString()
|
||||||
|
else -> r.skip(tag)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (value.isEmpty()) return null
|
||||||
|
return when (type) {
|
||||||
|
0 -> DomainRule.Keyword(value) // Plain
|
||||||
|
1 -> DomainRule.Regexp(value) // Regex
|
||||||
|
3 -> DomainRule.Exact(value) // Full
|
||||||
|
else -> DomainRule.Suffix(value) // RootDomain (2) and unknown
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Minimal protobuf wire reader (varint + length-delimited only; enough for geosite.dat).
|
||||||
|
* Bounds-checked: a truncated/corrupt asset throws a clean [IndexOutOfBoundsException] which
|
||||||
|
* [GeositeRepository][chat.vojo.proxy.data.GeositeRepository] turns into an empty index rather
|
||||||
|
* than an AIOOBE that would crash the search coroutine.
|
||||||
|
*/
|
||||||
|
private class ProtoReader(private val buf: ByteArray) {
|
||||||
|
private var pos = 0
|
||||||
|
fun hasMore() = pos < buf.size
|
||||||
|
|
||||||
|
fun readVarint(): Long {
|
||||||
|
var shift = 0
|
||||||
|
var result = 0L
|
||||||
|
while (true) {
|
||||||
|
if (pos >= buf.size || shift >= 64) throw IndexOutOfBoundsException("varint past end")
|
||||||
|
val b = buf[pos++].toInt() and 0xff
|
||||||
|
result = result or ((b and 0x7f).toLong() shl shift)
|
||||||
|
if (b < 0x80) return result
|
||||||
|
shift += 7
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun readTag(): Tag {
|
||||||
|
val t = readVarint().toInt()
|
||||||
|
return Tag(t ushr 3, t and 0x7)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun take(len: Int): Int {
|
||||||
|
if (len < 0 || pos + len > buf.size) throw IndexOutOfBoundsException("length-delimited past end")
|
||||||
|
val start = pos
|
||||||
|
pos += len
|
||||||
|
return start
|
||||||
|
}
|
||||||
|
|
||||||
|
fun readLengthDelimited(): ByteArray {
|
||||||
|
val len = readVarint().toInt()
|
||||||
|
val start = take(len)
|
||||||
|
return buf.copyOfRange(start, start + len)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun readString(): String {
|
||||||
|
val len = readVarint().toInt()
|
||||||
|
val start = take(len)
|
||||||
|
return String(buf, start, len, Charsets.UTF_8)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Skip an unexpected field by its wire type so parsing stays aligned. */
|
||||||
|
fun skip(tag: Tag) {
|
||||||
|
when (tag.wire) {
|
||||||
|
0 -> readVarint()
|
||||||
|
2 -> take(readVarint().toInt())
|
||||||
|
5 -> take(4)
|
||||||
|
1 -> take(8)
|
||||||
|
else -> pos = buf.size // unknown wire type — stop rather than spin
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private data class Tag(val field: Int, val wire: Int)
|
||||||
141
app/src/main/java/chat/vojo/proxy/core/route/SiteRouter.kt
Normal file
141
app/src/main/java/chat/vojo/proxy/core/route/SiteRouter.kt
Normal file
|
|
@ -0,0 +1,141 @@
|
||||||
|
package chat.vojo.proxy.core.route
|
||||||
|
|
||||||
|
import chat.vojo.proxy.core.RoutingMode
|
||||||
|
|
||||||
|
/** Prefix marking a [routedSites] entry as a bundled geosite category, e.g. `geosite:google`. */
|
||||||
|
const val GEOSITE_PREFIX = "geosite:"
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A single domain-matching rule, mirroring the v2fly domain-list-community rule kinds.
|
||||||
|
* Values are already lowercased ASCII (user domains via [normalizeDomain]; geosite
|
||||||
|
* values are ASCII at the source).
|
||||||
|
*/
|
||||||
|
sealed interface DomainRule {
|
||||||
|
val value: String
|
||||||
|
/** Matches the domain itself and any subdomain (v2fly `domain:` / RootDomain). */
|
||||||
|
data class Suffix(override val value: String) : DomainRule
|
||||||
|
/** Matches the exact domain only (v2fly `full:`). */
|
||||||
|
data class Exact(override val value: String) : DomainRule
|
||||||
|
/** Substring match anywhere in the host (v2fly `keyword:`). */
|
||||||
|
data class Keyword(override val value: String) : DomainRule
|
||||||
|
/** Regular-expression match (v2fly `regexp:`). Rare; partial (containsMatchIn) like RE2. */
|
||||||
|
data class Regexp(override val value: String) : DomainRule
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Resolves a geosite category name to its rules. Empty list for an unknown category. */
|
||||||
|
fun interface GeositeSource {
|
||||||
|
fun rules(category: String): List<DomainRule>
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
/** A source with no categories — used when the geosite asset is unavailable or in tests. */
|
||||||
|
val EMPTY = GeositeSource { emptyList() }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compiled domain matcher. Suffix/exact lookups are O(labels) hash probes; keyword and
|
||||||
|
* regexp are linear but rare (most geosite rules are suffix/exact). Built once per tunnel
|
||||||
|
* start and reused for every CONNECT, so matching never re-parses the rule set.
|
||||||
|
*/
|
||||||
|
class SiteMatcher private constructor(
|
||||||
|
private val suffixes: Set<String>,
|
||||||
|
private val exacts: Set<String>,
|
||||||
|
private val keywords: List<String>,
|
||||||
|
private val regexes: List<Regex>,
|
||||||
|
) {
|
||||||
|
/** True if [host] is selected by any rule. [host] is lowercased and de-dotted first. */
|
||||||
|
fun matches(host: String): Boolean {
|
||||||
|
if (isEmpty) return false
|
||||||
|
val h = host.lowercase().trim('.')
|
||||||
|
if (h.isEmpty()) return false
|
||||||
|
|
||||||
|
if (exacts.contains(h)) return true
|
||||||
|
// Walk progressive suffixes: a.b.example.com -> b.example.com -> example.com -> com.
|
||||||
|
var s = h
|
||||||
|
while (true) {
|
||||||
|
if (suffixes.contains(s)) return true
|
||||||
|
val dot = s.indexOf('.')
|
||||||
|
if (dot < 0) break
|
||||||
|
s = s.substring(dot + 1)
|
||||||
|
}
|
||||||
|
if (keywords.any { h.contains(it) }) return true
|
||||||
|
if (regexes.any { it.containsMatchIn(h) }) return true
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
val isEmpty: Boolean
|
||||||
|
get() = suffixes.isEmpty() && exacts.isEmpty() && keywords.isEmpty() && regexes.isEmpty()
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
val EMPTY = SiteMatcher(emptySet(), emptySet(), emptyList(), emptyList())
|
||||||
|
|
||||||
|
fun from(rules: Collection<DomainRule>): SiteMatcher {
|
||||||
|
if (rules.isEmpty()) return EMPTY
|
||||||
|
val suffixes = HashSet<String>()
|
||||||
|
val exacts = HashSet<String>()
|
||||||
|
val keywords = ArrayList<String>()
|
||||||
|
val regexes = ArrayList<Regex>()
|
||||||
|
for (r in rules) {
|
||||||
|
val v = r.value.lowercase().trim('.')
|
||||||
|
if (v.isEmpty()) continue
|
||||||
|
when (r) {
|
||||||
|
is DomainRule.Suffix -> suffixes.add(v)
|
||||||
|
is DomainRule.Exact -> exacts.add(v)
|
||||||
|
is DomainRule.Keyword -> keywords.add(v)
|
||||||
|
// Host is lowercased before matching, so compile case-insensitive — an
|
||||||
|
// upper-case char in a geosite regexp would otherwise never match.
|
||||||
|
is DomainRule.Regexp -> runCatching { Regex(r.value, RegexOption.IGNORE_CASE) }.getOrNull()?.let { regexes.add(it) }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return SiteMatcher(suffixes, exacts, keywords, regexes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Where a flow should go once its destination is known. */
|
||||||
|
enum class RouteDecision { PROXY, DIRECT }
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decides proxy-vs-direct for a tunnelled flow. [mode] is the global default and the
|
||||||
|
* matched sites are the exceptions, per the "Сайты" tab semantics:
|
||||||
|
* - [RoutingMode.PROXY_ALL]: everything proxied; matched sites go DIRECT.
|
||||||
|
* - [RoutingMode.DIRECT_ALL]: everything direct; matched sites go through the PROXY.
|
||||||
|
*
|
||||||
|
* Only domain destinations can match — IP-literal / IPv6 / non-DNS flows have no domain
|
||||||
|
* to test, so they always take the global default (the documented coverage gap).
|
||||||
|
*/
|
||||||
|
class SiteRouter(
|
||||||
|
val mode: RoutingMode,
|
||||||
|
private val matcher: SiteMatcher,
|
||||||
|
) {
|
||||||
|
fun decide(host: String, isDomain: Boolean): RouteDecision {
|
||||||
|
val matched = isDomain && matcher.matches(host)
|
||||||
|
return when (mode) {
|
||||||
|
RoutingMode.PROXY_ALL -> if (matched) RouteDecision.DIRECT else RouteDecision.PROXY
|
||||||
|
RoutingMode.DIRECT_ALL -> if (matched) RouteDecision.PROXY else RouteDecision.DIRECT
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Rule count drives whether the matcher has anything to do; for journal/diagnostics. */
|
||||||
|
val hasRules: Boolean get() = !matcher.isEmpty
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
/**
|
||||||
|
* Builds a router from persisted settings. Plain entries become suffix rules;
|
||||||
|
* `geosite:` entries expand via [geosite]. Invalid user entries are dropped.
|
||||||
|
*/
|
||||||
|
fun build(mode: RoutingMode, entries: Set<String>, geosite: GeositeSource): SiteRouter {
|
||||||
|
val rules = buildList {
|
||||||
|
for (entry in entries) {
|
||||||
|
val e = entry.trim()
|
||||||
|
if (e.startsWith(GEOSITE_PREFIX)) {
|
||||||
|
addAll(geosite.rules(e.removePrefix(GEOSITE_PREFIX).trim()))
|
||||||
|
} else {
|
||||||
|
normalizeDomain(e)?.let { add(DomainRule.Suffix(it)) }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return SiteRouter(mode, SiteMatcher.from(rules))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
44
app/src/main/java/chat/vojo/proxy/data/AssetGeositeSource.kt
Normal file
44
app/src/main/java/chat/vojo/proxy/data/AssetGeositeSource.kt
Normal file
|
|
@ -0,0 +1,44 @@
|
||||||
|
package chat.vojo.proxy.data
|
||||||
|
|
||||||
|
import android.content.Context
|
||||||
|
import chat.vojo.proxy.core.route.DomainRule
|
||||||
|
import chat.vojo.proxy.core.route.GeositeHit
|
||||||
|
import chat.vojo.proxy.core.route.GeositeIndex
|
||||||
|
import chat.vojo.proxy.core.route.GeositeSource
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Process-wide, lazily-parsed view of the bundled geosite database
|
||||||
|
* (`assets/geosite/geosite.dat`, MIT v2fly). The 2.2 MB protobuf is parsed once on first
|
||||||
|
* use (~hundreds of ms — call off the main thread) and cached for the process: both the
|
||||||
|
* UI search and the tunnel's [SiteRouter][chat.vojo.proxy.core.route.SiteRouter] build
|
||||||
|
* share the one index. The service and UI live in the same process, so this is one copy.
|
||||||
|
*/
|
||||||
|
object GeositeRepository {
|
||||||
|
@Volatile
|
||||||
|
private var index: GeositeIndex? = null
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the parsed index, building it from the asset on first call. A missing or corrupt
|
||||||
|
* asset yields [GeositeIndex.EMPTY] (cached, so we don't re-read/re-parse — and re-throw —
|
||||||
|
* on every keystroke), keeping both the tunnel build and the UI search crash-free.
|
||||||
|
*/
|
||||||
|
fun index(context: Context): GeositeIndex {
|
||||||
|
index?.let { return it }
|
||||||
|
return synchronized(this) {
|
||||||
|
index ?: runCatching {
|
||||||
|
val bytes = context.applicationContext.assets.open("geosite/geosite.dat")
|
||||||
|
.use { it.readBytes() }
|
||||||
|
GeositeIndex.parse(bytes)
|
||||||
|
}.getOrDefault(GeositeIndex.EMPTY).also { index = it }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun search(context: Context, query: String, limit: Int = 40): List<GeositeHit> =
|
||||||
|
index(context).search(query, limit)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** [GeositeSource] backed by [GeositeRepository]; resolves a `geosite:<category>` entry. */
|
||||||
|
class AssetGeositeSource(private val context: Context) : GeositeSource {
|
||||||
|
override fun rules(category: String): List<DomainRule> =
|
||||||
|
GeositeRepository.index(context).rules(category)
|
||||||
|
}
|
||||||
205
app/src/test/java/chat/vojo/proxy/core/route/SiteRoutingTest.kt
Normal file
205
app/src/test/java/chat/vojo/proxy/core/route/SiteRoutingTest.kt
Normal file
|
|
@ -0,0 +1,205 @@
|
||||||
|
package chat.vojo.proxy.core.route
|
||||||
|
|
||||||
|
import chat.vojo.proxy.core.RoutingMode
|
||||||
|
import org.junit.Assert.assertEquals
|
||||||
|
import org.junit.Assert.assertFalse
|
||||||
|
import org.junit.Assert.assertNull
|
||||||
|
import org.junit.Assert.assertTrue
|
||||||
|
import org.junit.Test
|
||||||
|
|
||||||
|
class SiteRoutingTest {
|
||||||
|
|
||||||
|
// --- normalizeDomain ---
|
||||||
|
|
||||||
|
@Test fun normalize_plainDomain() {
|
||||||
|
assertEquals("example.com", normalizeDomain("example.com"))
|
||||||
|
assertEquals("example.com", normalizeDomain(" Example.COM "))
|
||||||
|
assertEquals("example.com", normalizeDomain("example.com."))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test fun normalize_stripsUrlParts() {
|
||||||
|
assertEquals("example.com", normalizeDomain("https://example.com/path?q=1#frag"))
|
||||||
|
assertEquals("example.com", normalizeDomain("http://user:pw@example.com:8443/x"))
|
||||||
|
assertEquals("sub.example.com", normalizeDomain("HTTPS://Sub.Example.com:443"))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test fun normalize_idnToPunycode() {
|
||||||
|
// пример.рф and a German umlaut domain both punycode-encode.
|
||||||
|
assertEquals("xn--e1afmkfd.xn--p1ai", normalizeDomain("пример.рф"))
|
||||||
|
assertEquals("xn--mnchen-3ya.de", normalizeDomain("münchen.de"))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test fun normalize_rejectsJunk() {
|
||||||
|
assertNull(normalizeDomain(""))
|
||||||
|
assertNull(normalizeDomain(" "))
|
||||||
|
assertNull(normalizeDomain("localhost")) // single label = whole TLD-ish, rejected
|
||||||
|
assertNull(normalizeDomain("com"))
|
||||||
|
assertNull(normalizeDomain("a b.com")) // space is not a valid label char
|
||||||
|
assertNull(normalizeDomain("-bad.com")) // label can't start with '-'
|
||||||
|
assertNull(normalizeDomain("bad-.com"))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test fun normalize_rejectsIpLiterals() {
|
||||||
|
// IPs can never match (IP destinations bypass the matcher), so reject them at entry.
|
||||||
|
assertNull(normalizeDomain("8.8.8.8"))
|
||||||
|
assertNull(normalizeDomain("192.168.0.1"))
|
||||||
|
assertNull(normalizeDomain("1.2.3.4.5"))
|
||||||
|
assertNull(normalizeDomain("2001:db8::1")) // IPv6 already rejected by ':' handling
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- SiteMatcher ---
|
||||||
|
|
||||||
|
private fun matcher(vararg domains: String) =
|
||||||
|
SiteMatcher.from(domains.map { DomainRule.Suffix(it) })
|
||||||
|
|
||||||
|
@Test fun suffix_matchesSelfAndSubdomains() {
|
||||||
|
val m = matcher("example.com")
|
||||||
|
assertTrue(m.matches("example.com"))
|
||||||
|
assertTrue(m.matches("a.example.com"))
|
||||||
|
assertTrue(m.matches("a.b.example.com"))
|
||||||
|
assertTrue(m.matches("EXAMPLE.com")) // case-insensitive
|
||||||
|
assertTrue(m.matches("example.com.")) // trailing dot tolerated
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test fun suffix_doesNotOvermatch() {
|
||||||
|
val m = matcher("example.com")
|
||||||
|
assertFalse(m.matches("notexample.com")) // not a label boundary
|
||||||
|
assertFalse(m.matches("example.com.evil.com"))
|
||||||
|
assertFalse(m.matches("example.org"))
|
||||||
|
assertFalse(m.matches("com"))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test fun exact_keyword_regexp() {
|
||||||
|
val m = SiteMatcher.from(
|
||||||
|
listOf(
|
||||||
|
DomainRule.Exact("only.example.com"),
|
||||||
|
DomainRule.Keyword("doubleclick"),
|
||||||
|
DomainRule.Regexp("""^ads[0-9]+\.net$"""),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assertTrue(m.matches("only.example.com"))
|
||||||
|
assertFalse(m.matches("sub.only.example.com")) // exact: no subdomains
|
||||||
|
assertTrue(m.matches("stats.g.doubleclick.net"))
|
||||||
|
assertTrue(m.matches("ads42.net"))
|
||||||
|
assertFalse(m.matches("ads.net"))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test fun emptyMatcher_matchesNothing() {
|
||||||
|
assertTrue(SiteMatcher.EMPTY.isEmpty)
|
||||||
|
assertFalse(SiteMatcher.EMPTY.matches("example.com"))
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- decideRoute truth table ---
|
||||||
|
|
||||||
|
@Test fun proxyAll_listedGoDirect_restProxy() {
|
||||||
|
val r = SiteRouter(RoutingMode.PROXY_ALL, matcher("example.com"))
|
||||||
|
assertEquals(RouteDecision.DIRECT, r.decide("a.example.com", isDomain = true))
|
||||||
|
assertEquals(RouteDecision.PROXY, r.decide("other.com", isDomain = true))
|
||||||
|
// No domain (IP literal / IPv6) -> global default = PROXY.
|
||||||
|
assertEquals(RouteDecision.PROXY, r.decide("93.184.216.34", isDomain = false))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test fun directAll_listedGoProxy_restDirect() {
|
||||||
|
val r = SiteRouter(RoutingMode.DIRECT_ALL, matcher("example.com"))
|
||||||
|
assertEquals(RouteDecision.PROXY, r.decide("a.example.com", isDomain = true))
|
||||||
|
assertEquals(RouteDecision.DIRECT, r.decide("other.com", isDomain = true))
|
||||||
|
// No domain -> global default = DIRECT.
|
||||||
|
assertEquals(RouteDecision.DIRECT, r.decide("93.184.216.34", isDomain = false))
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- SiteRouter.build (entries -> rules) ---
|
||||||
|
|
||||||
|
@Test fun build_mixesPlainAndGeositeEntries() {
|
||||||
|
val geosite = GeositeSource { cat ->
|
||||||
|
if (cat == "google") listOf(DomainRule.Suffix("google.com"), DomainRule.Suffix("youtube.com"))
|
||||||
|
else emptyList()
|
||||||
|
}
|
||||||
|
val r = SiteRouter.build(
|
||||||
|
RoutingMode.PROXY_ALL,
|
||||||
|
setOf("My.Site.org", "geosite:google", "garbage entry", "geosite:unknown"),
|
||||||
|
geosite,
|
||||||
|
)
|
||||||
|
assertTrue(r.hasRules)
|
||||||
|
assertEquals(RouteDecision.DIRECT, r.decide("my.site.org", isDomain = true))
|
||||||
|
assertEquals(RouteDecision.DIRECT, r.decide("www.google.com", isDomain = true))
|
||||||
|
assertEquals(RouteDecision.DIRECT, r.decide("m.youtube.com", isDomain = true))
|
||||||
|
assertEquals(RouteDecision.PROXY, r.decide("example.net", isDomain = true))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test fun build_emptyEntries_inert() {
|
||||||
|
val r = SiteRouter.build(RoutingMode.PROXY_ALL, emptySet(), GeositeSource.EMPTY)
|
||||||
|
assertFalse(r.hasRules)
|
||||||
|
assertEquals(RouteDecision.PROXY, r.decide("anything.com", isDomain = true))
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- geosite.dat protobuf parsing + search ---
|
||||||
|
|
||||||
|
// Minimal protobuf encoders to build a synthetic GeoSiteList for the parser.
|
||||||
|
private fun varint(n: Int): ByteArray {
|
||||||
|
var x = n
|
||||||
|
val o = ArrayList<Byte>()
|
||||||
|
while (true) {
|
||||||
|
val b = x and 0x7f
|
||||||
|
x = x ushr 7
|
||||||
|
if (x != 0) o.add((b or 0x80).toByte()) else { o.add(b.toByte()); break }
|
||||||
|
}
|
||||||
|
return o.toByteArray()
|
||||||
|
}
|
||||||
|
private fun tag(field: Int, wire: Int) = varint((field shl 3) or wire)
|
||||||
|
private fun ld(field: Int, data: ByteArray) = tag(field, 2) + varint(data.size) + data
|
||||||
|
private fun vfield(field: Int, v: Int) = tag(field, 0) + varint(v)
|
||||||
|
private fun str(field: Int, s: String) = ld(field, s.toByteArray(Charsets.UTF_8))
|
||||||
|
|
||||||
|
private fun sampleDat(): ByteArray {
|
||||||
|
val dom1 = vfield(1, 2) + str(2, "example.com") // RootDomain -> suffix
|
||||||
|
val dom2 = vfield(1, 3) + str(2, "only.exact.com") // Full -> exact
|
||||||
|
val dom3 = vfield(1, 0) + str(2, "trackword") // Plain -> keyword
|
||||||
|
val site = str(1, "TESTCAT") + ld(2, dom1) + ld(2, dom2) + ld(2, dom3)
|
||||||
|
return ld(1, site)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test fun geositeIndex_parsesCategoriesAndRules() {
|
||||||
|
val idx = GeositeIndex.parse(sampleDat())
|
||||||
|
assertTrue("testcat" in idx.categories)
|
||||||
|
val m = SiteMatcher.from(idx.rules("TESTCAT")) // lookup is case-insensitive
|
||||||
|
assertTrue(m.matches("a.example.com")) // suffix
|
||||||
|
assertTrue(m.matches("example.com"))
|
||||||
|
assertTrue(m.matches("only.exact.com")) // exact
|
||||||
|
assertFalse(m.matches("sub.only.exact.com")) // exact: no subdomains
|
||||||
|
assertTrue(m.matches("xtrackwordy.net")) // keyword
|
||||||
|
assertFalse(m.matches("unrelated.org"))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test fun geositeIndex_searchByNameAndDomain() {
|
||||||
|
val idx = GeositeIndex.parse(sampleDat())
|
||||||
|
val byName = idx.search("testc")
|
||||||
|
assertEquals(1, byName.size)
|
||||||
|
assertEquals("testcat", byName[0].category)
|
||||||
|
assertTrue(byName[0].matchedByName)
|
||||||
|
// A domain query finds the category that covers it.
|
||||||
|
val byDomain = idx.search("www.example.com")
|
||||||
|
assertEquals(listOf("testcat"), byDomain.map { it.category })
|
||||||
|
assertFalse(byDomain[0].matchedByName)
|
||||||
|
assertTrue(idx.search("nothingmatches").isEmpty())
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test fun geositeIndex_routerResolvesCategory() {
|
||||||
|
val idx = GeositeIndex.parse(sampleDat())
|
||||||
|
val r = SiteRouter.build(RoutingMode.PROXY_ALL, setOf("geosite:testcat"), idx)
|
||||||
|
assertTrue(r.hasRules)
|
||||||
|
assertEquals(RouteDecision.DIRECT, r.decide("a.example.com", isDomain = true))
|
||||||
|
assertEquals(RouteDecision.PROXY, r.decide("other.net", isDomain = true))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout = 5000) fun geositeIndex_truncatedInputIsBounded() {
|
||||||
|
// Every truncation of a valid blob must finish with either a partial index or a bounded
|
||||||
|
// exception (mirrors GeositeRepository's runCatching -> EMPTY) — never an infinite loop
|
||||||
|
// or an uncaught AIOOBE. The test timeout guards against a spin.
|
||||||
|
val full = sampleDat()
|
||||||
|
for (cut in 0..full.size) {
|
||||||
|
val idx = runCatching { GeositeIndex.parse(full.copyOfRange(0, cut)) }.getOrDefault(GeositeIndex.EMPTY)
|
||||||
|
idx.search("test") // must not throw
|
||||||
|
idx.rules("testcat") // must not throw
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
15
tools/fetch-geosite.sh
Normal file
15
tools/fetch-geosite.sh
Normal file
|
|
@ -0,0 +1,15 @@
|
||||||
|
#!/bin/bash
|
||||||
|
# Refresh the bundled geosite database from the MIT-licensed v2fly release.
|
||||||
|
# dlc.dat == geosite.dat: a GeoSiteList protobuf (category -> domain rules), parsed
|
||||||
|
# on-device by core/route/GeositeIndex. ~2.2 MB raw; APK-compressed to a few hundred KB.
|
||||||
|
# License notice is vendored alongside at app/src/main/assets/geosite/LICENSE.
|
||||||
|
set -euo pipefail
|
||||||
|
DIR="$(cd "$(dirname "$0")/.." && pwd)/app/src/main/assets/geosite"
|
||||||
|
mkdir -p "$DIR"
|
||||||
|
curl -fsSL --max-time 60 \
|
||||||
|
"https://github.com/v2fly/domain-list-community/releases/latest/download/dlc.dat" \
|
||||||
|
-o "$DIR/geosite.dat"
|
||||||
|
curl -fsSL --max-time 30 \
|
||||||
|
"https://raw.githubusercontent.com/v2fly/domain-list-community/master/LICENSE" \
|
||||||
|
-o "$DIR/LICENSE"
|
||||||
|
echo "geosite.dat: $(wc -c < "$DIR/geosite.dat") bytes"
|
||||||
Loading…
Add table
Reference in a new issue