Skip to content

Commit

Permalink
feat(browse): add useragent generator #60
Browse files Browse the repository at this point in the history
  • Loading branch information
phodal committed Aug 26, 2024
1 parent 496cc20 commit df595f4
Show file tree
Hide file tree
Showing 8 changed files with 827 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package com.phodal.shirecore.agent.agenttool.browse
import com.phodal.shirecore.agent.agenttool.AgentToolContext
import com.phodal.shirecore.provider.agent.AgentTool
import com.phodal.shirecore.agent.agenttool.AgentToolResult
import com.phodal.shirecore.agent.agenttool.ua.RandomUserAgent
import org.jsoup.Jsoup
import org.jsoup.nodes.Document

Expand All @@ -24,7 +25,12 @@ class BrowseTool : AgentTool {
* Intellij API: [com.intellij.inspectopedia.extractor.utils.HtmlUtils.cleanupHtml]
*/
fun parse(url: String): DocumentContent {
val doc: Document = Jsoup.connect(url).get()
val doc: Document = Jsoup.connect(url)
.ignoreContentType(true)
.userAgent(RandomUserAgent.random())
.followRedirects(true)
.get()

return DocumentCleaner().cleanHtml(doc)
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
/**
* This is free and unencumbered software released into the public domain.
* GitHub: https://github.com/jonaskahn/user-agents
*/
package com.phodal.shirecore.agent.agenttool.ua

enum class BrowserType {
CHROME,
FIREFOX,
SAFARI;

companion object {
fun all(): List<BrowserType> {
return arrayListOf(CHROME, FIREFOX, SAFARI)
}

fun cross(): List<BrowserType> {
return arrayListOf(CHROME, FIREFOX)
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
/**
* This is free and unencumbered software released into the public domain.
* GitHub: https://github.com/jonaskahn/user-agents
*/
package com.phodal.shirecore.agent.agenttool.ua

enum class DeviceType {
MACOS,
LINUX,
WINDOWS,
IOS,
ANDROID;

companion object {
fun mobile(): List<DeviceType> {
return arrayListOf(IOS, ANDROID)
}

fun desktop(): List<DeviceType> {
return arrayListOf(MACOS, LINUX, WINDOWS)
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/**
* This is free and unencumbered software released into the public domain.
* GitHub: https://github.com/jonaskahn/user-agents
*/
package com.phodal.shirecore.agent.agenttool.ua

object Pattern {
const val WINDOWS_CHROME_AGENT = "Mozilla/5.0 (%s) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36"
const val WINDOWS_FIREFOX_AGENT = "Mozilla/5.0 (%s; rv:%s) Gecko/20100101 Firefox/%s"

const val LINUX_CHROME_AGENT = WINDOWS_CHROME_AGENT
const val LINUX_FIREFOX_AGENT = WINDOWS_FIREFOX_AGENT

const val MACOS_FIREFOX_AGENT = WINDOWS_FIREFOX_AGENT
const val MACOS_CHROME_AGENT = WINDOWS_CHROME_AGENT
const val MACOS_SAFARI_AGENT = "Mozilla/5.0 (%s) AppleWebKit/%s (KHTML, like Gecko) Version/%s Safari/%s"

const val IOS_FIREFOX_AGENT =
"Mozilla/5.0 (%s) AppleWebKit/%s (KHTML, like Gecko) FxiOS/%s Mobile/%s Safari/%s"
const val IOS_CHROME_AGENT =
"Mozilla/5.0 (%s) AppleWebKit/%s (KHTML, like Gecko) CriOS/%s Mobile/%s Safari/%s"
const val IOS_SAFARI_AGENT =
"Mozilla/5.0 (%s) AppleWebKit/%s (KHTML, like Gecko) Version/%s Mobile/%s Safari/%s"

const val ANDROID_FIREFOX_AGENT =
"Mozilla/5.0 (%s; rv:%s) Gecko/%s Firefox/%s"
const val ANDROID_CHROME_AGENT =
"Mozilla/5.0 (%s) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Mobile Safari/537.36"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,230 @@
/**
* This is free and unencumbered software released into the public domain.
* GitHub: https://github.com/jonaskahn/user-agents
*/
package com.phodal.shirecore.agent.agenttool.ua

import java.util.*
import java.util.concurrent.ThreadLocalRandom

object RandomUserAgent {

private val random = ThreadLocalRandom.current()

/**
* Random a user-agent, result can be a {@link #desktop(DeviceType, BrowserType) desktop}
* or {@link #mobile(DeviceType, BrowserType) mobile} user-agent type
*
*/
fun random(): String {
return if (random.nextBoolean()) desktop() else mobile()
}

/**
* Generate a desktop(MacOS, Linux, Windows) user-agent
* @param deviceType DeviceType must be a valid type or null. If device type is mobile, an exception will be thrown.
* @param browserType BrowserType must be a valid type or null.
* @return {@link String}
* @see one.ifelse.tools.useragent.types.DeviceType
* @see one.ifelse.tools.useragent.types.BrowserType
*/
fun desktop(
deviceType: DeviceType? = null,
browserType: BrowserType? = null
): String {
val device = deviceType ?: DeviceType.desktop().random()
return when (device) {
DeviceType.MACOS -> generateMacOS(browserType)
DeviceType.LINUX -> generateLinux(browserType)
DeviceType.WINDOWS -> generateWindowsAgent(browserType)
DeviceType.IOS -> throw UserAgentException("Require desktop device type like: ${DeviceType.desktop()}")
DeviceType.ANDROID -> throw UserAgentException("Require desktop device type like: ${DeviceType.desktop()}")
}
}

private fun generateWindowsAgent(browserType: BrowserType?): String {
val windowsVersion = Seeds.WINDOWS_DEVICES.random()
val type = browserType ?: BrowserType.cross().random()
return when (type) {
BrowserType.CHROME -> {
String.format(
Pattern.WINDOWS_CHROME_AGENT,
windowsVersion,
Seeds.CHROME_VERSIONS.random()
)
}

BrowserType.FIREFOX -> {
val firefoxVersion = Seeds.FIREFOX_VERSIONS.random()
String.format(
Pattern.WINDOWS_FIREFOX_AGENT,
windowsVersion,
firefoxVersion,
firefoxVersion
)
}

BrowserType.SAFARI -> {
throw UserAgentException("Cannot generate user-gent for Windows and Safari")
}
}
}

private fun generateLinux(browserType: BrowserType?): String {
val linuxVersion = Seeds.LINUX_DEVICES.random()
val type = browserType ?: BrowserType.cross().random()
return when (type) {
BrowserType.CHROME -> {
String.format(
Pattern.LINUX_CHROME_AGENT,
linuxVersion,
Seeds.CHROME_VERSIONS.random()
)
}

BrowserType.FIREFOX -> {
val firefoxVersion = Seeds.FIREFOX_VERSIONS.random()
return String.format(
Pattern.LINUX_FIREFOX_AGENT,
linuxVersion,
firefoxVersion,
firefoxVersion
)
}

BrowserType.SAFARI -> {
throw UserAgentException("Cannot generate user-gent for Linux and Safari")
}
}
}

private fun generateMacOS(browserType: BrowserType?): String {
val type = browserType ?: BrowserType.all().random()
val arch = Seeds.MAC_ARCHS.random()
val version = Seeds.MAC_VERSIONS.random()
val device = "$arch $version"
return when (type) {
BrowserType.CHROME -> {
String.format(
Pattern.MACOS_CHROME_AGENT,
device,
Seeds.CHROME_VERSIONS.random(),
)
}

BrowserType.FIREFOX -> {
val firefoxVersion = Seeds.FIREFOX_VERSIONS.random()
return String.format(
Pattern.MACOS_FIREFOX_AGENT,
device,
firefoxVersion,
firefoxVersion
)
}

BrowserType.SAFARI -> {
val safariVersion = Seeds.SAFARI_VERSIONS.random()
return String.format(
Pattern.MACOS_SAFARI_AGENT,
device,
safariVersion,
version.replace("_", "."),
safariVersion
)
}
}
}


/**
* Generate a mobile(IOS, Android) user-agent
* @param deviceType DeviceType must be a valid type or null. If device type is desktop, an exception will be thrown.
* @param browserType BrowserType must be a valid type or null.
* @return {@link String}
* @see one.ifelse.tools.useragent.types.DeviceType
* @see one.ifelse.tools.useragent.types.BrowserType
*/
fun mobile(
deviceType: DeviceType? = null,
browserType: BrowserType? = null
): String {
val device = deviceType ?: DeviceType.mobile().random()
return when (device) {
DeviceType.MACOS -> throw UserAgentException("Require mobile device type like: ${DeviceType.mobile()}")
DeviceType.LINUX -> throw UserAgentException("Require mobile device type like: ${DeviceType.mobile()}")
DeviceType.WINDOWS -> throw UserAgentException("Require mobile device type like: ${DeviceType.mobile()}")
DeviceType.IOS -> generateIOSAgent(browserType)
DeviceType.ANDROID -> generateAndroidAgent(browserType)
}
}

private fun generateIOSAgent(browserType: BrowserType?): String {
val type = browserType ?: BrowserType.all().random()
val version = Seeds.IOS_VERSIONS.entries.random()
val arch = Seeds.IOS_ARCHS.random()
val device = String.format(arch, version.key)
val safariVersion = Seeds.SAFARI_VERSIONS.random()
return when (type) {
BrowserType.CHROME -> {
String.format(
Pattern.IOS_CHROME_AGENT,
device,
safariVersion,
Seeds.CHROME_VERSIONS.random(),
version.value,
safariVersion
)
}

BrowserType.FIREFOX -> {
String.format(
Pattern.IOS_FIREFOX_AGENT,
device,
safariVersion,
Seeds.FIREFOX_VERSIONS.random(),
version.value,
safariVersion
)
}

BrowserType.SAFARI -> {
String.format(
Pattern.IOS_SAFARI_AGENT,
device,
safariVersion,
safariVersion,
version.value,
safariVersion
)
}
}
}

private fun generateAndroidAgent(browserType: BrowserType?): String {
val type = browserType ?: BrowserType.cross().random()
val device = Seeds.ANDROID_DEVICES.random()
return when (type) {
BrowserType.CHROME -> {
val chromeVersion = Seeds.CHROME_VERSIONS.random()
String.format(
Pattern.ANDROID_CHROME_AGENT,
device,
chromeVersion,
)
}

BrowserType.FIREFOX -> {
val firefoxVersion = Seeds.FIREFOX_VERSIONS.random()
String.format(
Pattern.ANDROID_FIREFOX_AGENT,
device,
firefoxVersion,
firefoxVersion,
firefoxVersion
)
}

BrowserType.SAFARI -> throw UserAgentException("Cannot generate user-gent for Android and Safari")
}
}
}
Loading

0 comments on commit df595f4

Please sign in to comment.