urx

# https://crates.io/crates/urx
cargo install urx

# https://formulae.brew.sh/formula/urx
brew install urx

git clone https://github.com/hahwul/urx.git
cd urx
cargo build --release

# Escanear un solo dominio
urx example.com

# Escanear múltiples dominios
urx example.com example.org

# Escanear dominios desde un archivo
cat domains.txt | urx

Usage: urx [OPTIONS] [DOMAINS]...

Arguments:
  [DOMAINS]...  Domains to fetch URLs for

Options:
  -c, --config <CONFIG>           Config file to load
      --provider-config <PATH>    Separate provider config file holding only API keys (default: $XDG_CONFIG_HOME/urx/provider-config.toml). CLI/env > provider-config > main config.
  -h, --help             Print help
  -V, --version          Print version

Input Options:
      --files <FILES>...        Read URLs directly from files (supports WARC, URLTeam compressed, and text files)
      --domain-list <PATH>      File of newline-separated domains to scan (repeatable; merged with positional DOMAINS and stdin; `#` comments allowed)

Output Options:
  -o, --output <OUTPUT>          Output file to write results
      --output-dir <PATH>        Write one file per domain into this directory (extension matches --format). Coexists with --output / stdout.
  -f, --format <FORMAT>          Output format (e.g., "plain", "json", "csv") [default: plain]
      --merge-endpoint   Merge endpoints with the same path and merge URL parameters
      --normalize-url    Normalize URLs for better deduplication (sorts query parameters, removes trailing slashes)

Provider Options:
      --providers <PROVIDERS>
          Providers to use (comma-separated, e.g., "wayback,cc,otx,arquivo,vt,urlscan") [default: wayback,cc,otx]
      --exclude-providers <EXCLUDE_PROVIDERS>
          Providers to exclude (comma-separated). Wins on conflict with --providers / --all-providers.
      --all-providers
          Enable every supported provider. API-keyed providers only activate when a key is available.
      --list-providers
          List every supported provider then exit.
      --subs
          Include subdomains when searching
      --cc-index <CC_INDEX>
          Common Crawl index to use; accepts comma-separated list to query multiple indexes in parallel (e.g. `CC-MAIN-2026-17,CC-MAIN-2025-51`). `latest` (the default) resolves the newest via collinfo.json. [default: latest]
      --wayback-from <DATE>
          Restrict Wayback Machine results to snapshots at or after DATE (YYYY/YYYYMM/YYYYMMDD/YYYYMMDDhhmmss)
      --wayback-to <DATE>
          Restrict Wayback Machine results to snapshots at or before DATE (same format as --wayback-from)
      --vt-api-key <VT_API_KEY>
          API key for VirusTotal (can be used multiple times for rotation, can also use URX_VT_API_KEY environment variable with comma-separated keys)
      --urlscan-api-key <URLSCAN_API_KEY>
          Optional API key for Urlscan; the provider also works anonymously (rate-limited ~30 req/min per IP). Can be used multiple times for rotation, or via URX_URLSCAN_API_KEY (comma-separated keys)
      --github-api-key <GITHUB_API_KEY>
          Personal access token for the GitHub Code Search provider (also reads URX_GITHUB_API_KEY, comma-separated for rotation)

Discovery Options:
      --exclude-robots   Exclude robots.txt discovery
      --exclude-sitemap  Exclude sitemap.xml discovery

Display Options:
  -v, --verbose       Show verbose output
      --silent        Silent mode (no output)
      --no-progress   No progress bar
      --show-sources  Annotate output URLs with the providers that returned them
      --stats         Print a per-provider summary to stderr at end of run

Filter Options:
  -p, --preset <PRESET>
          Filter Presets (e.g., "no-resources,no-images,no-audio,only-js,only-style")
  -e, --extensions <EXTENSIONS>
          Filter URLs to only include those with specific extensions (comma-separated, e.g., "js,php,aspx")
      --exclude-extensions <EXCLUDE_EXTENSIONS>
          Filter URLs to exclude those with specific extensions (comma-separated, e.g., "html,txt")
      --patterns <PATTERNS>
          Filter URLs to only include those containing specific patterns (comma-separated)
      --exclude-patterns <EXCLUDE_PATTERNS>
          Filter URLs to exclude those containing specific patterns (comma-separated)
      --show-only-host
          Only show the host part of the URLs
      --show-only-path
          Only show the path part of the URLs
      --show-only-param
          Only show the parameters part of the URLs
      --min-length <MIN_LENGTH>
          Minimum URL length to include
      --max-length <MAX_LENGTH>
          Maximum URL length to include
      --strict
          Enforce exact host validation (default)

Network Options:
      --network-scope <NETWORK_SCOPE>  Control which components network settings apply to (all, providers, testers, or providers,testers) [default: all]
      --proxy <PROXY>                  Use proxy for HTTP requests (format: <http://proxy.example.com:8080>)
      --proxy-auth <PROXY_AUTH>        Proxy authentication credentials (format: username:password)
      --insecure                       Skip SSL certificate verification (accept self-signed certs)
      --random-agent                   Use a random User-Agent for HTTP requests
      --timeout <TIMEOUT>              Request timeout in seconds [default: 120]
      --retries <RETRIES>              Number of retries for failed requests [default: 2]
      --parallel <PARALLEL>            Maximum domains fetched concurrently per provider (and concurrent URL tests); a provider's --rate-limit is shared across them [default: 5]
      --rate-limit <RATE_LIMIT>        Rate limit (requests per second)
      --rate-limit-by <PAIRS>          Per-provider rate overrides (e.g. `vt=1,wayback=10`); falls back to --rate-limit for unlisted providers
      --max-time <MAX_TIME>            Global ceiling on provider enumeration time in seconds (0 = unlimited) [default: 0]

Testing Options:
      --check-status
          Check HTTP status code of collected URLs [aliases: ----cs]
      --include-status <INCLUDE_STATUS>
          Include URLs with specific HTTP status codes or patterns (e.g., --is=200,30x) [aliases: ----is]
      --exclude-status <EXCLUDE_STATUS>
          Exclude URLs with specific HTTP status codes or patterns (e.g., --es=404,50x,5xx) [aliases: ----es]
      --extract-links
          Extract additional links from collected URLs (requires HTTP requests)

# Guardar resultados en un archivo
urx example.com -o resultados.txt

# Salida en formato JSON
urx example.com -f json -o resultados.json

# Filtrar solo archivos JavaScript
urx example.com -e js

# Excluir archivos HTML y de texto
urx example.com --exclude-extensions html,txt

# Filtrar endpoints de API
urx example.com --patterns api,v1,graphql

# Excluir patrones específicos
urx example.com --exclude-patterns static,images

# Usar ajuste predefinido (similar a --exclude-extensions=png,jpg,.....)
urx example.com -p no-images

# Usar proveedores específicos
urx example.com --providers wayback,otx

# Añadir el proveedor Arquivo.pt (archivo web portugués) sin clave
urx example.com --providers wayback,cc,otx,arquivo

# URLScan funciona sin clave (anónimo, con límite de velocidad); una clave solo aumenta los límites
urx example.com --providers urlscan

# Usar proveedores VirusTotal y URLScan
# 1. Añadir explícitamente a providers (con claves API por línea de comandos)
urx example.com --providers=vt,urlscan --vt-api-key=*** --urlscan-api-key=***

# 2. Usar variables de entorno para las claves API
URX_VT_API_KEY=*** URX_URLSCAN_API_KEY=*** urx example.com --providers=vt,urlscan

# 3. Autoactivación: los proveedores se añaden automáticamente cuando se proporcionan claves API
urx example.com --vt-api-key=*** --urlscan-api-key=*** # No es necesario especificar en --providers

# 4. Rotación de múltiples claves API (para mitigar límites de velocidad)
# Usar banderas repetidas para varias claves
urx example.com --vt-api-key=clave1 --vt-api-key=clave2 --vt-api-key=clave3

# Usar variables de entorno con claves separadas por comas
URX_VT_API_KEY=clave1,clave2,clave3 URX_URLSCAN_API_KEY=uclave1,uclave2 urx example.com

# Combinar banderas CLI y variables de entorno (las claves CLI se usan primero)
URX_VT_API_KEY=env_clave1,env_clave2 urx example.com --vt-api-key=cli_clave1 --vt-api-key=cli_clave2

# Las URLs de robots.txt y sitemap.xml se incluyen por defecto

# Excluir URLs de archivos robots.txt
urx example.com --exclude-robots

# Excluir URLs de sitemap
urx example.com --exclude-sitemap

# Incluir subdominios
urx example.com --subs

# Comprobar estado de las URLs recolectadas
urx example.com --check-status

# Leer URLs directamente desde un archivo de texto
urx --files urls.txt

# Combinar entrada de archivo con filtrado
urx --files urls.txt --patterns api,admin -f json

# Extraer enlaces adicionales de las URLs recolectadas
urx example.com --extract-links

# Configuración de red
urx example.com --proxy http://localhost:8080 --timeout 60 --parallel 10 --insecure

# Filtrado avanzado
urx example.com -e js,php --patterns admin,login --exclude-patterns logout,static --min-length 20

# Filtrado basado en código de estado HTTP
urx example.com --include-status 200,30x,405 --exclude-status 20x

# Desactivar validación de host
urx example.com --strict false

# Normalización y deduplicación de URLs
# Normalizar URLs ordenando parámetros de consulta y eliminando barras inclinadas al final
urx example.com --normalize-url

# Combinar normalización con fusión de endpoints para deduplicación completa
urx example.com --normalize-url --merge-endpoint

# Normalización de URLs con entrada de archivo
urx --files urls.txt --normalize-url

# Habilitar caché con SQLite (por defecto)
urx example.com --cache-type sqlite --cache-path ~/.urx/cache.db

# Usar Redis para caché distribuida
urx example.com --cache-type redis --redis-url redis://localhost:6379

# Escaneo incremental - solo mostrar URLs nuevas desde el último escaneo
urx example.com --incremental

# Establecer TTL de caché (tiempo de vida) a 12 horas
urx example.com --cache-ttl 43200

# Deshabilitar caché por completo
urx example.com --no-cache

# Combinar escaneo incremental con filtros
urx example.com --incremental -e js,php --patterns api

# Archivo de configuración con ajustes de caché
urx -c example/config.toml example.com

# Monitoreo diario - solo alertar sobre URLs nuevas
urx target.com --incremental --silent | notify-tool

# Procesamiento eficiente de listas de dominios
cat domains.txt | urx --incremental --cache-ttl 3600 > nuevas_urls.txt

# Escaneo en equipo distribuido con Redis
urx example.com --cache-type redis --redis-url redis://shared-cache:6379

# Re-escaneos rápidos durante el desarrollo
urx test-domain.com --cache-ttl 300  # Caché de 5 minutos para iteraciones rápidas

# Encontrar dominios, luego descubrir URLs
echo "example.com" | urx | grep "login" > objetivos_potenciales.txt

# Combinar con otras herramientas
cat domains.txt | urx --patterns api | otra-herramienta