pip install wxpath
# لدعم TUI:
pip install "wxpath[tui]"
# تشغيل TUI فورًا عبر uv:
uvx --from "wxpath[tui]" wxpath-tui

import wxpath

expr = "url('https://quotes.toscrape.com')//a/@href"

for link in wxpath.wxpath_async_blocking_iter(expr):
    print(link)

import wxpath

path_expr = """
url('https://quotes.toscrape.com')
  ///url(//a/@href)
    //a/@href
"""

for item in wxpath.wxpath_async_blocking_iter(path_expr, max_depth=1):
    print(item)

import wxpath
from wxpath.settings import CRAWLER_SETTINGS

# رؤوس مخصصة للتهذيب؛ ضرورية لبعض المواقع (مثل ويكيبيديا)
CRAWLER_SETTINGS.headers = {'User-Agent': 'my-app/0.4.0 (contact: [email protected])'}

# ازحف، استخرج الحقول، ابنِ رسمًا بيانيًا معرفيًا
path_expr = """
url('https://en.wikipedia.org/wiki/Expression_language')
  ///url(
        //main//a/@href[
            starts-with(., '/wiki/') and not(contains(., ':'))
        ]
    )
    /map{
        'title': (//span[contains(@class, "mw-page-title-main")]/text())[1] ! string(.),
        'url': string(base-uri(.)),
        'short_description': //div[contains(@class, 'shortdescription')]/text() ! string(.),
        'forward_links': //div[@id="mw-content-text"]//a/@href ! string(.)
    }
"""

for item in wxpath.wxpath_async_blocking_iter(path_expr, max_depth=1):
    print(item)

import asyncio
from wxpath import wxpath_async

items = []

async def main():
    path_expr = "url('https://en.wikipedia.org/wiki/Expression_language')///url(//@href[starts-with(., '/wiki/')])//a/@href"
    async for item in wxpath_async(path_expr, max_depth=1):
        items.append(item)

asyncio.run(main())

from wxpath import wxpath_async_blocking_iter

path_expr = "url('https://en.wikipedia.org/wiki/Expression_language')///url(//@href[starts-with(., '/wiki/')])//a/@href"
items = list(wxpath_async_blocking_iter(path_expr, max_depth=1))

path_expr = """
    url('https://en.wikipedia.org/wiki/Expression_language')
    ///url(//div[@id='mw-content-text']//a/@href)
    /map{ 
        'title':(//span[contains(@class, "mw-page-title-main")]/text())[1], 
        'short_description':(//div[contains(@class, "shortdescription")]/text())[1],
        'url'://link[@rel='canonical']/@href[1]
    }
"""
# [...
# {'title': 'Computer language',
# 'short_description': 'Formal language for communicating with a computer',
# 'url': 'https://en.wikipedia.org/wiki/Computer_language'},
# {'title': 'Machine-readable medium and data',
# 'short_description': 'Medium capable of storing data in a format readable by a machine',
# 'url': 'https://en.wikipedia.org/wiki/Machine-readable_medium_and_data'},
# {'title': 'Domain knowledge',
# 'short_description': 'Specialist knowledge within a specific field',
# 'url': 'https://en.wikipedia.org/wiki/Domain_knowledge'},
# ...]

items = wxpath.wxpath_async_blocking("...", progress=True)
> 100%|██████████████████████████████████████████████████████████▎| 469/471 [00:05<00:00, 72.00it/s, depth=2, yielded=457]

> wxpath --depth 1 \
    --header "User-Agent: my-app/0.1 (contact: [email protected])" \
    "url('https://en.wikipedia.org/wiki/Expression_language') \
    ///url(//div[@id='mw-content-text']//a/@href[starts-with(., '/wiki/') \
        and not(matches(@href, '^(?:/wiki/)?(?:Wikipedia|File|Template|Special|Template_talk|Help):'))]) \
    /map{ \
        'title':(//span[contains(@class, 'mw-page-title-main')]/text())[1], \
        'short_description':(//div[contains(@class, 'shortdescription')]/text())[1], \
        'url':string(base-uri(.)), \
        'backlink':wx:backlink(.), \
        'depth':wx:depth(.) \
        }"

{"title": "Computer language", "short_description": "Formal language for communicating with a computer", "url": "https://en.wikipedia.org/wiki/Computer_language", "backlink": "https://en.wikipedia.org/wiki/Expression_language", "depth": 1.0}
{"title": "Machine-readable medium and data", "short_description": "Medium capable of storing data in a format readable by a machine", "url": "https://en.wikipedia.org/wiki/Machine_readable", "backlink": "https://en.wikipedia.org/wiki/Expression_language", "depth": 1.0}
{"title": "Domain knowledge", "short_description": "Specialist knowledge within a specific field", "url": "https://en.wikipedia.org/wiki/Domain_knowledge", "backlink": "https://en.wikipedia.org/wiki/Expression_language", "depth": 1.0}
{"title": "Advanced Boolean Expression Language", "short_description": "Hardware description language and software", "url": "https://en.wikipedia.org/wiki/Advanced_Boolean_Expression_Language", "backlink": "https://en.wikipedia.org/wiki/Expression_language", "depth": 1.0}
{"title": "Data Analysis Expressions", "short_description": "Formula and data query language", "url": "https://en.wikipedia.org/wiki/Data_Analysis_Expressions", "backlink": "https://en.wikipedia.org/wiki/Expression_language", "depth": 1.0}
{"title": "Jakarta Expression Language", "short_description": "Computer programming language", "url": "https://en.wikipedia.org/wiki/Jakarta_Expression_Language", "backlink": "https://en.wikipedia.org/wiki/Expression_language", "depth": 1.0}
{"title": "Rights Expression Language", "short_description": [], "url": "https://en.wikipedia.org/wiki/Rights_Expression_Language", "backlink": "https://en.wikipedia.org/wiki/Expression_language", "depth": 1.0}
{"title": "Computer science", "short_description": "Study of computation", "url": "https://en.wikipedia.org/wiki/Computer_science", "backlink": "https://en.wikipedia.org/wiki/Expression_language", "depth": 1.0}

--depth                <depth>       أقصى عمق للزحف
--verbose              [true|false]  يوفر معلومات CLI سطحية
--debug                [true|false]  يوفر إخراج تشغيل ومعلومات مفصلة
--concurrency          <concurrency> عدد الجلبات المتزامنة
--concurrency-per-host <concurrency> عدد الجلبات المتزامنة لكل مضيف
--header               "Key:Value"   أضف رأسًا مخصصًا (مثل 'Key:Value'). يمكن استخدامه عدة مرات.
--respect-robots       [true|false] (الافتراضي: True) يحترم robots.txt
--cache                [true|false] (الافتراضي: False) يستمر حفظ نتائج الزحف في قاعدة بيانات محلية

pip install wxpath[cache-sqlite]
pip install wxpath[cache-redis]

from wxpath.settings import SETTINGS

# لتمكين التخزين المؤقت؛ sqlite هو الافتراضي
SETTINGS.http.client.cache.enabled = True

# لواجهة redis الخلفية
SETTINGS.http.client.cache.enabled = True
SETTINGS.http.client.cache.backend = "redis"
SETTINGS.http.client.cache.redis.address = "redis://localhost:6379/0"

# تشغيل wxpath كالمعتاد
items = list(wxpath_async_blocking_iter('...', max_depth=1, engine=engine))

from wxpath import hooks

@hooks.register
class OnlyEnglish:
    def post_parse(self, ctx, elem):
        lang = elem.xpath('string(/html/@lang)').lower()[:2]
        return elem if lang in ("en", "") else None

from wxpath import hooks

@hooks.register
class OnlyEnglish:
    async def post_parse(self, ctx, elem):
        lang = elem.xpath('string(/html/@lang)').lower()[:2]
        return elem if lang in ("en", "") else None

from wxpath import hooks
hooks.register(hooks.JSONLWriter)

pip install wxpath

pip install wxpath[cache-sqlite]
pip install wxpath[cache-redis]

from wxpath import wxpath_async_blocking_iter
from wxpath.core.runtime import WXPathEngine
from wxpath.http.client.crawler import Crawler

crawler = Crawler(
    concurrency=8,
    per_host=2,
    timeout=10,
    respect_robots=False,
    headers={
        "User-Agent": "my-app/0.1.0 (contact: [email protected])", # مواقع مثل ويكيبيديا ستقدر هذا
    },
)

# إذا لم يتم تحديد `crawler`، سيتم إنشاء Crawler افتراضي باستخدام
# قيم concurrency و per_host و respect_robots المقدمة، أو بالقيم الافتراضية.
engine = WXPathEngine(
    # concurrency: int = 16, 
    # per_host: int = 8,
    # respect_robots: bool = True,
    # allowed_response_codes: set[int] = {200},
    # allow_redirects: bool = True,
    crawler=crawler,
)

path_expr = "url('https://en.wikipedia.org/wiki/Expression_language')//url(//main//a/@href)"

items = list(wxpath_async_blocking_iter(path_expr, max_depth=1, engine=engine))