현대 웹 크롤링을 위한 적응형 프레임워크로, 반지능적 요소 추적과 안티봇 우회를 지원합니다

pip install scrapling

python -c "from scrapling.fetchers import Fetcher; page = Fetcher.get('https://quotes.toscrape.com/'); print(page.css('.quote .text::text').getall()[:1])"

python << 'EOF'
from scrapling.fetchers import FetcherSession
with FetcherSession(impersonate='chrome') as session:
    page = session.get('https://quotes.toscrape.com/', stealthy_headers=True)
    quotes = page.css('.quote .text::text').getall()
    print(f"수집한 명언 수: {len(quotes)}")
EOF

python << 'EOF'
from scrapling.spiders import Spider, Response

class QuotesSpider(Spider):
    name = "quotes"
    start_urls = ["https://quotes.toscrape.com/"]
    concurrent_requests = 5
    
    async def parse(self, response: Response):
        for quote in response.css('.quote'):
            yield {
                "text": quote.css('.text::text').get(),
                "author": quote.css('.author::text').get(),
            }

result = QuotesSpider().start()
print(f"수집된 항목: {len(result.items)}")
EOF

python << 'EOF'
from scrapling.fetchers import StealthyFetcher
page = StealthyFetcher.fetch('https://nopecha.com/demo/cloudflare', headless=True)
data = page.css('#padded_content a').getall()
print(f"수집된 요소: {len(data)}")
EOF

python << 'EOF'
from scrapling.spiders import Spider, Response

class MySpider(Spider):
    name = "demo"
    start_urls = ["https://quotes.toscrape.com/"]
    
    async def parse(self, response: Response):
        for item in response.css('.quote'):
            yield {"title": item.css('.text::text').get()}

result = MySpider().start()
result.items.to_json("output.json")
print("JSON 파일로 저장 완료")
EOF