initial commit
This commit is contained in:
@@ -0,0 +1,314 @@
|
||||
package scraper
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"bargain-scraper/internal/models"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type Scraper struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
type ScrapeProgress struct {
|
||||
Category string
|
||||
Current int
|
||||
Total int
|
||||
Status string
|
||||
ProductName string
|
||||
}
|
||||
|
||||
func New() (*Scraper, error) {
|
||||
return &Scraper{
|
||||
client: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Scraper) Close() {
|
||||
}
|
||||
|
||||
func (s *Scraper) ScrapeCategory(ctx context.Context, category models.Category, progressChan chan ScrapeProgress) ([]models.StorageProduct, error) {
|
||||
products := []models.StorageProduct{}
|
||||
|
||||
progressChan <- ScrapeProgress{
|
||||
Category: category.Name,
|
||||
Current: 0,
|
||||
Total: 0,
|
||||
Status: "Fetching page...",
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", category.URL, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
|
||||
req.Header.Set("Accept-Language", "en-US,en;q=0.5")
|
||||
|
||||
resp, err := s.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch page: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("failed to fetch page: status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse HTML: %w", err)
|
||||
}
|
||||
|
||||
productItems := doc.Find("li.product-item")
|
||||
total := productItems.Length()
|
||||
|
||||
progressChan <- ScrapeProgress{
|
||||
Category: category.Name,
|
||||
Current: 0,
|
||||
Total: total,
|
||||
Status: "Parsing products...",
|
||||
}
|
||||
|
||||
productItems.Each(func(i int, sel *goquery.Selection) {
|
||||
product := s.parseProductItem(sel, category)
|
||||
if product.Name != "" {
|
||||
product.ID = uuid.New().String()
|
||||
product.ScrapedAt = time.Now()
|
||||
products = append(products, product)
|
||||
}
|
||||
|
||||
progressChan <- ScrapeProgress{
|
||||
Category: category.Name,
|
||||
Current: i + 1,
|
||||
Total: total,
|
||||
Status: "Parsing products...",
|
||||
ProductName: product.Name,
|
||||
}
|
||||
})
|
||||
|
||||
pageNum := 2
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return products, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
nextPageURL := fmt.Sprintf("%s?p=%d", category.URL, pageNum)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", nextPageURL, nil)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
|
||||
|
||||
resp, err := s.client.Do(req)
|
||||
if err != nil || resp.StatusCode != 200 {
|
||||
break
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
|
||||
nextItems := doc.Find("li.product-item")
|
||||
if nextItems.Length() == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
progressChan <- ScrapeProgress{
|
||||
Category: category.Name,
|
||||
Current: len(products),
|
||||
Total: len(products) + nextItems.Length(),
|
||||
Status: fmt.Sprintf("Parsing page %d...", pageNum),
|
||||
ProductName: "",
|
||||
}
|
||||
|
||||
nextItems.Each(func(i int, sel *goquery.Selection) {
|
||||
product := s.parseProductItem(sel, category)
|
||||
if product.Name != "" {
|
||||
product.ID = uuid.New().String()
|
||||
product.ScrapedAt = time.Now()
|
||||
products = append(products, product)
|
||||
}
|
||||
})
|
||||
|
||||
pageNum++
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
|
||||
return products, nil
|
||||
}
|
||||
|
||||
func (s *Scraper) parseProductItem(sel *goquery.Selection, category models.Category) models.StorageProduct {
|
||||
product := models.StorageProduct{
|
||||
FormFactor: extractFormFactor(category.Name),
|
||||
Interface: extractInterface(category.Name),
|
||||
}
|
||||
|
||||
link := sel.Find("a.product-item-link")
|
||||
if href, ok := link.Attr("href"); ok {
|
||||
product.URL = href
|
||||
}
|
||||
|
||||
product.Name = strings.TrimSpace(link.Text())
|
||||
if product.Name == "" {
|
||||
product.Name = strings.TrimSpace(sel.Find(".product-item-name").Text())
|
||||
}
|
||||
|
||||
price := sel.Find(".price").First().Text()
|
||||
product.Price = strings.TrimSpace(price)
|
||||
if product.Price == "" {
|
||||
product.Price = sel.Find("[data-price-type='finalPrice']").First().Text()
|
||||
}
|
||||
|
||||
desc := sel.Find(".product-item-description").Text()
|
||||
|
||||
capacity := extractCapacityFromText(desc)
|
||||
if capacity == "" {
|
||||
capacity = extractCapacityFromText(product.Name)
|
||||
}
|
||||
product.Capacity = capacity
|
||||
|
||||
product.Brand = extractBrandFromText(desc, product.Name)
|
||||
if product.Brand == "" {
|
||||
product.Brand = inferBrand(product.Name)
|
||||
}
|
||||
|
||||
stockText := sel.Find(".stock span").Text()
|
||||
if stockText == "" {
|
||||
stockText = sel.Find(".stock.available").Text()
|
||||
}
|
||||
product.StockStatus = strings.TrimSpace(stockText)
|
||||
if product.StockStatus == "" {
|
||||
if sel.Find(".stock.not-available").Length() > 0 {
|
||||
product.StockStatus = "Out of stock"
|
||||
} else if sel.Find(".stock.available").Length() > 0 {
|
||||
product.StockStatus = "In stock"
|
||||
}
|
||||
}
|
||||
|
||||
return product
|
||||
}
|
||||
|
||||
func extractFormFactor(categoryName string) string {
|
||||
lower := strings.ToLower(categoryName)
|
||||
if strings.Contains(lower, "3.5") || strings.Contains(lower, "lff") {
|
||||
return "3.5\""
|
||||
}
|
||||
if strings.Contains(lower, "2.5") || strings.Contains(lower, "sff") {
|
||||
return "2.5\""
|
||||
}
|
||||
if strings.Contains(lower, "m.2") {
|
||||
return "M.2"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func extractInterface(categoryName string) string {
|
||||
lower := strings.ToLower(categoryName)
|
||||
if strings.Contains(lower, "nvme") {
|
||||
return "NVMe"
|
||||
}
|
||||
if strings.Contains(lower, "sas") {
|
||||
return "SAS"
|
||||
}
|
||||
if strings.Contains(lower, "sata") {
|
||||
return "SATA"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func extractCapacityFromText(text string) string {
|
||||
patterns := []string{
|
||||
`(?i)capacity[:\s]*(\d+(?:\.\d+)?\s*TB)`,
|
||||
`(?i)capacity[:\s]*(\d+(?:\.\d+)?\s*GB)`,
|
||||
`(\d+(?:\.\d+)?\s*TB)`,
|
||||
`(\d+(?:\.\d+)?\s*GB)`,
|
||||
`(960GB|480GB|1\.92TB|3\.84TB|7\.68TB|15\.36TB|31\.44TB|1TB|2TB|4TB|8TB|12TB|14TB|16TB|18TB|20TB|22TB|24TB|26TB|28TB|30TB|32TB)`,
|
||||
}
|
||||
|
||||
for _, pattern := range patterns {
|
||||
re := regexp.MustCompile(pattern)
|
||||
matches := re.FindStringSubmatch(text)
|
||||
if len(matches) > 1 {
|
||||
return strings.ToUpper(matches[1])
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func extractBrandFromText(text, name string) string {
|
||||
brands := []string{"Dell", "HP", "HPE", "Seagate", "Western Digital", "WD", "Samsung", "Micron", "Intel", "Toshiba", "Hitachi", "IBM", "Lenovo", "Supermicro", "Generic", "Stec", "White Label"}
|
||||
|
||||
textLower := strings.ToLower(text)
|
||||
nameLower := strings.ToLower(name)
|
||||
|
||||
for _, brand := range brands {
|
||||
if strings.Contains(textLower, strings.ToLower(brand)) || strings.Contains(nameLower, strings.ToLower(brand)) {
|
||||
if brand == "WD" {
|
||||
return "Western Digital"
|
||||
}
|
||||
if brand == "HPE" {
|
||||
return "HP"
|
||||
}
|
||||
return brand
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func inferBrand(name string) string {
|
||||
if name == "" {
|
||||
return ""
|
||||
}
|
||||
brands := []string{"Dell", "HP", "HPE", "Seagate", "Western Digital", "Samsung", "Micron", "Intel", "Toshiba", "Hitachi", "IBM", "Lenovo", "Supermicro", "Generic", "Stec", "White Label"}
|
||||
lower := strings.ToLower(name)
|
||||
for _, brand := range brands {
|
||||
if strings.Contains(lower, strings.ToLower(brand)) {
|
||||
if brand == "WD" {
|
||||
return "Western Digital"
|
||||
}
|
||||
if brand == "HPE" {
|
||||
return "HP"
|
||||
}
|
||||
return brand
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (s *Scraper) ScrapeAllCategories(ctx context.Context, progressChan chan ScrapeProgress) ([]models.StorageProduct, error) {
|
||||
allProducts := []models.StorageProduct{}
|
||||
|
||||
for _, category := range models.StorageCategories {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return allProducts, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
products, err := s.ScrapeCategory(ctx, category, progressChan)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
allProducts = append(allProducts, products...)
|
||||
}
|
||||
|
||||
return allProducts, nil
|
||||
}
|
||||
Reference in New Issue
Block a user