bargain-hardware/internal/scraper/scraper.go

package scraper

import (
	"context"
	"fmt"
	"net/http"
	"regexp"
	"strings"
	"time"

	"bargain-scraper/internal/models"

	"github.com/PuerkitoBio/goquery"
	"github.com/google/uuid"
)

type Scraper struct {
	client *http.Client
}

type ScrapeProgress struct {
	Category    string
	Current     int
	Total       int
	Status      string
	ProductName string
}

func New() (*Scraper, error) {
	return &Scraper{
		client: &http.Client{
			Timeout: 30 * time.Second,
		},
	}, nil
}

func (s *Scraper) Close() {
}

func (s *Scraper) ScrapeCategory(ctx context.Context, category models.Category, progressChan chan ScrapeProgress) ([]models.StorageProduct, error) {
	products := []models.StorageProduct{}

	progressChan <- ScrapeProgress{
		Category: category.Name,
		Current:  0,
		Total:    0,
		Status:   "Fetching page...",
	}

	req, err := http.NewRequestWithContext(ctx, "GET", category.URL, nil)
	if err != nil {
		return nil, fmt.Errorf("failed to create request: %w", err)
	}

	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
	req.Header.Set("Accept-Language", "en-US,en;q=0.5")

	resp, err := s.client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("failed to fetch page: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("failed to fetch page: status %d", resp.StatusCode)
	}

	doc, err := goquery.NewDocumentFromReader(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("failed to parse HTML: %w", err)
	}

	productItems := doc.Find("li.product-item")
	total := productItems.Length()

	progressChan <- ScrapeProgress{
		Category: category.Name,
		Current:  0,
		Total:    total,
		Status:   "Parsing products...",
	}

	productItems.Each(func(i int, sel *goquery.Selection) {
		product := s.parseProductItem(sel, category)
		if product.Name != "" {
			product.ID = uuid.New().String()
			product.ScrapedAt = time.Now()
			products = append(products, product)
		}

		progressChan <- ScrapeProgress{
			Category:    category.Name,
			Current:     i + 1,
			Total:       total,
			Status:      "Parsing products...",
			ProductName: product.Name,
		}
	})

	pageNum := 2
	for {
		select {
		case <-ctx.Done():
			return products, ctx.Err()
		default:
		}

		nextPageURL := fmt.Sprintf("%s?p=%d", category.URL, pageNum)
		req, err := http.NewRequestWithContext(ctx, "GET", nextPageURL, nil)
		if err != nil {
			break
		}
		req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")

		resp, err := s.client.Do(req)
		if err != nil || resp.StatusCode != 200 {
			break
		}
		defer resp.Body.Close()

		doc, err := goquery.NewDocumentFromReader(resp.Body)
		if err != nil {
			break
		}

		nextItems := doc.Find("li.product-item")
		if nextItems.Length() == 0 {
			break
		}

		progressChan <- ScrapeProgress{
			Category:    category.Name,
			Current:     len(products),
			Total:       len(products) + nextItems.Length(),
			Status:      fmt.Sprintf("Parsing page %d...", pageNum),
			ProductName: "",
		}

		nextItems.Each(func(i int, sel *goquery.Selection) {
			product := s.parseProductItem(sel, category)
			if product.Name != "" {
				product.ID = uuid.New().String()
				product.ScrapedAt = time.Now()
				products = append(products, product)
			}
		})

		pageNum++
		time.Sleep(500 * time.Millisecond)
	}

	return products, nil
}

func (s *Scraper) parseProductItem(sel *goquery.Selection, category models.Category) models.StorageProduct {
	product := models.StorageProduct{
		FormFactor: extractFormFactor(category.Name),
		Interface:  extractInterface(category.Name),
	}

	link := sel.Find("a.product-item-link")
	if href, ok := link.Attr("href"); ok {
		product.URL = href
	}

	product.Name = strings.TrimSpace(link.Text())
	if product.Name == "" {
		product.Name = strings.TrimSpace(sel.Find(".product-item-name").Text())
	}

	price := sel.Find(".price").First().Text()
	product.Price = strings.TrimSpace(price)
	if product.Price == "" {
		product.Price = sel.Find("[data-price-type='finalPrice']").First().Text()
	}

	desc := sel.Find(".product-item-description").Text()

	capacity := extractCapacityFromText(desc)
	if capacity == "" {
		capacity = extractCapacityFromText(product.Name)
	}
	product.Capacity = capacity

	product.Brand = extractBrandFromText(desc, product.Name)
	if product.Brand == "" {
		product.Brand = inferBrand(product.Name)
	}

	stockText := sel.Find(".stock span").Text()
	if stockText == "" {
		stockText = sel.Find(".stock.available").Text()
	}
	product.StockStatus = strings.TrimSpace(stockText)
	if product.StockStatus == "" {
		if sel.Find(".stock.not-available").Length() > 0 {
			product.StockStatus = "Out of stock"
		} else if sel.Find(".stock.available").Length() > 0 {
			product.StockStatus = "In stock"
		}
	}

	return product
}

func extractFormFactor(categoryName string) string {
	lower := strings.ToLower(categoryName)
	if strings.Contains(lower, "3.5") || strings.Contains(lower, "lff") {
		return "3.5\""
	}
	if strings.Contains(lower, "2.5") || strings.Contains(lower, "sff") {
		return "2.5\""
	}
	if strings.Contains(lower, "m.2") {
		return "M.2"
	}
	return ""
}

func extractInterface(categoryName string) string {
	lower := strings.ToLower(categoryName)
	if strings.Contains(lower, "nvme") {
		return "NVMe"
	}
	if strings.Contains(lower, "sas") {
		return "SAS"
	}
	if strings.Contains(lower, "sata") {
		return "SATA"
	}
	return ""
}

func extractCapacityFromText(text string) string {
	patterns := []string{
		`(?i)capacity[:\s]*(\d+(?:\.\d+)?\s*TB)`,
		`(?i)capacity[:\s]*(\d+(?:\.\d+)?\s*GB)`,
		`(\d+(?:\.\d+)?\s*TB)`,
		`(\d+(?:\.\d+)?\s*GB)`,
		`(960GB|480GB|1\.92TB|3\.84TB|7\.68TB|15\.36TB|31\.44TB|1TB|2TB|4TB|8TB|12TB|14TB|16TB|18TB|20TB|22TB|24TB|26TB|28TB|30TB|32TB)`,
	}

	for _, pattern := range patterns {
		re := regexp.MustCompile(pattern)
		matches := re.FindStringSubmatch(text)
		if len(matches) > 1 {
			return strings.ToUpper(matches[1])
		}
	}

	return ""
}

func extractBrandFromText(text, name string) string {
	brands := []string{"Dell", "HP", "HPE", "Seagate", "Western Digital", "WD", "Samsung", "Micron", "Intel", "Toshiba", "Hitachi", "IBM", "Lenovo", "Supermicro", "Generic", "Stec", "White Label"}

	textLower := strings.ToLower(text)
	nameLower := strings.ToLower(name)

	for _, brand := range brands {
		if strings.Contains(textLower, strings.ToLower(brand)) || strings.Contains(nameLower, strings.ToLower(brand)) {
			if brand == "WD" {
				return "Western Digital"
			}
			if brand == "HPE" {
				return "HP"
			}
			return brand
		}
	}
	return ""
}

func inferBrand(name string) string {
	if name == "" {
		return ""
	}
	brands := []string{"Dell", "HP", "HPE", "Seagate", "Western Digital", "Samsung", "Micron", "Intel", "Toshiba", "Hitachi", "IBM", "Lenovo", "Supermicro", "Generic", "Stec", "White Label"}
	lower := strings.ToLower(name)
	for _, brand := range brands {
		if strings.Contains(lower, strings.ToLower(brand)) {
			if brand == "WD" {
				return "Western Digital"
			}
			if brand == "HPE" {
				return "HP"
			}
			return brand
		}
	}
	return ""
}

func (s *Scraper) ScrapeAllCategories(ctx context.Context, progressChan chan ScrapeProgress) ([]models.StorageProduct, error) {
	allProducts := []models.StorageProduct{}

	for _, category := range models.StorageCategories {
		select {
		case <-ctx.Done():
			return allProducts, ctx.Err()
		default:
		}

		products, err := s.ScrapeCategory(ctx, category, progressChan)
		if err != nil {
			continue
		}

		allProducts = append(allProducts, products...)
	}

	return allProducts, nil
}