package scraper import ( "context" "fmt" "net/http" "regexp" "strings" "time" "bargain-scraper/internal/models" "github.com/PuerkitoBio/goquery" "github.com/google/uuid" ) type Scraper struct { client *http.Client } type ScrapeProgress struct { Category string Current int Total int Status string ProductName string } func New() (*Scraper, error) { return &Scraper{ client: &http.Client{ Timeout: 30 * time.Second, }, }, nil } func (s *Scraper) Close() { } func (s *Scraper) ScrapeCategory(ctx context.Context, category models.Category, progressChan chan ScrapeProgress) ([]models.StorageProduct, error) { products := []models.StorageProduct{} progressChan <- ScrapeProgress{ Category: category.Name, Current: 0, Total: 0, Status: "Fetching page...", } req, err := http.NewRequestWithContext(ctx, "GET", category.URL, nil) if err != nil { return nil, fmt.Errorf("failed to create request: %w", err) } req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8") req.Header.Set("Accept-Language", "en-US,en;q=0.5") resp, err := s.client.Do(req) if err != nil { return nil, fmt.Errorf("failed to fetch page: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("failed to fetch page: status %d", resp.StatusCode) } doc, err := goquery.NewDocumentFromReader(resp.Body) if err != nil { return nil, fmt.Errorf("failed to parse HTML: %w", err) } productItems := doc.Find("li.product-item") total := productItems.Length() progressChan <- ScrapeProgress{ Category: category.Name, Current: 0, Total: total, Status: "Parsing products...", } productItems.Each(func(i int, sel *goquery.Selection) { product := s.parseProductItem(sel, category) if product.Name != "" { product.ID = uuid.New().String() product.ScrapedAt = time.Now() products = append(products, product) } progressChan <- ScrapeProgress{ Category: category.Name, Current: i + 1, Total: total, Status: "Parsing products...", ProductName: product.Name, } }) pageNum := 2 for { select { case <-ctx.Done(): return products, ctx.Err() default: } nextPageURL := fmt.Sprintf("%s?p=%d", category.URL, pageNum) req, err := http.NewRequestWithContext(ctx, "GET", nextPageURL, nil) if err != nil { break } req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36") resp, err := s.client.Do(req) if err != nil || resp.StatusCode != 200 { break } defer resp.Body.Close() doc, err := goquery.NewDocumentFromReader(resp.Body) if err != nil { break } nextItems := doc.Find("li.product-item") if nextItems.Length() == 0 { break } progressChan <- ScrapeProgress{ Category: category.Name, Current: len(products), Total: len(products) + nextItems.Length(), Status: fmt.Sprintf("Parsing page %d...", pageNum), ProductName: "", } nextItems.Each(func(i int, sel *goquery.Selection) { product := s.parseProductItem(sel, category) if product.Name != "" { product.ID = uuid.New().String() product.ScrapedAt = time.Now() products = append(products, product) } }) pageNum++ time.Sleep(500 * time.Millisecond) } return products, nil } func (s *Scraper) parseProductItem(sel *goquery.Selection, category models.Category) models.StorageProduct { product := models.StorageProduct{ FormFactor: extractFormFactor(category.Name), Interface: extractInterface(category.Name), } link := sel.Find("a.product-item-link") if href, ok := link.Attr("href"); ok { product.URL = href } product.Name = strings.TrimSpace(link.Text()) if product.Name == "" { product.Name = strings.TrimSpace(sel.Find(".product-item-name").Text()) } price := sel.Find(".price").First().Text() product.Price = strings.TrimSpace(price) if product.Price == "" { product.Price = sel.Find("[data-price-type='finalPrice']").First().Text() } desc := sel.Find(".product-item-description").Text() capacity := extractCapacityFromText(desc) if capacity == "" { capacity = extractCapacityFromText(product.Name) } product.Capacity = capacity product.Brand = extractBrandFromText(desc, product.Name) if product.Brand == "" { product.Brand = inferBrand(product.Name) } stockText := sel.Find(".stock span").Text() if stockText == "" { stockText = sel.Find(".stock.available").Text() } product.StockStatus = strings.TrimSpace(stockText) if product.StockStatus == "" { if sel.Find(".stock.not-available").Length() > 0 { product.StockStatus = "Out of stock" } else if sel.Find(".stock.available").Length() > 0 { product.StockStatus = "In stock" } } return product } func extractFormFactor(categoryName string) string { lower := strings.ToLower(categoryName) if strings.Contains(lower, "3.5") || strings.Contains(lower, "lff") { return "3.5\"" } if strings.Contains(lower, "2.5") || strings.Contains(lower, "sff") { return "2.5\"" } if strings.Contains(lower, "m.2") { return "M.2" } return "" } func extractInterface(categoryName string) string { lower := strings.ToLower(categoryName) if strings.Contains(lower, "nvme") { return "NVMe" } if strings.Contains(lower, "sas") { return "SAS" } if strings.Contains(lower, "sata") { return "SATA" } return "" } func extractCapacityFromText(text string) string { patterns := []string{ `(?i)capacity[:\s]*(\d+(?:\.\d+)?\s*TB)`, `(?i)capacity[:\s]*(\d+(?:\.\d+)?\s*GB)`, `(\d+(?:\.\d+)?\s*TB)`, `(\d+(?:\.\d+)?\s*GB)`, `(960GB|480GB|1\.92TB|3\.84TB|7\.68TB|15\.36TB|31\.44TB|1TB|2TB|4TB|8TB|12TB|14TB|16TB|18TB|20TB|22TB|24TB|26TB|28TB|30TB|32TB)`, } for _, pattern := range patterns { re := regexp.MustCompile(pattern) matches := re.FindStringSubmatch(text) if len(matches) > 1 { return strings.ToUpper(matches[1]) } } return "" } func extractBrandFromText(text, name string) string { brands := []string{"Dell", "HP", "HPE", "Seagate", "Western Digital", "WD", "Samsung", "Micron", "Intel", "Toshiba", "Hitachi", "IBM", "Lenovo", "Supermicro", "Generic", "Stec", "White Label"} textLower := strings.ToLower(text) nameLower := strings.ToLower(name) for _, brand := range brands { if strings.Contains(textLower, strings.ToLower(brand)) || strings.Contains(nameLower, strings.ToLower(brand)) { if brand == "WD" { return "Western Digital" } if brand == "HPE" { return "HP" } return brand } } return "" } func inferBrand(name string) string { if name == "" { return "" } brands := []string{"Dell", "HP", "HPE", "Seagate", "Western Digital", "Samsung", "Micron", "Intel", "Toshiba", "Hitachi", "IBM", "Lenovo", "Supermicro", "Generic", "Stec", "White Label"} lower := strings.ToLower(name) for _, brand := range brands { if strings.Contains(lower, strings.ToLower(brand)) { if brand == "WD" { return "Western Digital" } if brand == "HPE" { return "HP" } return brand } } return "" } func (s *Scraper) ScrapeAllCategories(ctx context.Context, progressChan chan ScrapeProgress) ([]models.StorageProduct, error) { allProducts := []models.StorageProduct{} for _, category := range models.StorageCategories { select { case <-ctx.Done(): return allProducts, ctx.Err() default: } products, err := s.ScrapeCategory(ctx, category, progressChan) if err != nil { continue } allProducts = append(allProducts, products...) } return allProducts, nil }