initial commit
This commit is contained in:
@@ -0,0 +1,151 @@
|
||||
package exporter
|
||||
|
||||
import (
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"bargain-scraper/internal/models"
|
||||
)
|
||||
|
||||
type Exporter struct{}
|
||||
|
||||
func New() *Exporter {
|
||||
return &Exporter{}
|
||||
}
|
||||
|
||||
func (e *Exporter) ExportToJSON(products []models.StorageProduct, filename string) error {
|
||||
if filename == "" {
|
||||
filename = fmt.Sprintf("storage_products_%s.json", time.Now().Format("2006-01-02_150405"))
|
||||
}
|
||||
|
||||
data, err := json.MarshalIndent(products, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal JSON: %w", err)
|
||||
}
|
||||
|
||||
err = os.WriteFile(filename, data, 0644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write JSON file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *Exporter) ExportToCSV(products []models.StorageProduct, filename string) error {
|
||||
if filename == "" {
|
||||
filename = fmt.Sprintf("storage_products_%s.csv", time.Now().Format("2006-01-02_150405"))
|
||||
}
|
||||
|
||||
file, err := os.Create(filename)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create CSV file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
writer := csv.NewWriter(file)
|
||||
defer writer.Flush()
|
||||
|
||||
headers := []string{"ID", "Name", "Price", "Capacity", "Interface", "FormFactor", "Brand", "PartNumber", "StockStatus", "URL", "ScrapedAt"}
|
||||
if err := writer.Write(headers); err != nil {
|
||||
return fmt.Errorf("failed to write CSV headers: %w", err)
|
||||
}
|
||||
|
||||
for _, p := range products {
|
||||
row := []string{
|
||||
p.ID,
|
||||
p.Name,
|
||||
p.Price,
|
||||
p.Capacity,
|
||||
p.Interface,
|
||||
p.FormFactor,
|
||||
p.Brand,
|
||||
p.PartNumber,
|
||||
p.StockStatus,
|
||||
p.URL,
|
||||
p.ScrapedAt.Format(time.RFC3339),
|
||||
}
|
||||
if err := writer.Write(row); err != nil {
|
||||
return fmt.Errorf("failed to write CSV row: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *Exporter) PrintToConsole(products []models.StorageProduct) string {
|
||||
var b strings.Builder
|
||||
|
||||
b.WriteString("\n")
|
||||
b.WriteString("═══════════════════════════════════════════════════════════════════════════════\n")
|
||||
b.WriteString(fmt.Sprintf(" STORAGE PRODUCTS (%d items) \n", len(products)))
|
||||
b.WriteString("═══════════════════════════════════════════════════════════════════════════════\n")
|
||||
b.WriteString("\n")
|
||||
|
||||
headerFmt := " %-4s | %-40s | %-10s | %-8s | %-6s | %-10s | %-12s\n"
|
||||
separator := "──────┼──────────────────────────────────────────┬────────────┬──────────┼────────┼────────────┼──────────────\n"
|
||||
|
||||
b.WriteString(fmt.Sprintf(headerFmt, "#", "Name", "Capacity", "Interface", "Factor", "Stock", "Price"))
|
||||
b.WriteString(separator)
|
||||
|
||||
for i, p := range products {
|
||||
name := p.Name
|
||||
if len(name) > 40 {
|
||||
name = name[:37] + "..."
|
||||
}
|
||||
|
||||
capacity := p.Capacity
|
||||
if capacity == "" {
|
||||
capacity = "-"
|
||||
}
|
||||
|
||||
interface_ := p.Interface
|
||||
if interface_ == "" {
|
||||
interface_ = "-"
|
||||
}
|
||||
|
||||
formFactor := p.FormFactor
|
||||
if formFactor == "" {
|
||||
formFactor = "-"
|
||||
}
|
||||
|
||||
stock := p.StockStatus
|
||||
if stock == "" {
|
||||
stock = "-"
|
||||
}
|
||||
|
||||
price := p.Price
|
||||
if price == "" {
|
||||
price = "-"
|
||||
}
|
||||
|
||||
b.WriteString(fmt.Sprintf(headerFmt,
|
||||
fmt.Sprintf("%d", i+1),
|
||||
name,
|
||||
capacity,
|
||||
interface_,
|
||||
formFactor,
|
||||
stock,
|
||||
price,
|
||||
))
|
||||
}
|
||||
|
||||
b.WriteString("\n")
|
||||
b.WriteString(fmt.Sprintf("Total: %d products\n", len(products)))
|
||||
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func (e *Exporter) Export(products []models.StorageProduct, format string, filename string) error {
|
||||
switch strings.ToLower(format) {
|
||||
case "json":
|
||||
return e.ExportToJSON(products, filename)
|
||||
case "csv":
|
||||
return e.ExportToCSV(products, filename)
|
||||
default:
|
||||
return fmt.Errorf("unsupported format: %s (use 'json' or 'csv')", format)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package models
|
||||
|
||||
import "time"
|
||||
|
||||
type StorageProduct struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Price string `json:"price"`
|
||||
Capacity string `json:"capacity"`
|
||||
Interface string `json:"interface"`
|
||||
FormFactor string `json:"form_factor"`
|
||||
Brand string `json:"brand"`
|
||||
PartNumber string `json:"part_number"`
|
||||
StockStatus string `json:"stock_status"`
|
||||
URL string `json:"url"`
|
||||
ScrapedAt time.Time `json:"scraped_at"`
|
||||
}
|
||||
|
||||
type Category struct {
|
||||
Name string
|
||||
URL string
|
||||
Description string
|
||||
}
|
||||
|
||||
var StorageCategories = []Category{
|
||||
{
|
||||
Name: "3.5\" SATA HDDs",
|
||||
URL: "https://www.bargainhardware.co.uk/components/enterprise-hdds-sdds-storage/large-form-factor-sata-drives",
|
||||
Description: "Large Form Factor SATA Hard Drives",
|
||||
},
|
||||
{
|
||||
Name: "3.5\" SAS HDDs",
|
||||
URL: "https://www.bargainhardware.co.uk/components/enterprise-hdds-sdds-storage/large-form-factor-sas-drives",
|
||||
Description: "Large Form Factor SAS Hard Drives",
|
||||
},
|
||||
{
|
||||
Name: "2.5\" SATA HDDs",
|
||||
URL: "https://www.bargainhardware.co.uk/components/enterprise-hdds-sdds-storage/small-form-factor-sata-drives",
|
||||
Description: "Small Form Factor SATA Hard Drives",
|
||||
},
|
||||
{
|
||||
Name: "2.5\" SAS HDDs",
|
||||
URL: "https://www.bargainhardware.co.uk/components/enterprise-hdds-sdds-storage/small-form-factor-sas-drives",
|
||||
Description: "Small Form Factor SAS Hard Drives",
|
||||
},
|
||||
{
|
||||
Name: "SATA SSDs (2.5\")",
|
||||
URL: "https://www.bargainhardware.co.uk/components/enterprise-hdds-sdds-storage/sata-ssds",
|
||||
Description: "SATA Solid State Drives",
|
||||
},
|
||||
{
|
||||
Name: "SAS SSDs",
|
||||
URL: "https://www.bargainhardware.co.uk/components/enterprise-hdds-sdds-storage/sas-ssd",
|
||||
Description: "SAS Solid State Drives",
|
||||
},
|
||||
{
|
||||
Name: "NVMe SSDs",
|
||||
URL: "https://www.bargainhardware.co.uk/components/enterprise-hdds-sdds-storage/enterprise-nvme-drives",
|
||||
Description: "NVMe Solid State Drives",
|
||||
},
|
||||
{
|
||||
Name: "M.2 / Other SSDs",
|
||||
URL: "https://www.bargainhardware.co.uk/components/hard-drives-and-solid-state-drives/nvme-ssds",
|
||||
Description: "M.2 and Other SSD Form Factors",
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
package scraper
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"bargain-scraper/internal/models"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type Scraper struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
type ScrapeProgress struct {
|
||||
Category string
|
||||
Current int
|
||||
Total int
|
||||
Status string
|
||||
ProductName string
|
||||
}
|
||||
|
||||
func New() (*Scraper, error) {
|
||||
return &Scraper{
|
||||
client: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Scraper) Close() {
|
||||
}
|
||||
|
||||
func (s *Scraper) ScrapeCategory(ctx context.Context, category models.Category, progressChan chan ScrapeProgress) ([]models.StorageProduct, error) {
|
||||
products := []models.StorageProduct{}
|
||||
|
||||
progressChan <- ScrapeProgress{
|
||||
Category: category.Name,
|
||||
Current: 0,
|
||||
Total: 0,
|
||||
Status: "Fetching page...",
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", category.URL, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
|
||||
req.Header.Set("Accept-Language", "en-US,en;q=0.5")
|
||||
|
||||
resp, err := s.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch page: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("failed to fetch page: status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse HTML: %w", err)
|
||||
}
|
||||
|
||||
productItems := doc.Find("li.product-item")
|
||||
total := productItems.Length()
|
||||
|
||||
progressChan <- ScrapeProgress{
|
||||
Category: category.Name,
|
||||
Current: 0,
|
||||
Total: total,
|
||||
Status: "Parsing products...",
|
||||
}
|
||||
|
||||
productItems.Each(func(i int, sel *goquery.Selection) {
|
||||
product := s.parseProductItem(sel, category)
|
||||
if product.Name != "" {
|
||||
product.ID = uuid.New().String()
|
||||
product.ScrapedAt = time.Now()
|
||||
products = append(products, product)
|
||||
}
|
||||
|
||||
progressChan <- ScrapeProgress{
|
||||
Category: category.Name,
|
||||
Current: i + 1,
|
||||
Total: total,
|
||||
Status: "Parsing products...",
|
||||
ProductName: product.Name,
|
||||
}
|
||||
})
|
||||
|
||||
pageNum := 2
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return products, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
nextPageURL := fmt.Sprintf("%s?p=%d", category.URL, pageNum)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", nextPageURL, nil)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
|
||||
|
||||
resp, err := s.client.Do(req)
|
||||
if err != nil || resp.StatusCode != 200 {
|
||||
break
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
|
||||
nextItems := doc.Find("li.product-item")
|
||||
if nextItems.Length() == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
progressChan <- ScrapeProgress{
|
||||
Category: category.Name,
|
||||
Current: len(products),
|
||||
Total: len(products) + nextItems.Length(),
|
||||
Status: fmt.Sprintf("Parsing page %d...", pageNum),
|
||||
ProductName: "",
|
||||
}
|
||||
|
||||
nextItems.Each(func(i int, sel *goquery.Selection) {
|
||||
product := s.parseProductItem(sel, category)
|
||||
if product.Name != "" {
|
||||
product.ID = uuid.New().String()
|
||||
product.ScrapedAt = time.Now()
|
||||
products = append(products, product)
|
||||
}
|
||||
})
|
||||
|
||||
pageNum++
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
|
||||
return products, nil
|
||||
}
|
||||
|
||||
func (s *Scraper) parseProductItem(sel *goquery.Selection, category models.Category) models.StorageProduct {
|
||||
product := models.StorageProduct{
|
||||
FormFactor: extractFormFactor(category.Name),
|
||||
Interface: extractInterface(category.Name),
|
||||
}
|
||||
|
||||
link := sel.Find("a.product-item-link")
|
||||
if href, ok := link.Attr("href"); ok {
|
||||
product.URL = href
|
||||
}
|
||||
|
||||
product.Name = strings.TrimSpace(link.Text())
|
||||
if product.Name == "" {
|
||||
product.Name = strings.TrimSpace(sel.Find(".product-item-name").Text())
|
||||
}
|
||||
|
||||
price := sel.Find(".price").First().Text()
|
||||
product.Price = strings.TrimSpace(price)
|
||||
if product.Price == "" {
|
||||
product.Price = sel.Find("[data-price-type='finalPrice']").First().Text()
|
||||
}
|
||||
|
||||
desc := sel.Find(".product-item-description").Text()
|
||||
|
||||
capacity := extractCapacityFromText(desc)
|
||||
if capacity == "" {
|
||||
capacity = extractCapacityFromText(product.Name)
|
||||
}
|
||||
product.Capacity = capacity
|
||||
|
||||
product.Brand = extractBrandFromText(desc, product.Name)
|
||||
if product.Brand == "" {
|
||||
product.Brand = inferBrand(product.Name)
|
||||
}
|
||||
|
||||
stockText := sel.Find(".stock span").Text()
|
||||
if stockText == "" {
|
||||
stockText = sel.Find(".stock.available").Text()
|
||||
}
|
||||
product.StockStatus = strings.TrimSpace(stockText)
|
||||
if product.StockStatus == "" {
|
||||
if sel.Find(".stock.not-available").Length() > 0 {
|
||||
product.StockStatus = "Out of stock"
|
||||
} else if sel.Find(".stock.available").Length() > 0 {
|
||||
product.StockStatus = "In stock"
|
||||
}
|
||||
}
|
||||
|
||||
return product
|
||||
}
|
||||
|
||||
func extractFormFactor(categoryName string) string {
|
||||
lower := strings.ToLower(categoryName)
|
||||
if strings.Contains(lower, "3.5") || strings.Contains(lower, "lff") {
|
||||
return "3.5\""
|
||||
}
|
||||
if strings.Contains(lower, "2.5") || strings.Contains(lower, "sff") {
|
||||
return "2.5\""
|
||||
}
|
||||
if strings.Contains(lower, "m.2") {
|
||||
return "M.2"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func extractInterface(categoryName string) string {
|
||||
lower := strings.ToLower(categoryName)
|
||||
if strings.Contains(lower, "nvme") {
|
||||
return "NVMe"
|
||||
}
|
||||
if strings.Contains(lower, "sas") {
|
||||
return "SAS"
|
||||
}
|
||||
if strings.Contains(lower, "sata") {
|
||||
return "SATA"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func extractCapacityFromText(text string) string {
|
||||
patterns := []string{
|
||||
`(?i)capacity[:\s]*(\d+(?:\.\d+)?\s*TB)`,
|
||||
`(?i)capacity[:\s]*(\d+(?:\.\d+)?\s*GB)`,
|
||||
`(\d+(?:\.\d+)?\s*TB)`,
|
||||
`(\d+(?:\.\d+)?\s*GB)`,
|
||||
`(960GB|480GB|1\.92TB|3\.84TB|7\.68TB|15\.36TB|31\.44TB|1TB|2TB|4TB|8TB|12TB|14TB|16TB|18TB|20TB|22TB|24TB|26TB|28TB|30TB|32TB)`,
|
||||
}
|
||||
|
||||
for _, pattern := range patterns {
|
||||
re := regexp.MustCompile(pattern)
|
||||
matches := re.FindStringSubmatch(text)
|
||||
if len(matches) > 1 {
|
||||
return strings.ToUpper(matches[1])
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func extractBrandFromText(text, name string) string {
|
||||
brands := []string{"Dell", "HP", "HPE", "Seagate", "Western Digital", "WD", "Samsung", "Micron", "Intel", "Toshiba", "Hitachi", "IBM", "Lenovo", "Supermicro", "Generic", "Stec", "White Label"}
|
||||
|
||||
textLower := strings.ToLower(text)
|
||||
nameLower := strings.ToLower(name)
|
||||
|
||||
for _, brand := range brands {
|
||||
if strings.Contains(textLower, strings.ToLower(brand)) || strings.Contains(nameLower, strings.ToLower(brand)) {
|
||||
if brand == "WD" {
|
||||
return "Western Digital"
|
||||
}
|
||||
if brand == "HPE" {
|
||||
return "HP"
|
||||
}
|
||||
return brand
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func inferBrand(name string) string {
|
||||
if name == "" {
|
||||
return ""
|
||||
}
|
||||
brands := []string{"Dell", "HP", "HPE", "Seagate", "Western Digital", "Samsung", "Micron", "Intel", "Toshiba", "Hitachi", "IBM", "Lenovo", "Supermicro", "Generic", "Stec", "White Label"}
|
||||
lower := strings.ToLower(name)
|
||||
for _, brand := range brands {
|
||||
if strings.Contains(lower, strings.ToLower(brand)) {
|
||||
if brand == "WD" {
|
||||
return "Western Digital"
|
||||
}
|
||||
if brand == "HPE" {
|
||||
return "HP"
|
||||
}
|
||||
return brand
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (s *Scraper) ScrapeAllCategories(ctx context.Context, progressChan chan ScrapeProgress) ([]models.StorageProduct, error) {
|
||||
allProducts := []models.StorageProduct{}
|
||||
|
||||
for _, category := range models.StorageCategories {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return allProducts, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
products, err := s.ScrapeCategory(ctx, category, progressChan)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
allProducts = append(allProducts, products...)
|
||||
}
|
||||
|
||||
return allProducts, nil
|
||||
}
|
||||
@@ -0,0 +1,709 @@
|
||||
package tui
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"bargain-scraper/internal/models"
|
||||
"bargain-scraper/internal/scraper"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/charmbracelet/lipgloss"
|
||||
)
|
||||
|
||||
var (
|
||||
headerStyle = lipgloss.NewStyle().Bold(true).Foreground(lipgloss.Color("#FFFFFF")).Background(lipgloss.Color("#1E40AF")).Padding(0, 1)
|
||||
subtleStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("#64748B"))
|
||||
selectedStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("#FFFFFF")).Background(lipgloss.Color("#3B82F6")).Padding(0, 1)
|
||||
normalStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("#E2E8F0")).Padding(0, 1)
|
||||
errorStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("#EF4444")).Bold(true)
|
||||
)
|
||||
|
||||
type SortOption struct {
|
||||
Name string
|
||||
Key string
|
||||
Order string
|
||||
}
|
||||
|
||||
type Model struct {
|
||||
products []models.StorageProduct
|
||||
filteredProducts []models.StorageProduct
|
||||
categories []models.Category
|
||||
selectedCategory int
|
||||
selectedProduct int
|
||||
searchQuery string
|
||||
isScraping bool
|
||||
progress scraper.ScrapeProgress
|
||||
err error
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
view string
|
||||
sortBy SortOption
|
||||
sortMenuIndex int
|
||||
ScrapeCallback func(models.Category)
|
||||
ExportCallback func(string) error
|
||||
lastExportError string
|
||||
}
|
||||
|
||||
var sortOptions = []SortOption{
|
||||
{Name: "Price (Low to High)", Key: "price", Order: "asc"},
|
||||
{Name: "Price (High to Low)", Key: "price", Order: "desc"},
|
||||
{Name: "Capacity (Small to Large)", Key: "capacity", Order: "asc"},
|
||||
{Name: "Capacity (Large to Small)", Key: "capacity", Order: "desc"},
|
||||
{Name: "Stock Status (In Stock First)", Key: "stock", Order: "desc"},
|
||||
{Name: "Name (A-Z)", Key: "name", Order: "asc"},
|
||||
{Name: "Name (Z-A)", Key: "name", Order: "desc"},
|
||||
}
|
||||
|
||||
func New() *Model {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
return &Model{
|
||||
products: []models.StorageProduct{},
|
||||
filteredProducts: []models.StorageProduct{},
|
||||
categories: models.StorageCategories,
|
||||
selectedCategory: 0,
|
||||
selectedProduct: 0,
|
||||
searchQuery: "",
|
||||
isScraping: false,
|
||||
view: "categories",
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Model) Init() tea.Cmd {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
switch msg := msg.(type) {
|
||||
case tea.KeyMsg:
|
||||
return m.handleKeyPress(msg)
|
||||
case scraper.ScrapeProgress:
|
||||
m.progress = msg
|
||||
return m, nil
|
||||
case ScrapingDoneMsg:
|
||||
m.isScraping = false
|
||||
if m.view == "products" {
|
||||
return m, nil
|
||||
}
|
||||
m.view = "products"
|
||||
return m, nil
|
||||
default:
|
||||
return m, nil
|
||||
}
|
||||
}
|
||||
|
||||
type ScrapingDoneMsg struct{}
|
||||
|
||||
func (m *Model) applySort() {
|
||||
if len(m.filteredProducts) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
sorted := make([]models.StorageProduct, len(m.filteredProducts))
|
||||
copy(sorted, m.filteredProducts)
|
||||
|
||||
switch m.sortBy.Key {
|
||||
case "price":
|
||||
sort.Slice(sorted, func(i, j int) bool {
|
||||
priceI := extractPriceValue(sorted[i].Price)
|
||||
priceJ := extractPriceValue(sorted[j].Price)
|
||||
if m.sortBy.Order == "asc" {
|
||||
return priceI < priceJ
|
||||
}
|
||||
return priceI > priceJ
|
||||
})
|
||||
case "capacity":
|
||||
sort.Slice(sorted, func(i, j int) bool {
|
||||
capI := extractCapacityValue(sorted[i].Capacity)
|
||||
capJ := extractCapacityValue(sorted[j].Capacity)
|
||||
if m.sortBy.Order == "asc" {
|
||||
return capI < capJ
|
||||
}
|
||||
return capI > capJ
|
||||
})
|
||||
case "stock":
|
||||
sort.Slice(sorted, func(i, j int) bool {
|
||||
stockI := sorted[i].StockStatus
|
||||
stockJ := sorted[j].StockStatus
|
||||
inStockI := strings.Contains(strings.ToLower(stockI), "in stock")
|
||||
inStockJ := strings.Contains(strings.ToLower(stockJ), "in stock")
|
||||
if inStockI && !inStockJ {
|
||||
return true
|
||||
}
|
||||
if !inStockI && inStockJ {
|
||||
return false
|
||||
}
|
||||
return sorted[i].Name < sorted[j].Name
|
||||
})
|
||||
case "name":
|
||||
sort.Slice(sorted, func(i, j int) bool {
|
||||
if m.sortBy.Order == "asc" {
|
||||
return sorted[i].Name < sorted[j].Name
|
||||
}
|
||||
return sorted[i].Name > sorted[j].Name
|
||||
})
|
||||
}
|
||||
|
||||
m.filteredProducts = sorted
|
||||
}
|
||||
|
||||
func extractPriceValue(priceStr string) float64 {
|
||||
priceStr = strings.ReplaceAll(priceStr, "£", "")
|
||||
priceStr = strings.ReplaceAll(priceStr, ",", "")
|
||||
parts := strings.Split(priceStr, " ")
|
||||
if len(parts) > 0 {
|
||||
priceStr = parts[0]
|
||||
}
|
||||
price, _ := strconv.ParseFloat(priceStr, 64)
|
||||
return price
|
||||
}
|
||||
|
||||
func extractCapacityValue(capStr string) float64 {
|
||||
capStr = strings.ToUpper(capStr)
|
||||
multiplier := 1.0
|
||||
if strings.Contains(capStr, "TB") {
|
||||
multiplier = 1000.0
|
||||
}
|
||||
capStr = strings.ReplaceAll(capStr, "TB", "")
|
||||
capStr = strings.ReplaceAll(capStr, "GB", "")
|
||||
capStr = strings.ReplaceAll(capStr, " ", "")
|
||||
cap, _ := strconv.ParseFloat(capStr, 64)
|
||||
return cap * multiplier
|
||||
}
|
||||
|
||||
func (m *Model) SetSort(option SortOption) {
|
||||
m.sortBy = option
|
||||
m.applySort()
|
||||
}
|
||||
|
||||
func (m *Model) View() string {
|
||||
if m.err != nil {
|
||||
return m.renderError()
|
||||
}
|
||||
|
||||
if m.isScraping {
|
||||
return m.renderProgress()
|
||||
}
|
||||
|
||||
switch m.view {
|
||||
case "categories":
|
||||
return m.renderCategories()
|
||||
case "products":
|
||||
return m.renderProducts()
|
||||
case "export":
|
||||
return m.renderExport()
|
||||
default:
|
||||
return m.renderCategories()
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Model) handleKeyPress(msg tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
if m.isScraping {
|
||||
if msg.String() == "esc" {
|
||||
m.CancelScraping()
|
||||
m.isScraping = false
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
switch m.view {
|
||||
case "categories":
|
||||
return m.handleCategoriesKeyPress(msg)
|
||||
case "products":
|
||||
return m.handleProductsKeyPress(msg)
|
||||
case "export":
|
||||
return m.handleExportKeyPress(msg)
|
||||
case "sort":
|
||||
return m.handleSortMenuKeyPress(msg)
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m *Model) handleCategoriesKeyPress(msg tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
switch msg.String() {
|
||||
case "up", "k":
|
||||
if m.selectedCategory > 0 {
|
||||
m.selectedCategory--
|
||||
}
|
||||
case "down", "j":
|
||||
if m.selectedCategory < len(m.categories)-1 {
|
||||
m.selectedCategory++
|
||||
}
|
||||
case "enter":
|
||||
m.view = "products"
|
||||
m.isScraping = true
|
||||
if m.ScrapeCallback != nil {
|
||||
go m.ScrapeCallback(m.categories[m.selectedCategory])
|
||||
}
|
||||
case "esc":
|
||||
return m, tea.Quit
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m *Model) handleProductsKeyPress(msg tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
switch msg.String() {
|
||||
case "up", "k":
|
||||
if m.selectedProduct > 0 {
|
||||
m.selectedProduct--
|
||||
}
|
||||
case "down", "j":
|
||||
if m.selectedProduct < len(m.filteredProducts)-1 {
|
||||
m.selectedProduct++
|
||||
}
|
||||
case "enter":
|
||||
if len(m.filteredProducts) > 0 && m.selectedProduct < len(m.filteredProducts) {
|
||||
url := m.filteredProducts[m.selectedProduct].URL
|
||||
if url != "" {
|
||||
go func() {
|
||||
cmd := exec.Command("xdg-open", url)
|
||||
cmd.Start()
|
||||
}()
|
||||
}
|
||||
}
|
||||
case "/":
|
||||
m.searchQuery = ""
|
||||
case "r":
|
||||
m.isScraping = true
|
||||
m.view = "products"
|
||||
case "e":
|
||||
m.view = "export"
|
||||
case "o":
|
||||
m.sortMenuIndex = 0
|
||||
m.view = "sort"
|
||||
case "p":
|
||||
m.SetSort(SortOption{Name: "Price (Low to High)", Key: "price", Order: "asc"})
|
||||
case "P":
|
||||
m.SetSort(SortOption{Name: "Price (High to Low)", Key: "price", Order: "desc"})
|
||||
case "c":
|
||||
m.SetSort(SortOption{Name: "Capacity (Small to Large)", Key: "capacity", Order: "asc"})
|
||||
case "C":
|
||||
m.SetSort(SortOption{Name: "Capacity (Large to Small)", Key: "capacity", Order: "desc"})
|
||||
case "s":
|
||||
m.SetSort(SortOption{Name: "Stock Status (In Stock First)", Key: "stock", Order: "desc"})
|
||||
case "esc":
|
||||
m.view = "categories"
|
||||
m.searchQuery = ""
|
||||
default:
|
||||
if len(msg.String()) == 1 {
|
||||
m.searchQuery += msg.String()
|
||||
m.applyFilter()
|
||||
}
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m *Model) handleExportKeyPress(msg tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
if m.ExportCallback == nil {
|
||||
m.view = "products"
|
||||
return m, nil
|
||||
}
|
||||
|
||||
var format string
|
||||
switch msg.String() {
|
||||
case "j", "J":
|
||||
format = "json"
|
||||
case "c", "C":
|
||||
format = "csv"
|
||||
case "p", "P":
|
||||
format = "print"
|
||||
case "esc":
|
||||
m.view = "products"
|
||||
return m, nil
|
||||
default:
|
||||
return m, nil
|
||||
}
|
||||
|
||||
err := m.ExportCallback(format)
|
||||
if err != nil {
|
||||
m.lastExportError = err.Error()
|
||||
} else {
|
||||
m.lastExportError = ""
|
||||
}
|
||||
|
||||
m.view = "products"
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m *Model) handleSortMenuKeyPress(msg tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
switch msg.String() {
|
||||
case "up", "k":
|
||||
if m.sortMenuIndex > 0 {
|
||||
m.sortMenuIndex--
|
||||
}
|
||||
case "down", "j":
|
||||
if m.sortMenuIndex < len(sortOptions) {
|
||||
m.sortMenuIndex++
|
||||
}
|
||||
case "enter":
|
||||
if m.sortMenuIndex >= 0 && m.sortMenuIndex < len(sortOptions) {
|
||||
m.SetSort(sortOptions[m.sortMenuIndex])
|
||||
}
|
||||
m.view = "products"
|
||||
case "esc":
|
||||
m.view = "products"
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m *Model) GetProducts() []models.StorageProduct {
|
||||
return m.products
|
||||
}
|
||||
|
||||
func (m *Model) GetFilteredProducts() []models.StorageProduct {
|
||||
return m.filteredProducts
|
||||
}
|
||||
|
||||
func (m *Model) GetCategories() []models.Category {
|
||||
return m.categories
|
||||
}
|
||||
|
||||
func (m *Model) GetSelectedCategory() int {
|
||||
return m.selectedCategory
|
||||
}
|
||||
|
||||
func (m *Model) GetSelectedProduct() int {
|
||||
return m.selectedProduct
|
||||
}
|
||||
|
||||
func (m *Model) GetSearchQuery() string {
|
||||
return m.searchQuery
|
||||
}
|
||||
|
||||
func (m *Model) IsScraping() bool {
|
||||
return m.isScraping
|
||||
}
|
||||
|
||||
func (m *Model) GetProgress() scraper.ScrapeProgress {
|
||||
return m.progress
|
||||
}
|
||||
|
||||
func (m *Model) GetError() error {
|
||||
return m.err
|
||||
}
|
||||
|
||||
func (m *Model) GetView() string {
|
||||
return m.view
|
||||
}
|
||||
|
||||
func (m *Model) SetProducts(products []models.StorageProduct) {
|
||||
m.products = products
|
||||
m.filteredProducts = products
|
||||
m.selectedProduct = 0
|
||||
}
|
||||
|
||||
func (m *Model) SetScraping(isScraping bool) {
|
||||
m.isScraping = isScraping
|
||||
}
|
||||
|
||||
func (m *Model) SetProgress(progress scraper.ScrapeProgress) {
|
||||
m.progress = progress
|
||||
}
|
||||
|
||||
func (m *Model) SetError(err error) {
|
||||
m.err = err
|
||||
}
|
||||
|
||||
func (m *Model) SetView(view string) {
|
||||
m.view = view
|
||||
}
|
||||
|
||||
func (m *Model) SetSelectedCategory(index int) {
|
||||
if index >= 0 && index < len(m.categories) {
|
||||
m.selectedCategory = index
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Model) SetSelectedProduct(index int) {
|
||||
if index >= 0 && index < len(m.filteredProducts) {
|
||||
m.selectedProduct = index
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Model) SetSearchQuery(query string) {
|
||||
m.searchQuery = query
|
||||
m.applyFilter()
|
||||
}
|
||||
|
||||
func (m *Model) applyFilter() {
|
||||
if m.searchQuery == "" {
|
||||
m.filteredProducts = m.products
|
||||
} else {
|
||||
query := toLower(m.searchQuery)
|
||||
var filtered []models.StorageProduct
|
||||
for _, p := range m.products {
|
||||
if contains(toLower(p.Name), query) ||
|
||||
contains(toLower(p.Capacity), query) ||
|
||||
contains(toLower(p.Brand), query) ||
|
||||
contains(toLower(p.Interface), query) ||
|
||||
contains(toLower(p.FormFactor), query) ||
|
||||
contains(toLower(p.PartNumber), query) {
|
||||
filtered = append(filtered, p)
|
||||
}
|
||||
}
|
||||
m.filteredProducts = filtered
|
||||
}
|
||||
m.selectedProduct = 0
|
||||
}
|
||||
|
||||
func toLower(s string) string {
|
||||
result := make([]rune, len(s))
|
||||
for i, r := range s {
|
||||
if r >= 'A' && r <= 'Z' {
|
||||
result[i] = r + 32
|
||||
} else {
|
||||
result[i] = r
|
||||
}
|
||||
}
|
||||
return string(result)
|
||||
}
|
||||
|
||||
func contains(s, substr string) bool {
|
||||
if len(substr) == 0 {
|
||||
return true
|
||||
}
|
||||
if len(s) < len(substr) {
|
||||
return false
|
||||
}
|
||||
for i := 0; i <= len(s)-len(substr); i++ {
|
||||
if s[i:i+len(substr)] == substr {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (m *Model) FilterByCategory(categoryIndex int) {
|
||||
if categoryIndex < 0 || categoryIndex >= len(m.categories) {
|
||||
return
|
||||
}
|
||||
|
||||
category := m.categories[categoryIndex]
|
||||
m.selectedCategory = categoryIndex
|
||||
|
||||
if category.Name == "All Categories" {
|
||||
m.filteredProducts = m.products
|
||||
} else {
|
||||
var filtered []models.StorageProduct
|
||||
for _, p := range m.products {
|
||||
if matchesCategory(p, category) {
|
||||
filtered = append(filtered, p)
|
||||
}
|
||||
}
|
||||
m.filteredProducts = filtered
|
||||
}
|
||||
m.selectedProduct = 0
|
||||
}
|
||||
|
||||
func matchesCategory(p models.StorageProduct, category models.Category) bool {
|
||||
catLower := toLower(category.Name)
|
||||
|
||||
if contains(catLower, "3.5") && p.FormFactor == "3.5\"" {
|
||||
return true
|
||||
}
|
||||
if contains(catLower, "2.5") && p.FormFactor == "2.5\"" {
|
||||
return true
|
||||
}
|
||||
if contains(catLower, "nvme") && contains(toLower(p.Interface), "nvme") {
|
||||
return true
|
||||
}
|
||||
if contains(catLower, "sata") && contains(toLower(p.Interface), "sata") {
|
||||
return true
|
||||
}
|
||||
if contains(catLower, "sas") && contains(toLower(p.Interface), "sas") {
|
||||
return true
|
||||
}
|
||||
if contains(catLower, "ssd") && (p.Interface == "SATA" || p.Interface == "SAS" || p.Interface == "NVMe") {
|
||||
return true
|
||||
}
|
||||
if contains(catLower, "hdd") && (p.Interface == "SATA" || p.Interface == "SAS") && p.Capacity != "" {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (m *Model) GetContext() context.Context {
|
||||
return m.ctx
|
||||
}
|
||||
|
||||
func (m *Model) CancelScraping() {
|
||||
if m.cancel != nil {
|
||||
m.cancel()
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Model) renderCategories() string {
|
||||
var result string
|
||||
result += headerStyle.Width(60).Render(" Select Storage Category ")
|
||||
result += "\n\n"
|
||||
|
||||
result += subtleStyle.Render("Use arrow keys to navigate, Enter to select, Esc to go back\n")
|
||||
result += "\n"
|
||||
|
||||
for i, cat := range m.categories {
|
||||
if i == m.selectedCategory {
|
||||
result += selectedStyle.Width(58).Render(" " + cat.Name)
|
||||
} else {
|
||||
result += normalStyle.Width(58).Render(" " + cat.Name)
|
||||
}
|
||||
result += "\n"
|
||||
}
|
||||
|
||||
result += "\n"
|
||||
result += subtleStyle.Render(fmt.Sprintf("Categories: %d | Press Enter to scrape selected category", len(m.categories)))
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func (m *Model) renderProducts() string {
|
||||
var result string
|
||||
|
||||
title := fmt.Sprintf(" Storage Products - %s ", m.categories[m.selectedCategory].Name)
|
||||
result += headerStyle.Width(80).Render(title)
|
||||
result += "\n\n"
|
||||
|
||||
if len(m.products) == 0 {
|
||||
result += subtleStyle.Render("No products scraped yet. Select a category and press Enter to scrape.\n")
|
||||
result += "\n"
|
||||
result += subtleStyle.Render("Press Esc to go back to category selection.\n")
|
||||
return result
|
||||
}
|
||||
|
||||
result += subtleStyle.Render(fmt.Sprintf("Search: %s | ", m.searchQuery))
|
||||
result += fmt.Sprintf("Showing: %d/%d products", len(m.filteredProducts), len(m.products))
|
||||
result += "\n\n"
|
||||
|
||||
result += m.renderProductTable()
|
||||
|
||||
result += "\n\n"
|
||||
result += subtleStyle.Render("Controls: ↑↓ Navigate | / Search | E Export | Esc Back | R Rescrape")
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func (m *Model) renderProductTable() string {
|
||||
if len(m.filteredProducts) == 0 {
|
||||
return " No products found matching your criteria."
|
||||
}
|
||||
|
||||
var result string
|
||||
|
||||
header := " # Name Capacity Interface FormFactor Price \n"
|
||||
result += headerStyle.Render(header)
|
||||
|
||||
start := m.selectedProduct - 5
|
||||
if start < 0 {
|
||||
start = 0
|
||||
}
|
||||
end := start + 15
|
||||
if end > len(m.filteredProducts) {
|
||||
end = len(m.filteredProducts)
|
||||
}
|
||||
|
||||
for i := start; i < end; i++ {
|
||||
p := m.filteredProducts[i]
|
||||
row := fmt.Sprintf(" %-4d %-35s %-10s %-8s %-10s %-12s\n",
|
||||
i+1,
|
||||
truncate(p.Name, 35),
|
||||
p.Capacity,
|
||||
p.Interface,
|
||||
p.FormFactor,
|
||||
truncate(p.Price, 12))
|
||||
|
||||
if i == m.selectedProduct {
|
||||
result += selectedStyle.Width(79).Render(row)
|
||||
} else {
|
||||
result += normalStyle.Width(79).Render(row)
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func (m *Model) renderProgress() string {
|
||||
var result string
|
||||
|
||||
result += headerStyle.Width(60).Render(" Scraping Progress ")
|
||||
result += "\n\n"
|
||||
|
||||
result += fmt.Sprintf(" Category: %s\n\n", m.progress.Category)
|
||||
result += fmt.Sprintf(" Status: %s\n\n", m.progress.Status)
|
||||
|
||||
if m.progress.Total > 0 {
|
||||
percent := float64(m.progress.Current) / float64(m.progress.Total) * 100
|
||||
barWidth := 40
|
||||
filled := int(float64(barWidth) * percent / 100)
|
||||
|
||||
bar := "["
|
||||
for i := 0; i < barWidth; i++ {
|
||||
if i < filled {
|
||||
bar += "="
|
||||
} else if i == filled {
|
||||
bar += ">"
|
||||
} else {
|
||||
bar += " "
|
||||
}
|
||||
}
|
||||
bar += "]"
|
||||
|
||||
result += fmt.Sprintf(" Progress: %s %d/%d (%.1f%%)\n\n", bar, m.progress.Current, m.progress.Total, percent)
|
||||
}
|
||||
|
||||
if m.progress.ProductName != "" {
|
||||
result += fmt.Sprintf(" Current: %s\n", truncate(m.progress.ProductName, 50))
|
||||
}
|
||||
|
||||
result += "\n\n"
|
||||
result += subtleStyle.Render("Press Esc to cancel...")
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func (m *Model) renderError() string {
|
||||
var result string
|
||||
|
||||
result += headerStyle.Width(60).Render(" Error ")
|
||||
result += "\n\n"
|
||||
|
||||
result += errorStyle.Render(fmt.Sprintf(" %v\n", m.err))
|
||||
result += "\n\n"
|
||||
result += subtleStyle.Render("Press any key to continue...")
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func (m *Model) renderExport() string {
|
||||
var result string
|
||||
|
||||
result += headerStyle.Width(60).Render(" Export Data ")
|
||||
result += "\n\n"
|
||||
|
||||
result += fmt.Sprintf(" Products: %d\n", len(m.products))
|
||||
result += fmt.Sprintf(" Filtered: %d\n\n", len(m.filteredProducts))
|
||||
|
||||
result += " Export Options:\n"
|
||||
result += " [J] Export to JSON\n"
|
||||
result += " [C] Export to CSV\n"
|
||||
result += " [P] Print to console\n"
|
||||
result += " [Esc] Go back\n"
|
||||
|
||||
result += "\n\n"
|
||||
result += subtleStyle.Render("Select an option...")
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func truncate(s string, maxLen int) string {
|
||||
if len(s) <= maxLen {
|
||||
return s
|
||||
}
|
||||
return s[:maxLen-3] + "..."
|
||||
}
|
||||
Reference in New Issue
Block a user