initial commit
This commit is contained in:
+42
@@ -0,0 +1,42 @@
|
||||
# Binaries
|
||||
*.exe
|
||||
*.exe~
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
bargain-scraper
|
||||
/test*
|
||||
|
||||
# Test binary
|
||||
*.test
|
||||
|
||||
# Output of the go coverage tool
|
||||
*.out
|
||||
|
||||
# Go workspace file
|
||||
go.work
|
||||
|
||||
# Dependency directories
|
||||
vendor/
|
||||
|
||||
# IDE
|
||||
.idea/
|
||||
.vscode/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Build output
|
||||
/bin/
|
||||
/dist/
|
||||
|
||||
# Log files
|
||||
*.log
|
||||
|
||||
# CSV/JSON exports (generated at runtime)
|
||||
*.csv
|
||||
*.json
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"bargain-scraper/internal/exporter"
|
||||
"bargain-scraper/internal/models"
|
||||
"bargain-scraper/internal/scraper"
|
||||
"bargain-scraper/internal/tui"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
)
|
||||
|
||||
func main() {
|
||||
log.SetFlags(0)
|
||||
log.SetOutput(os.Stderr)
|
||||
|
||||
if len(os.Args) > 1 && os.Args[1] == "--help" {
|
||||
printHelp()
|
||||
return
|
||||
}
|
||||
|
||||
app := newApp()
|
||||
if err := app.run(); err != nil {
|
||||
log.Fatalf("Error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func printHelp() {
|
||||
fmt.Println("Bargain Hardware Storage Scraper")
|
||||
fmt.Println("")
|
||||
fmt.Println("Usage:")
|
||||
fmt.Println(" bargain-scraper Start the interactive TUI")
|
||||
fmt.Println(" bargain-scraper --help Show this help message")
|
||||
fmt.Println("")
|
||||
fmt.Println("In the TUI:")
|
||||
fmt.Println(" ↑/↓ or j/k Navigate")
|
||||
fmt.Println(" Enter Select/scrape category")
|
||||
fmt.Println(" / Start search")
|
||||
fmt.Println(" E Export data")
|
||||
fmt.Println(" R Rescrape current category")
|
||||
fmt.Println(" Esc Go back")
|
||||
}
|
||||
|
||||
type app struct {
|
||||
model *tui.Model
|
||||
scraper *scraper.Scraper
|
||||
exp *exporter.Exporter
|
||||
progressChan chan scraper.ScrapeProgress
|
||||
doneChan chan struct{}
|
||||
}
|
||||
|
||||
func newApp() *app {
|
||||
return &app{
|
||||
model: tui.New(),
|
||||
exp: exporter.New(),
|
||||
progressChan: make(chan scraper.ScrapeProgress, 100),
|
||||
doneChan: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (a *app) run() error {
|
||||
s, err := scraper.New()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create scraper: %w", err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
a.scraper = s
|
||||
|
||||
a.model.ScrapeCallback = func(category models.Category) {
|
||||
go a.scrapeCategory(category)
|
||||
}
|
||||
|
||||
a.model.ExportCallback = func(format string) error {
|
||||
switch format {
|
||||
case "json":
|
||||
return a.exp.ExportToJSON(a.model.GetFilteredProducts(), "")
|
||||
case "csv":
|
||||
return a.exp.ExportToCSV(a.model.GetFilteredProducts(), "")
|
||||
case "print":
|
||||
fmt.Print(a.exp.PrintToConsole(a.model.GetFilteredProducts()))
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
p := tea.NewProgram(a.model, tea.WithAltScreen())
|
||||
|
||||
go a.handleProgress(p)
|
||||
|
||||
if _, err := p.Run(); err != nil {
|
||||
return fmt.Errorf("error running TUI: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *app) handleProgress(p *tea.Program) {
|
||||
for {
|
||||
select {
|
||||
case progress := <-a.progressChan:
|
||||
a.model.SetProgress(progress)
|
||||
p.Send(progress)
|
||||
case <-a.doneChan:
|
||||
p.Send(ScrapingDoneMsg{})
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type ScrapingDoneMsg struct{}
|
||||
|
||||
func (a *app) scrapeCategory(category models.Category) {
|
||||
products, err := a.scraper.ScrapeCategory(a.model.GetContext(), category, a.progressChan)
|
||||
|
||||
a.model.SetScraping(false)
|
||||
|
||||
if err != nil {
|
||||
a.model.SetError(err)
|
||||
}
|
||||
|
||||
a.model.SetProducts(products)
|
||||
a.doneChan <- struct{}{}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
module bargain-scraper
|
||||
|
||||
go 1.21
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.9.2
|
||||
github.com/charmbracelet/bubbletea v1.3.1
|
||||
github.com/charmbracelet/lipgloss v1.0.0
|
||||
github.com/google/uuid v1.6.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/andybalholm/cascadia v1.3.2 // indirect
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||
github.com/charmbracelet/x/ansi v0.8.0 // indirect
|
||||
github.com/charmbracelet/x/term v0.2.1 // indirect
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-localereader v0.0.1 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.16 // indirect
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
|
||||
github.com/muesli/cancelreader v0.2.2 // indirect
|
||||
github.com/muesli/termenv v0.15.2 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
golang.org/x/net v0.24.0 // indirect
|
||||
golang.org/x/sync v0.11.0 // indirect
|
||||
golang.org/x/sys v0.30.0 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,79 @@
|
||||
github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE=
|
||||
github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
|
||||
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
|
||||
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
|
||||
github.com/charmbracelet/bubbletea v1.3.1 h1:3qoZgqwtq2HUK5mvMzSMwPBnfz4yhwrmi7ORvwcafd8=
|
||||
github.com/charmbracelet/bubbletea v1.3.1/go.mod h1:dtcUCyCGEX3g9tosuYiut3MXgY/Jsv9nKVdibKKRRXo=
|
||||
github.com/charmbracelet/lipgloss v1.0.0 h1:O7VkGDvqEdGi93X+DeqsQ7PKHDgtQfF8j8/O2qFMQNg=
|
||||
github.com/charmbracelet/lipgloss v1.0.0/go.mod h1:U5fy9Z+C38obMs+T+tJqst9VGzlOYGj4ri9reL3qUlo=
|
||||
github.com/charmbracelet/x/ansi v0.8.0 h1:9GTq3xq9caJW8ZrBTe0LIe2fvfLR/bYXKTx2llXn7xE=
|
||||
github.com/charmbracelet/x/ansi v0.8.0/go.mod h1:wdYl/ONOLHLIVmQaxbIYEC/cRKOQyjTkowiI4blgS9Q=
|
||||
github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ=
|
||||
github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg=
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
|
||||
github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
|
||||
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
|
||||
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI=
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo=
|
||||
github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=
|
||||
github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
|
||||
github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo=
|
||||
github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
|
||||
golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
|
||||
golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
|
||||
golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
|
||||
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
@@ -0,0 +1,151 @@
|
||||
package exporter
|
||||
|
||||
import (
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"bargain-scraper/internal/models"
|
||||
)
|
||||
|
||||
type Exporter struct{}
|
||||
|
||||
func New() *Exporter {
|
||||
return &Exporter{}
|
||||
}
|
||||
|
||||
func (e *Exporter) ExportToJSON(products []models.StorageProduct, filename string) error {
|
||||
if filename == "" {
|
||||
filename = fmt.Sprintf("storage_products_%s.json", time.Now().Format("2006-01-02_150405"))
|
||||
}
|
||||
|
||||
data, err := json.MarshalIndent(products, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal JSON: %w", err)
|
||||
}
|
||||
|
||||
err = os.WriteFile(filename, data, 0644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write JSON file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *Exporter) ExportToCSV(products []models.StorageProduct, filename string) error {
|
||||
if filename == "" {
|
||||
filename = fmt.Sprintf("storage_products_%s.csv", time.Now().Format("2006-01-02_150405"))
|
||||
}
|
||||
|
||||
file, err := os.Create(filename)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create CSV file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
writer := csv.NewWriter(file)
|
||||
defer writer.Flush()
|
||||
|
||||
headers := []string{"ID", "Name", "Price", "Capacity", "Interface", "FormFactor", "Brand", "PartNumber", "StockStatus", "URL", "ScrapedAt"}
|
||||
if err := writer.Write(headers); err != nil {
|
||||
return fmt.Errorf("failed to write CSV headers: %w", err)
|
||||
}
|
||||
|
||||
for _, p := range products {
|
||||
row := []string{
|
||||
p.ID,
|
||||
p.Name,
|
||||
p.Price,
|
||||
p.Capacity,
|
||||
p.Interface,
|
||||
p.FormFactor,
|
||||
p.Brand,
|
||||
p.PartNumber,
|
||||
p.StockStatus,
|
||||
p.URL,
|
||||
p.ScrapedAt.Format(time.RFC3339),
|
||||
}
|
||||
if err := writer.Write(row); err != nil {
|
||||
return fmt.Errorf("failed to write CSV row: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *Exporter) PrintToConsole(products []models.StorageProduct) string {
|
||||
var b strings.Builder
|
||||
|
||||
b.WriteString("\n")
|
||||
b.WriteString("═══════════════════════════════════════════════════════════════════════════════\n")
|
||||
b.WriteString(fmt.Sprintf(" STORAGE PRODUCTS (%d items) \n", len(products)))
|
||||
b.WriteString("═══════════════════════════════════════════════════════════════════════════════\n")
|
||||
b.WriteString("\n")
|
||||
|
||||
headerFmt := " %-4s | %-40s | %-10s | %-8s | %-6s | %-10s | %-12s\n"
|
||||
separator := "──────┼──────────────────────────────────────────┬────────────┬──────────┼────────┼────────────┼──────────────\n"
|
||||
|
||||
b.WriteString(fmt.Sprintf(headerFmt, "#", "Name", "Capacity", "Interface", "Factor", "Stock", "Price"))
|
||||
b.WriteString(separator)
|
||||
|
||||
for i, p := range products {
|
||||
name := p.Name
|
||||
if len(name) > 40 {
|
||||
name = name[:37] + "..."
|
||||
}
|
||||
|
||||
capacity := p.Capacity
|
||||
if capacity == "" {
|
||||
capacity = "-"
|
||||
}
|
||||
|
||||
interface_ := p.Interface
|
||||
if interface_ == "" {
|
||||
interface_ = "-"
|
||||
}
|
||||
|
||||
formFactor := p.FormFactor
|
||||
if formFactor == "" {
|
||||
formFactor = "-"
|
||||
}
|
||||
|
||||
stock := p.StockStatus
|
||||
if stock == "" {
|
||||
stock = "-"
|
||||
}
|
||||
|
||||
price := p.Price
|
||||
if price == "" {
|
||||
price = "-"
|
||||
}
|
||||
|
||||
b.WriteString(fmt.Sprintf(headerFmt,
|
||||
fmt.Sprintf("%d", i+1),
|
||||
name,
|
||||
capacity,
|
||||
interface_,
|
||||
formFactor,
|
||||
stock,
|
||||
price,
|
||||
))
|
||||
}
|
||||
|
||||
b.WriteString("\n")
|
||||
b.WriteString(fmt.Sprintf("Total: %d products\n", len(products)))
|
||||
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func (e *Exporter) Export(products []models.StorageProduct, format string, filename string) error {
|
||||
switch strings.ToLower(format) {
|
||||
case "json":
|
||||
return e.ExportToJSON(products, filename)
|
||||
case "csv":
|
||||
return e.ExportToCSV(products, filename)
|
||||
default:
|
||||
return fmt.Errorf("unsupported format: %s (use 'json' or 'csv')", format)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package models
|
||||
|
||||
import "time"
|
||||
|
||||
type StorageProduct struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Price string `json:"price"`
|
||||
Capacity string `json:"capacity"`
|
||||
Interface string `json:"interface"`
|
||||
FormFactor string `json:"form_factor"`
|
||||
Brand string `json:"brand"`
|
||||
PartNumber string `json:"part_number"`
|
||||
StockStatus string `json:"stock_status"`
|
||||
URL string `json:"url"`
|
||||
ScrapedAt time.Time `json:"scraped_at"`
|
||||
}
|
||||
|
||||
type Category struct {
|
||||
Name string
|
||||
URL string
|
||||
Description string
|
||||
}
|
||||
|
||||
var StorageCategories = []Category{
|
||||
{
|
||||
Name: "3.5\" SATA HDDs",
|
||||
URL: "https://www.bargainhardware.co.uk/components/enterprise-hdds-sdds-storage/large-form-factor-sata-drives",
|
||||
Description: "Large Form Factor SATA Hard Drives",
|
||||
},
|
||||
{
|
||||
Name: "3.5\" SAS HDDs",
|
||||
URL: "https://www.bargainhardware.co.uk/components/enterprise-hdds-sdds-storage/large-form-factor-sas-drives",
|
||||
Description: "Large Form Factor SAS Hard Drives",
|
||||
},
|
||||
{
|
||||
Name: "2.5\" SATA HDDs",
|
||||
URL: "https://www.bargainhardware.co.uk/components/enterprise-hdds-sdds-storage/small-form-factor-sata-drives",
|
||||
Description: "Small Form Factor SATA Hard Drives",
|
||||
},
|
||||
{
|
||||
Name: "2.5\" SAS HDDs",
|
||||
URL: "https://www.bargainhardware.co.uk/components/enterprise-hdds-sdds-storage/small-form-factor-sas-drives",
|
||||
Description: "Small Form Factor SAS Hard Drives",
|
||||
},
|
||||
{
|
||||
Name: "SATA SSDs (2.5\")",
|
||||
URL: "https://www.bargainhardware.co.uk/components/enterprise-hdds-sdds-storage/sata-ssds",
|
||||
Description: "SATA Solid State Drives",
|
||||
},
|
||||
{
|
||||
Name: "SAS SSDs",
|
||||
URL: "https://www.bargainhardware.co.uk/components/enterprise-hdds-sdds-storage/sas-ssd",
|
||||
Description: "SAS Solid State Drives",
|
||||
},
|
||||
{
|
||||
Name: "NVMe SSDs",
|
||||
URL: "https://www.bargainhardware.co.uk/components/enterprise-hdds-sdds-storage/enterprise-nvme-drives",
|
||||
Description: "NVMe Solid State Drives",
|
||||
},
|
||||
{
|
||||
Name: "M.2 / Other SSDs",
|
||||
URL: "https://www.bargainhardware.co.uk/components/hard-drives-and-solid-state-drives/nvme-ssds",
|
||||
Description: "M.2 and Other SSD Form Factors",
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
package scraper
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"bargain-scraper/internal/models"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type Scraper struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
type ScrapeProgress struct {
|
||||
Category string
|
||||
Current int
|
||||
Total int
|
||||
Status string
|
||||
ProductName string
|
||||
}
|
||||
|
||||
func New() (*Scraper, error) {
|
||||
return &Scraper{
|
||||
client: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Scraper) Close() {
|
||||
}
|
||||
|
||||
func (s *Scraper) ScrapeCategory(ctx context.Context, category models.Category, progressChan chan ScrapeProgress) ([]models.StorageProduct, error) {
|
||||
products := []models.StorageProduct{}
|
||||
|
||||
progressChan <- ScrapeProgress{
|
||||
Category: category.Name,
|
||||
Current: 0,
|
||||
Total: 0,
|
||||
Status: "Fetching page...",
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", category.URL, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
|
||||
req.Header.Set("Accept-Language", "en-US,en;q=0.5")
|
||||
|
||||
resp, err := s.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch page: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("failed to fetch page: status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse HTML: %w", err)
|
||||
}
|
||||
|
||||
productItems := doc.Find("li.product-item")
|
||||
total := productItems.Length()
|
||||
|
||||
progressChan <- ScrapeProgress{
|
||||
Category: category.Name,
|
||||
Current: 0,
|
||||
Total: total,
|
||||
Status: "Parsing products...",
|
||||
}
|
||||
|
||||
productItems.Each(func(i int, sel *goquery.Selection) {
|
||||
product := s.parseProductItem(sel, category)
|
||||
if product.Name != "" {
|
||||
product.ID = uuid.New().String()
|
||||
product.ScrapedAt = time.Now()
|
||||
products = append(products, product)
|
||||
}
|
||||
|
||||
progressChan <- ScrapeProgress{
|
||||
Category: category.Name,
|
||||
Current: i + 1,
|
||||
Total: total,
|
||||
Status: "Parsing products...",
|
||||
ProductName: product.Name,
|
||||
}
|
||||
})
|
||||
|
||||
pageNum := 2
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return products, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
nextPageURL := fmt.Sprintf("%s?p=%d", category.URL, pageNum)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", nextPageURL, nil)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
|
||||
|
||||
resp, err := s.client.Do(req)
|
||||
if err != nil || resp.StatusCode != 200 {
|
||||
break
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
|
||||
nextItems := doc.Find("li.product-item")
|
||||
if nextItems.Length() == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
progressChan <- ScrapeProgress{
|
||||
Category: category.Name,
|
||||
Current: len(products),
|
||||
Total: len(products) + nextItems.Length(),
|
||||
Status: fmt.Sprintf("Parsing page %d...", pageNum),
|
||||
ProductName: "",
|
||||
}
|
||||
|
||||
nextItems.Each(func(i int, sel *goquery.Selection) {
|
||||
product := s.parseProductItem(sel, category)
|
||||
if product.Name != "" {
|
||||
product.ID = uuid.New().String()
|
||||
product.ScrapedAt = time.Now()
|
||||
products = append(products, product)
|
||||
}
|
||||
})
|
||||
|
||||
pageNum++
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
|
||||
return products, nil
|
||||
}
|
||||
|
||||
func (s *Scraper) parseProductItem(sel *goquery.Selection, category models.Category) models.StorageProduct {
|
||||
product := models.StorageProduct{
|
||||
FormFactor: extractFormFactor(category.Name),
|
||||
Interface: extractInterface(category.Name),
|
||||
}
|
||||
|
||||
link := sel.Find("a.product-item-link")
|
||||
if href, ok := link.Attr("href"); ok {
|
||||
product.URL = href
|
||||
}
|
||||
|
||||
product.Name = strings.TrimSpace(link.Text())
|
||||
if product.Name == "" {
|
||||
product.Name = strings.TrimSpace(sel.Find(".product-item-name").Text())
|
||||
}
|
||||
|
||||
price := sel.Find(".price").First().Text()
|
||||
product.Price = strings.TrimSpace(price)
|
||||
if product.Price == "" {
|
||||
product.Price = sel.Find("[data-price-type='finalPrice']").First().Text()
|
||||
}
|
||||
|
||||
desc := sel.Find(".product-item-description").Text()
|
||||
|
||||
capacity := extractCapacityFromText(desc)
|
||||
if capacity == "" {
|
||||
capacity = extractCapacityFromText(product.Name)
|
||||
}
|
||||
product.Capacity = capacity
|
||||
|
||||
product.Brand = extractBrandFromText(desc, product.Name)
|
||||
if product.Brand == "" {
|
||||
product.Brand = inferBrand(product.Name)
|
||||
}
|
||||
|
||||
stockText := sel.Find(".stock span").Text()
|
||||
if stockText == "" {
|
||||
stockText = sel.Find(".stock.available").Text()
|
||||
}
|
||||
product.StockStatus = strings.TrimSpace(stockText)
|
||||
if product.StockStatus == "" {
|
||||
if sel.Find(".stock.not-available").Length() > 0 {
|
||||
product.StockStatus = "Out of stock"
|
||||
} else if sel.Find(".stock.available").Length() > 0 {
|
||||
product.StockStatus = "In stock"
|
||||
}
|
||||
}
|
||||
|
||||
return product
|
||||
}
|
||||
|
||||
func extractFormFactor(categoryName string) string {
|
||||
lower := strings.ToLower(categoryName)
|
||||
if strings.Contains(lower, "3.5") || strings.Contains(lower, "lff") {
|
||||
return "3.5\""
|
||||
}
|
||||
if strings.Contains(lower, "2.5") || strings.Contains(lower, "sff") {
|
||||
return "2.5\""
|
||||
}
|
||||
if strings.Contains(lower, "m.2") {
|
||||
return "M.2"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func extractInterface(categoryName string) string {
|
||||
lower := strings.ToLower(categoryName)
|
||||
if strings.Contains(lower, "nvme") {
|
||||
return "NVMe"
|
||||
}
|
||||
if strings.Contains(lower, "sas") {
|
||||
return "SAS"
|
||||
}
|
||||
if strings.Contains(lower, "sata") {
|
||||
return "SATA"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func extractCapacityFromText(text string) string {
|
||||
patterns := []string{
|
||||
`(?i)capacity[:\s]*(\d+(?:\.\d+)?\s*TB)`,
|
||||
`(?i)capacity[:\s]*(\d+(?:\.\d+)?\s*GB)`,
|
||||
`(\d+(?:\.\d+)?\s*TB)`,
|
||||
`(\d+(?:\.\d+)?\s*GB)`,
|
||||
`(960GB|480GB|1\.92TB|3\.84TB|7\.68TB|15\.36TB|31\.44TB|1TB|2TB|4TB|8TB|12TB|14TB|16TB|18TB|20TB|22TB|24TB|26TB|28TB|30TB|32TB)`,
|
||||
}
|
||||
|
||||
for _, pattern := range patterns {
|
||||
re := regexp.MustCompile(pattern)
|
||||
matches := re.FindStringSubmatch(text)
|
||||
if len(matches) > 1 {
|
||||
return strings.ToUpper(matches[1])
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func extractBrandFromText(text, name string) string {
|
||||
brands := []string{"Dell", "HP", "HPE", "Seagate", "Western Digital", "WD", "Samsung", "Micron", "Intel", "Toshiba", "Hitachi", "IBM", "Lenovo", "Supermicro", "Generic", "Stec", "White Label"}
|
||||
|
||||
textLower := strings.ToLower(text)
|
||||
nameLower := strings.ToLower(name)
|
||||
|
||||
for _, brand := range brands {
|
||||
if strings.Contains(textLower, strings.ToLower(brand)) || strings.Contains(nameLower, strings.ToLower(brand)) {
|
||||
if brand == "WD" {
|
||||
return "Western Digital"
|
||||
}
|
||||
if brand == "HPE" {
|
||||
return "HP"
|
||||
}
|
||||
return brand
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func inferBrand(name string) string {
|
||||
if name == "" {
|
||||
return ""
|
||||
}
|
||||
brands := []string{"Dell", "HP", "HPE", "Seagate", "Western Digital", "Samsung", "Micron", "Intel", "Toshiba", "Hitachi", "IBM", "Lenovo", "Supermicro", "Generic", "Stec", "White Label"}
|
||||
lower := strings.ToLower(name)
|
||||
for _, brand := range brands {
|
||||
if strings.Contains(lower, strings.ToLower(brand)) {
|
||||
if brand == "WD" {
|
||||
return "Western Digital"
|
||||
}
|
||||
if brand == "HPE" {
|
||||
return "HP"
|
||||
}
|
||||
return brand
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (s *Scraper) ScrapeAllCategories(ctx context.Context, progressChan chan ScrapeProgress) ([]models.StorageProduct, error) {
|
||||
allProducts := []models.StorageProduct{}
|
||||
|
||||
for _, category := range models.StorageCategories {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return allProducts, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
products, err := s.ScrapeCategory(ctx, category, progressChan)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
allProducts = append(allProducts, products...)
|
||||
}
|
||||
|
||||
return allProducts, nil
|
||||
}
|
||||
@@ -0,0 +1,709 @@
|
||||
package tui
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"bargain-scraper/internal/models"
|
||||
"bargain-scraper/internal/scraper"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/charmbracelet/lipgloss"
|
||||
)
|
||||
|
||||
var (
|
||||
headerStyle = lipgloss.NewStyle().Bold(true).Foreground(lipgloss.Color("#FFFFFF")).Background(lipgloss.Color("#1E40AF")).Padding(0, 1)
|
||||
subtleStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("#64748B"))
|
||||
selectedStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("#FFFFFF")).Background(lipgloss.Color("#3B82F6")).Padding(0, 1)
|
||||
normalStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("#E2E8F0")).Padding(0, 1)
|
||||
errorStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("#EF4444")).Bold(true)
|
||||
)
|
||||
|
||||
type SortOption struct {
|
||||
Name string
|
||||
Key string
|
||||
Order string
|
||||
}
|
||||
|
||||
type Model struct {
|
||||
products []models.StorageProduct
|
||||
filteredProducts []models.StorageProduct
|
||||
categories []models.Category
|
||||
selectedCategory int
|
||||
selectedProduct int
|
||||
searchQuery string
|
||||
isScraping bool
|
||||
progress scraper.ScrapeProgress
|
||||
err error
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
view string
|
||||
sortBy SortOption
|
||||
sortMenuIndex int
|
||||
ScrapeCallback func(models.Category)
|
||||
ExportCallback func(string) error
|
||||
lastExportError string
|
||||
}
|
||||
|
||||
var sortOptions = []SortOption{
|
||||
{Name: "Price (Low to High)", Key: "price", Order: "asc"},
|
||||
{Name: "Price (High to Low)", Key: "price", Order: "desc"},
|
||||
{Name: "Capacity (Small to Large)", Key: "capacity", Order: "asc"},
|
||||
{Name: "Capacity (Large to Small)", Key: "capacity", Order: "desc"},
|
||||
{Name: "Stock Status (In Stock First)", Key: "stock", Order: "desc"},
|
||||
{Name: "Name (A-Z)", Key: "name", Order: "asc"},
|
||||
{Name: "Name (Z-A)", Key: "name", Order: "desc"},
|
||||
}
|
||||
|
||||
func New() *Model {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
return &Model{
|
||||
products: []models.StorageProduct{},
|
||||
filteredProducts: []models.StorageProduct{},
|
||||
categories: models.StorageCategories,
|
||||
selectedCategory: 0,
|
||||
selectedProduct: 0,
|
||||
searchQuery: "",
|
||||
isScraping: false,
|
||||
view: "categories",
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Model) Init() tea.Cmd {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
switch msg := msg.(type) {
|
||||
case tea.KeyMsg:
|
||||
return m.handleKeyPress(msg)
|
||||
case scraper.ScrapeProgress:
|
||||
m.progress = msg
|
||||
return m, nil
|
||||
case ScrapingDoneMsg:
|
||||
m.isScraping = false
|
||||
if m.view == "products" {
|
||||
return m, nil
|
||||
}
|
||||
m.view = "products"
|
||||
return m, nil
|
||||
default:
|
||||
return m, nil
|
||||
}
|
||||
}
|
||||
|
||||
type ScrapingDoneMsg struct{}
|
||||
|
||||
func (m *Model) applySort() {
|
||||
if len(m.filteredProducts) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
sorted := make([]models.StorageProduct, len(m.filteredProducts))
|
||||
copy(sorted, m.filteredProducts)
|
||||
|
||||
switch m.sortBy.Key {
|
||||
case "price":
|
||||
sort.Slice(sorted, func(i, j int) bool {
|
||||
priceI := extractPriceValue(sorted[i].Price)
|
||||
priceJ := extractPriceValue(sorted[j].Price)
|
||||
if m.sortBy.Order == "asc" {
|
||||
return priceI < priceJ
|
||||
}
|
||||
return priceI > priceJ
|
||||
})
|
||||
case "capacity":
|
||||
sort.Slice(sorted, func(i, j int) bool {
|
||||
capI := extractCapacityValue(sorted[i].Capacity)
|
||||
capJ := extractCapacityValue(sorted[j].Capacity)
|
||||
if m.sortBy.Order == "asc" {
|
||||
return capI < capJ
|
||||
}
|
||||
return capI > capJ
|
||||
})
|
||||
case "stock":
|
||||
sort.Slice(sorted, func(i, j int) bool {
|
||||
stockI := sorted[i].StockStatus
|
||||
stockJ := sorted[j].StockStatus
|
||||
inStockI := strings.Contains(strings.ToLower(stockI), "in stock")
|
||||
inStockJ := strings.Contains(strings.ToLower(stockJ), "in stock")
|
||||
if inStockI && !inStockJ {
|
||||
return true
|
||||
}
|
||||
if !inStockI && inStockJ {
|
||||
return false
|
||||
}
|
||||
return sorted[i].Name < sorted[j].Name
|
||||
})
|
||||
case "name":
|
||||
sort.Slice(sorted, func(i, j int) bool {
|
||||
if m.sortBy.Order == "asc" {
|
||||
return sorted[i].Name < sorted[j].Name
|
||||
}
|
||||
return sorted[i].Name > sorted[j].Name
|
||||
})
|
||||
}
|
||||
|
||||
m.filteredProducts = sorted
|
||||
}
|
||||
|
||||
func extractPriceValue(priceStr string) float64 {
|
||||
priceStr = strings.ReplaceAll(priceStr, "£", "")
|
||||
priceStr = strings.ReplaceAll(priceStr, ",", "")
|
||||
parts := strings.Split(priceStr, " ")
|
||||
if len(parts) > 0 {
|
||||
priceStr = parts[0]
|
||||
}
|
||||
price, _ := strconv.ParseFloat(priceStr, 64)
|
||||
return price
|
||||
}
|
||||
|
||||
func extractCapacityValue(capStr string) float64 {
|
||||
capStr = strings.ToUpper(capStr)
|
||||
multiplier := 1.0
|
||||
if strings.Contains(capStr, "TB") {
|
||||
multiplier = 1000.0
|
||||
}
|
||||
capStr = strings.ReplaceAll(capStr, "TB", "")
|
||||
capStr = strings.ReplaceAll(capStr, "GB", "")
|
||||
capStr = strings.ReplaceAll(capStr, " ", "")
|
||||
cap, _ := strconv.ParseFloat(capStr, 64)
|
||||
return cap * multiplier
|
||||
}
|
||||
|
||||
func (m *Model) SetSort(option SortOption) {
|
||||
m.sortBy = option
|
||||
m.applySort()
|
||||
}
|
||||
|
||||
func (m *Model) View() string {
|
||||
if m.err != nil {
|
||||
return m.renderError()
|
||||
}
|
||||
|
||||
if m.isScraping {
|
||||
return m.renderProgress()
|
||||
}
|
||||
|
||||
switch m.view {
|
||||
case "categories":
|
||||
return m.renderCategories()
|
||||
case "products":
|
||||
return m.renderProducts()
|
||||
case "export":
|
||||
return m.renderExport()
|
||||
default:
|
||||
return m.renderCategories()
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Model) handleKeyPress(msg tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
if m.isScraping {
|
||||
if msg.String() == "esc" {
|
||||
m.CancelScraping()
|
||||
m.isScraping = false
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
switch m.view {
|
||||
case "categories":
|
||||
return m.handleCategoriesKeyPress(msg)
|
||||
case "products":
|
||||
return m.handleProductsKeyPress(msg)
|
||||
case "export":
|
||||
return m.handleExportKeyPress(msg)
|
||||
case "sort":
|
||||
return m.handleSortMenuKeyPress(msg)
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m *Model) handleCategoriesKeyPress(msg tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
switch msg.String() {
|
||||
case "up", "k":
|
||||
if m.selectedCategory > 0 {
|
||||
m.selectedCategory--
|
||||
}
|
||||
case "down", "j":
|
||||
if m.selectedCategory < len(m.categories)-1 {
|
||||
m.selectedCategory++
|
||||
}
|
||||
case "enter":
|
||||
m.view = "products"
|
||||
m.isScraping = true
|
||||
if m.ScrapeCallback != nil {
|
||||
go m.ScrapeCallback(m.categories[m.selectedCategory])
|
||||
}
|
||||
case "esc":
|
||||
return m, tea.Quit
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m *Model) handleProductsKeyPress(msg tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
switch msg.String() {
|
||||
case "up", "k":
|
||||
if m.selectedProduct > 0 {
|
||||
m.selectedProduct--
|
||||
}
|
||||
case "down", "j":
|
||||
if m.selectedProduct < len(m.filteredProducts)-1 {
|
||||
m.selectedProduct++
|
||||
}
|
||||
case "enter":
|
||||
if len(m.filteredProducts) > 0 && m.selectedProduct < len(m.filteredProducts) {
|
||||
url := m.filteredProducts[m.selectedProduct].URL
|
||||
if url != "" {
|
||||
go func() {
|
||||
cmd := exec.Command("xdg-open", url)
|
||||
cmd.Start()
|
||||
}()
|
||||
}
|
||||
}
|
||||
case "/":
|
||||
m.searchQuery = ""
|
||||
case "r":
|
||||
m.isScraping = true
|
||||
m.view = "products"
|
||||
case "e":
|
||||
m.view = "export"
|
||||
case "o":
|
||||
m.sortMenuIndex = 0
|
||||
m.view = "sort"
|
||||
case "p":
|
||||
m.SetSort(SortOption{Name: "Price (Low to High)", Key: "price", Order: "asc"})
|
||||
case "P":
|
||||
m.SetSort(SortOption{Name: "Price (High to Low)", Key: "price", Order: "desc"})
|
||||
case "c":
|
||||
m.SetSort(SortOption{Name: "Capacity (Small to Large)", Key: "capacity", Order: "asc"})
|
||||
case "C":
|
||||
m.SetSort(SortOption{Name: "Capacity (Large to Small)", Key: "capacity", Order: "desc"})
|
||||
case "s":
|
||||
m.SetSort(SortOption{Name: "Stock Status (In Stock First)", Key: "stock", Order: "desc"})
|
||||
case "esc":
|
||||
m.view = "categories"
|
||||
m.searchQuery = ""
|
||||
default:
|
||||
if len(msg.String()) == 1 {
|
||||
m.searchQuery += msg.String()
|
||||
m.applyFilter()
|
||||
}
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m *Model) handleExportKeyPress(msg tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
if m.ExportCallback == nil {
|
||||
m.view = "products"
|
||||
return m, nil
|
||||
}
|
||||
|
||||
var format string
|
||||
switch msg.String() {
|
||||
case "j", "J":
|
||||
format = "json"
|
||||
case "c", "C":
|
||||
format = "csv"
|
||||
case "p", "P":
|
||||
format = "print"
|
||||
case "esc":
|
||||
m.view = "products"
|
||||
return m, nil
|
||||
default:
|
||||
return m, nil
|
||||
}
|
||||
|
||||
err := m.ExportCallback(format)
|
||||
if err != nil {
|
||||
m.lastExportError = err.Error()
|
||||
} else {
|
||||
m.lastExportError = ""
|
||||
}
|
||||
|
||||
m.view = "products"
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m *Model) handleSortMenuKeyPress(msg tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
switch msg.String() {
|
||||
case "up", "k":
|
||||
if m.sortMenuIndex > 0 {
|
||||
m.sortMenuIndex--
|
||||
}
|
||||
case "down", "j":
|
||||
if m.sortMenuIndex < len(sortOptions) {
|
||||
m.sortMenuIndex++
|
||||
}
|
||||
case "enter":
|
||||
if m.sortMenuIndex >= 0 && m.sortMenuIndex < len(sortOptions) {
|
||||
m.SetSort(sortOptions[m.sortMenuIndex])
|
||||
}
|
||||
m.view = "products"
|
||||
case "esc":
|
||||
m.view = "products"
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m *Model) GetProducts() []models.StorageProduct {
|
||||
return m.products
|
||||
}
|
||||
|
||||
func (m *Model) GetFilteredProducts() []models.StorageProduct {
|
||||
return m.filteredProducts
|
||||
}
|
||||
|
||||
func (m *Model) GetCategories() []models.Category {
|
||||
return m.categories
|
||||
}
|
||||
|
||||
func (m *Model) GetSelectedCategory() int {
|
||||
return m.selectedCategory
|
||||
}
|
||||
|
||||
func (m *Model) GetSelectedProduct() int {
|
||||
return m.selectedProduct
|
||||
}
|
||||
|
||||
func (m *Model) GetSearchQuery() string {
|
||||
return m.searchQuery
|
||||
}
|
||||
|
||||
func (m *Model) IsScraping() bool {
|
||||
return m.isScraping
|
||||
}
|
||||
|
||||
func (m *Model) GetProgress() scraper.ScrapeProgress {
|
||||
return m.progress
|
||||
}
|
||||
|
||||
func (m *Model) GetError() error {
|
||||
return m.err
|
||||
}
|
||||
|
||||
func (m *Model) GetView() string {
|
||||
return m.view
|
||||
}
|
||||
|
||||
func (m *Model) SetProducts(products []models.StorageProduct) {
|
||||
m.products = products
|
||||
m.filteredProducts = products
|
||||
m.selectedProduct = 0
|
||||
}
|
||||
|
||||
func (m *Model) SetScraping(isScraping bool) {
|
||||
m.isScraping = isScraping
|
||||
}
|
||||
|
||||
func (m *Model) SetProgress(progress scraper.ScrapeProgress) {
|
||||
m.progress = progress
|
||||
}
|
||||
|
||||
func (m *Model) SetError(err error) {
|
||||
m.err = err
|
||||
}
|
||||
|
||||
func (m *Model) SetView(view string) {
|
||||
m.view = view
|
||||
}
|
||||
|
||||
func (m *Model) SetSelectedCategory(index int) {
|
||||
if index >= 0 && index < len(m.categories) {
|
||||
m.selectedCategory = index
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Model) SetSelectedProduct(index int) {
|
||||
if index >= 0 && index < len(m.filteredProducts) {
|
||||
m.selectedProduct = index
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Model) SetSearchQuery(query string) {
|
||||
m.searchQuery = query
|
||||
m.applyFilter()
|
||||
}
|
||||
|
||||
func (m *Model) applyFilter() {
|
||||
if m.searchQuery == "" {
|
||||
m.filteredProducts = m.products
|
||||
} else {
|
||||
query := toLower(m.searchQuery)
|
||||
var filtered []models.StorageProduct
|
||||
for _, p := range m.products {
|
||||
if contains(toLower(p.Name), query) ||
|
||||
contains(toLower(p.Capacity), query) ||
|
||||
contains(toLower(p.Brand), query) ||
|
||||
contains(toLower(p.Interface), query) ||
|
||||
contains(toLower(p.FormFactor), query) ||
|
||||
contains(toLower(p.PartNumber), query) {
|
||||
filtered = append(filtered, p)
|
||||
}
|
||||
}
|
||||
m.filteredProducts = filtered
|
||||
}
|
||||
m.selectedProduct = 0
|
||||
}
|
||||
|
||||
func toLower(s string) string {
|
||||
result := make([]rune, len(s))
|
||||
for i, r := range s {
|
||||
if r >= 'A' && r <= 'Z' {
|
||||
result[i] = r + 32
|
||||
} else {
|
||||
result[i] = r
|
||||
}
|
||||
}
|
||||
return string(result)
|
||||
}
|
||||
|
||||
func contains(s, substr string) bool {
|
||||
if len(substr) == 0 {
|
||||
return true
|
||||
}
|
||||
if len(s) < len(substr) {
|
||||
return false
|
||||
}
|
||||
for i := 0; i <= len(s)-len(substr); i++ {
|
||||
if s[i:i+len(substr)] == substr {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (m *Model) FilterByCategory(categoryIndex int) {
|
||||
if categoryIndex < 0 || categoryIndex >= len(m.categories) {
|
||||
return
|
||||
}
|
||||
|
||||
category := m.categories[categoryIndex]
|
||||
m.selectedCategory = categoryIndex
|
||||
|
||||
if category.Name == "All Categories" {
|
||||
m.filteredProducts = m.products
|
||||
} else {
|
||||
var filtered []models.StorageProduct
|
||||
for _, p := range m.products {
|
||||
if matchesCategory(p, category) {
|
||||
filtered = append(filtered, p)
|
||||
}
|
||||
}
|
||||
m.filteredProducts = filtered
|
||||
}
|
||||
m.selectedProduct = 0
|
||||
}
|
||||
|
||||
func matchesCategory(p models.StorageProduct, category models.Category) bool {
|
||||
catLower := toLower(category.Name)
|
||||
|
||||
if contains(catLower, "3.5") && p.FormFactor == "3.5\"" {
|
||||
return true
|
||||
}
|
||||
if contains(catLower, "2.5") && p.FormFactor == "2.5\"" {
|
||||
return true
|
||||
}
|
||||
if contains(catLower, "nvme") && contains(toLower(p.Interface), "nvme") {
|
||||
return true
|
||||
}
|
||||
if contains(catLower, "sata") && contains(toLower(p.Interface), "sata") {
|
||||
return true
|
||||
}
|
||||
if contains(catLower, "sas") && contains(toLower(p.Interface), "sas") {
|
||||
return true
|
||||
}
|
||||
if contains(catLower, "ssd") && (p.Interface == "SATA" || p.Interface == "SAS" || p.Interface == "NVMe") {
|
||||
return true
|
||||
}
|
||||
if contains(catLower, "hdd") && (p.Interface == "SATA" || p.Interface == "SAS") && p.Capacity != "" {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (m *Model) GetContext() context.Context {
|
||||
return m.ctx
|
||||
}
|
||||
|
||||
func (m *Model) CancelScraping() {
|
||||
if m.cancel != nil {
|
||||
m.cancel()
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Model) renderCategories() string {
|
||||
var result string
|
||||
result += headerStyle.Width(60).Render(" Select Storage Category ")
|
||||
result += "\n\n"
|
||||
|
||||
result += subtleStyle.Render("Use arrow keys to navigate, Enter to select, Esc to go back\n")
|
||||
result += "\n"
|
||||
|
||||
for i, cat := range m.categories {
|
||||
if i == m.selectedCategory {
|
||||
result += selectedStyle.Width(58).Render(" " + cat.Name)
|
||||
} else {
|
||||
result += normalStyle.Width(58).Render(" " + cat.Name)
|
||||
}
|
||||
result += "\n"
|
||||
}
|
||||
|
||||
result += "\n"
|
||||
result += subtleStyle.Render(fmt.Sprintf("Categories: %d | Press Enter to scrape selected category", len(m.categories)))
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func (m *Model) renderProducts() string {
|
||||
var result string
|
||||
|
||||
title := fmt.Sprintf(" Storage Products - %s ", m.categories[m.selectedCategory].Name)
|
||||
result += headerStyle.Width(80).Render(title)
|
||||
result += "\n\n"
|
||||
|
||||
if len(m.products) == 0 {
|
||||
result += subtleStyle.Render("No products scraped yet. Select a category and press Enter to scrape.\n")
|
||||
result += "\n"
|
||||
result += subtleStyle.Render("Press Esc to go back to category selection.\n")
|
||||
return result
|
||||
}
|
||||
|
||||
result += subtleStyle.Render(fmt.Sprintf("Search: %s | ", m.searchQuery))
|
||||
result += fmt.Sprintf("Showing: %d/%d products", len(m.filteredProducts), len(m.products))
|
||||
result += "\n\n"
|
||||
|
||||
result += m.renderProductTable()
|
||||
|
||||
result += "\n\n"
|
||||
result += subtleStyle.Render("Controls: ↑↓ Navigate | / Search | E Export | Esc Back | R Rescrape")
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func (m *Model) renderProductTable() string {
|
||||
if len(m.filteredProducts) == 0 {
|
||||
return " No products found matching your criteria."
|
||||
}
|
||||
|
||||
var result string
|
||||
|
||||
header := " # Name Capacity Interface FormFactor Price \n"
|
||||
result += headerStyle.Render(header)
|
||||
|
||||
start := m.selectedProduct - 5
|
||||
if start < 0 {
|
||||
start = 0
|
||||
}
|
||||
end := start + 15
|
||||
if end > len(m.filteredProducts) {
|
||||
end = len(m.filteredProducts)
|
||||
}
|
||||
|
||||
for i := start; i < end; i++ {
|
||||
p := m.filteredProducts[i]
|
||||
row := fmt.Sprintf(" %-4d %-35s %-10s %-8s %-10s %-12s\n",
|
||||
i+1,
|
||||
truncate(p.Name, 35),
|
||||
p.Capacity,
|
||||
p.Interface,
|
||||
p.FormFactor,
|
||||
truncate(p.Price, 12))
|
||||
|
||||
if i == m.selectedProduct {
|
||||
result += selectedStyle.Width(79).Render(row)
|
||||
} else {
|
||||
result += normalStyle.Width(79).Render(row)
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func (m *Model) renderProgress() string {
|
||||
var result string
|
||||
|
||||
result += headerStyle.Width(60).Render(" Scraping Progress ")
|
||||
result += "\n\n"
|
||||
|
||||
result += fmt.Sprintf(" Category: %s\n\n", m.progress.Category)
|
||||
result += fmt.Sprintf(" Status: %s\n\n", m.progress.Status)
|
||||
|
||||
if m.progress.Total > 0 {
|
||||
percent := float64(m.progress.Current) / float64(m.progress.Total) * 100
|
||||
barWidth := 40
|
||||
filled := int(float64(barWidth) * percent / 100)
|
||||
|
||||
bar := "["
|
||||
for i := 0; i < barWidth; i++ {
|
||||
if i < filled {
|
||||
bar += "="
|
||||
} else if i == filled {
|
||||
bar += ">"
|
||||
} else {
|
||||
bar += " "
|
||||
}
|
||||
}
|
||||
bar += "]"
|
||||
|
||||
result += fmt.Sprintf(" Progress: %s %d/%d (%.1f%%)\n\n", bar, m.progress.Current, m.progress.Total, percent)
|
||||
}
|
||||
|
||||
if m.progress.ProductName != "" {
|
||||
result += fmt.Sprintf(" Current: %s\n", truncate(m.progress.ProductName, 50))
|
||||
}
|
||||
|
||||
result += "\n\n"
|
||||
result += subtleStyle.Render("Press Esc to cancel...")
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func (m *Model) renderError() string {
|
||||
var result string
|
||||
|
||||
result += headerStyle.Width(60).Render(" Error ")
|
||||
result += "\n\n"
|
||||
|
||||
result += errorStyle.Render(fmt.Sprintf(" %v\n", m.err))
|
||||
result += "\n\n"
|
||||
result += subtleStyle.Render("Press any key to continue...")
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func (m *Model) renderExport() string {
|
||||
var result string
|
||||
|
||||
result += headerStyle.Width(60).Render(" Export Data ")
|
||||
result += "\n\n"
|
||||
|
||||
result += fmt.Sprintf(" Products: %d\n", len(m.products))
|
||||
result += fmt.Sprintf(" Filtered: %d\n\n", len(m.filteredProducts))
|
||||
|
||||
result += " Export Options:\n"
|
||||
result += " [J] Export to JSON\n"
|
||||
result += " [C] Export to CSV\n"
|
||||
result += " [P] Print to console\n"
|
||||
result += " [Esc] Go back\n"
|
||||
|
||||
result += "\n\n"
|
||||
result += subtleStyle.Render("Select an option...")
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func truncate(s string, maxLen int) string {
|
||||
if len(s) <= maxLen {
|
||||
return s
|
||||
}
|
||||
return s[:maxLen-3] + "..."
|
||||
}
|
||||
Reference in New Issue
Block a user