fixed some smaller issues

jakopako · Mar 6, 2022 · d4b8853 · d4b8853
1 parent f36e695
commit d4b8853
Show file tree

Hide file tree

Showing 6 changed files with 105 additions and 66 deletions.
diff --git a/go.mod b/go.mod
@@ -9,4 +9,8 @@ require (
 	gopkg.in/yaml.v2 v2.4.0
 )
 
-require github.com/andybalholm/cascadia v1.3.1 // indirect
+require (
+	github.com/andybalholm/cascadia v1.3.1 // indirect
+	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
+	golang.org/x/tools v0.1.9 // indirect
+)
diff --git a/go.sum b/go.sum
@@ -4,18 +4,33 @@ github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x0
 github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/goodsign/monday v1.0.0 h1:Yyk/s/WgudMbAJN6UWSU5xAs8jtNewfqtVblAlw0yoc=
 github.com/goodsign/monday v1.0.0/go.mod h1:r4T4breXpoFwspQNM+u2sLxJb2zyTaxVGqUfTBjWOu8=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug=
+golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.1.9 h1:j9KsMiaP1c3B0OTQGth0/k+miLGTgLsAFUCrF2vLcF8=
+golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=

diff --git a/main.go b/main.go
@@ -11,7 +11,7 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
-func NewConfig(configPath string) (*scraper.Config, error) {
+func newConfig(configPath string) (*scraper.Config, error) {
 	config := &scraper.Config{}
 	file, err := os.Open(configPath)
 	if err != nil {
@@ -32,7 +32,7 @@ func main() {
 
 	flag.Parse()
 
-	config, err := NewConfig(*configFile)
+	config, err := newConfig(*configFile)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -44,18 +44,18 @@ func main() {
 			if *singleScraper == s.Name {
 				wg.Add(1)
 				if *storeData {
-					output.WriteEventsToAPI(&wg, s)
+					output.WriteItemsToAPI(&wg, s)
 				} else {
-					output.PrettyPrintEvents(&wg, s)
+					output.PrettyPrintItems(&wg, s)
 				}
 				break
 			}
 		} else {
 			wg.Add(1)
 			if *storeData {
-				go output.WriteEventsToAPI(&wg, s)
+				go output.WriteItemsToAPI(&wg, s)
 			} else {
-				go output.PrettyPrintEvents(&wg, s)
+				go output.PrettyPrintItems(&wg, s)
 			}
 		}
 	}

diff --git a/output/api.go b/output/api.go
@@ -15,16 +15,19 @@ import (
 	"github.com/jakopako/goskyr/scraper"
 )
 
-func WriteEventsToAPI(wg *sync.WaitGroup, c scraper.Scraper) {
+// WriteItemsToAPI writes the scraped events to an API defined through
+// env vars. This method is not really useful because it is tailored to
+// one specific API. Might change in the future.
+func WriteItemsToAPI(wg *sync.WaitGroup, c scraper.Scraper) {
 	// This function is not yet documented in the README because it might soon change and the entire result / output handling
 	// might be refactored / improved.
 	log.Printf("crawling %s\n", c.Name)
 	defer wg.Done()
-	apiUrl := os.Getenv("EVENT_API")
+	apiURL := os.Getenv("EVENT_API")
 	client := &http.Client{
 		Timeout: time.Second * 10,
 	}
-	events, err := c.GetEvents()
+	events, err := c.GetItems()
 
 	if err != nil {
 		log.Printf("%s ERROR: %s", c.Name, err)
@@ -40,8 +43,8 @@ func WriteEventsToAPI(wg *sync.WaitGroup, c scraper.Scraper) {
 	// delete events of this scraper from first date on
 
 	firstDate := events[0]["date"].(time.Time).UTC().Format("2006-01-02 15:04")
-	deleteUrl := fmt.Sprintf("%s?location=%s&datetime=%s", apiUrl, url.QueryEscape(c.Name), url.QueryEscape(firstDate))
-	req, _ := http.NewRequest("DELETE", deleteUrl, nil)
+	deleteURL := fmt.Sprintf("%s?location=%s&datetime=%s", apiURL, url.QueryEscape(c.Name), url.QueryEscape(firstDate))
+	req, _ := http.NewRequest("DELETE", deleteURL, nil)
 	req.SetBasicAuth(os.Getenv("API_USER"), os.Getenv("API_PASSWORD"))
 	resp, err := client.Do(req)
 	if err != nil {
@@ -53,7 +56,7 @@ func WriteEventsToAPI(wg *sync.WaitGroup, c scraper.Scraper) {
 		if err != nil {
 			log.Fatal(err)
 		}
-		log.Fatalf("Something went wrong while deleting events. Status Code: %d\nUrl: %s Response: %s", resp.StatusCode, deleteUrl, body)
+		log.Fatalf("Something went wrong while deleting events. Status Code: %d\nUrl: %s Response: %s", resp.StatusCode, deleteURL, body)
 	}
 
 	// add new events
@@ -62,7 +65,7 @@ func WriteEventsToAPI(wg *sync.WaitGroup, c scraper.Scraper) {
 		if err != nil {
 			log.Fatal(err)
 		}
-		req, _ := http.NewRequest("POST", apiUrl, bytes.NewBuffer(concertJSON))
+		req, _ := http.NewRequest("POST", apiURL, bytes.NewBuffer(concertJSON))
 		req.Header = map[string][]string{
 			"Content-Type": {"application/json"},
 		}

diff --git a/output/stdout.go b/output/stdout.go
@@ -10,9 +10,10 @@ import (
 	"github.com/jakopako/goskyr/scraper"
 )
 
-func PrettyPrintEvents(wg *sync.WaitGroup, c scraper.Scraper) {
+// PrettyPrintItems pretty prints the scraped items
+func PrettyPrintItems(wg *sync.WaitGroup, c scraper.Scraper) {
 	defer wg.Done()
-	events, err := c.GetEvents()
+	events, err := c.GetItems()
 	if err != nil {
 		log.Printf("%s ERROR: %s", c.Name, err)
 		return