Iteration Helpers

README.md

@@ -375,8 +375,8 @@ make test
 ```bash
 gofmt -w `find . -name '*.go' | grep -v vendor`
 go vet -all $(go list ./... | grep -v datatypes)
-go get -d -v -t ./...
-go test $(go list ./... | grep -v '/vendor/') -timeout=30s -parallel=4 -coverprofile coverage.out
+go mod vendor
+go test $(go list ./... | grep -v '/vendor/') -timeout=30s -coverprofile coverage.out
 ```
 
 ### Updating dependencies

datatypes/billing.go

@@ -1368,6 +1368,14 @@ type Billing_Item_Gateway_Appliance_Cluster struct {
 	Billing_Item
 }
 
+// The SoftLayer_Billing_Item_Gateway_License data type contains general information relating to a single SoftLayer billing item for a bare_metal_gateway_license
+type Billing_Item_Gateway_License struct {
+	Billing_Item
+
+	// no documentation yet
+	Resource *Network_Gateway `json:"resource,omitempty" xmlrpc:"resource,omitempty"`
+}
+
 // The SoftLayer_Billing_Item_Hardware data type contains general information relating to a single SoftLayer billing item for hardware.
 type Billing_Item_Hardware struct {
 	Billing_Item

examples/cmd/virtual_iter.go

@@ -0,0 +1,65 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"github.com/softlayer/softlayer-go/filter"
+	"github.com/softlayer/softlayer-go/helpers/virtual"
+	"github.com/softlayer/softlayer-go/session"
+	"github.com/softlayer/softlayer-go/sl"
+)
+
+func init() {
+	rootCmd.AddCommand(listVirtCmd)
+}
+
+var listVirtCmd = &cobra.Command{
+	Use:   "virt-list",
+	Short: "Lists all VSI on the account",
+	Long:  `Lists all VSI on the account using an iterative aproach.`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return RunListVirtCmd(cmd, args)
+	},
+}
+
+func RunListVirtCmd(cmd *cobra.Command, args []string) error {
+
+	objectMask := "mask[id,hostname,domain,primaryIpAddress,primaryBackendIpAddress]"
+	// When using a result Limit to break up your API request, its important to include an orderBy objectFilter
+	// to enforce an order on the query, as the database might not always return results in the same order between
+	// queries otherwise
+	filters := filter.New()
+	filters = append(filters, filter.Path("virtualGuests.id").OrderBy("ASC"))
+	objectFilter := filters.Build()
+	// Sets up the session with authentication headers.
+	sess := session.New()
+	// uncomment to output API calls as they are made.
+	sess.Debug = true
+
+	// Sets the mask, filter, result limit, and then makes the API call SoftLayer_Account::getHardware()
+	limit := 5
+	options := sl.Options{
+		Mask:   objectMask,
+		Filter: objectFilter,
+		Limit:  &limit,
+	}
+
+	servers, err := virtual.GetVirtualGuestsIter(sess, &options)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("Id, Hostname, Domain, IP Address\n")
+
+	for _, server := range servers {
+		ipAddress := "-"
+		// Servers with a private only connection will not have a primary IP
+		if server.PrimaryIpAddress != nil {
+			ipAddress = *server.PrimaryIpAddress
+		}
+		fmt.Printf("%v, %v, %v, %v\n", *server.Id, *server.Hostname, *server.Domain, ipAddress)
+	}
+
+	return nil
+}

examples/go.mod

@@ -2,6 +2,8 @@ module github.com/softlayer/softlayer-go/examples
 
 go 1.21
 
+replace github.com/softlayer/softlayer-go => ../
+
 require (
 	github.com/jarcoal/httpmock v1.0.5
 	github.com/jedib0t/go-pretty/v6 v6.5.4
@@ -26,6 +28,7 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/net v0.20.0 // indirect
+	golang.org/x/sync v0.6.0 // indirect
 	golang.org/x/sys v0.16.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

examples/go.sum

@@ -54,6 +54,8 @@ golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
 golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
 golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
+golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=

generator/templates.go

@@ -120,7 +120,8 @@ import (
 		{{end}}err = r.Session.DoRequest("{{$rawBase}}", "{{.Name}}", {{if len .Parameters | lt 0}}params{{else}}nil{{end}}, &r.Options, &resp)
 	return
 	}
-	{{end}}
+
+	{{end }}
 
 {{end}}
 `, license, codegenWarning)
@@ -187,3 +188,35 @@ var _ = Describe("{{(index . 0 ).ServiceGroup}} Tests", func() {
 {{ end }}
 })
 `
+
+// Not Used, but could be added to the Service template if you want ALL methods that accept a resultLimit to page through results
+var IterTemplate = `{{if .TypeArray}}
+	func (r {{$base}}) {{.Name|titleCase}}Iter({{range .Parameters}}{{phraseMethodArg $methodName .Name .TypeArray .Type}}{{end}}) ({{if .Type|ne "void"}}resp {{if .TypeArray}}[]{{end}}{{convertType .Type "services"}}, {{end}}err error) {
+		{{if len .Parameters | lt 0}}params := []interface{}{
+			{{range .Parameters}}{{.Name|removeReserved}},
+			{{end}}
+		}
+		{{end}}limit := r.Options.ValidateLimit()
+		err = r.Session.DoRequest("{{$rawBase}}", "{{.Name}}", {{if len .Parameters | lt 0}}params{{else}}nil{{end}}, &r.Options, &resp)
+		if err != nil {
+			return
+		}
+		apicalls := r.Options.GetRemainingAPICalls()
+		var wg sync.WaitGroup
+		for x := 1; x <= apicalls; x++ {
+			wg.Add(1)
+			go func(i int) {
+				defer wg.Done()
+				offset := i * limit
+				this_resp := []{{convertType .Type "services"}}{}
+				options := r.Options
+				options.Offset = &offset
+				err = r.Session.DoRequest("{{$rawBase}}", "{{.Name}}", {{if len .Parameters | lt 0}}params{{else}}nil{{end}}, &options, &this_resp)
+				resp = append(resp, this_resp...)
+			}(x)
+		}
+		wg.Wait()
+	return
+	}
+	{{end }}
+`

go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/net v0.20.0 // indirect
+	golang.org/x/sync v0.6.0 // indirect
 	golang.org/x/sys v0.16.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -51,6 +51,7 @@ golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
 golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
 golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
 golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=

helpers/hardware/hardware.go

@@ -18,12 +18,13 @@ package hardware
 
 import (
 	"fmt"
-
 	"github.com/softlayer/softlayer-go/datatypes"
 	"github.com/softlayer/softlayer-go/helpers/location"
 	"github.com/softlayer/softlayer-go/services"
 	"github.com/softlayer/softlayer-go/session"
+	"github.com/softlayer/softlayer-go/sl"
 	"regexp"
+	"sync"
 )
 
 // GeRouterByName returns a Hardware that matches the provided hostname,
@@ -59,3 +60,36 @@ func GetRouterByName(sess *session.Session, hostname string, args ...interface{}
 
 	return datatypes.Hardware{}, fmt.Errorf("No routers found with hostname of %s", hostname)
 }
+
+// Use go-routines to iterate through all hardware results.
+// options should be any Mask or Filter you need, and a Limit if the default is too large.
+// Any error in the subsequent API calls will be logged, but largely ignored
+func GetHardwareIter(session session.SLSession, options *sl.Options) (resp []datatypes.Hardware, err error) {
+
+	options.SetOffset(0)
+	limit := options.ValidateLimit()
+
+	// Can't call service.GetVirtualGuests because it passes a copy of options, not the address to options sadly.
+	err = session.DoRequest("SoftLayer_Account", "getHardware", nil, options, &resp)
+	if err != nil {
+		return
+	}
+	apicalls := options.GetRemainingAPICalls()
+	var wg sync.WaitGroup
+	for x := 1; x <= apicalls; x++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			offset := i * limit
+			this_resp := []datatypes.Hardware{}
+			options.Offset = &offset
+			err = session.DoRequest("SoftLayer_Account", "getHardware", nil, options, &this_resp)
+			if err != nil {
+				fmt.Printf("[ERROR] %v\n", err)
+			}
+			resp = append(resp, this_resp...)
+		}(x)
+	}
+	wg.Wait()
+	return resp, err
+}

helpers/hardware/hardware_test.go

@@ -0,0 +1,38 @@
+package hardware_test
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/softlayer/softlayer-go/helpers/hardware"
+	"github.com/softlayer/softlayer-go/session/sessionfakes"
+	"github.com/softlayer/softlayer-go/sl"
+	"testing"
+)
+
+func TestServices(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Helper Hardware Tests")
+}
+
+var _ = Describe("Helper Hardware Tests", func() {
+	var slsession *sessionfakes.FakeSLSession
+	var options *sl.Options
+	BeforeEach(func() {
+		limit := 10
+		slsession = &sessionfakes.FakeSLSession{}
+		options = &sl.Options{
+			Mask:   "mask[id,hostname]",
+			Filter: "",
+			Limit:  &limit,
+		}
+	})
+
+	Context("GetHardwareIter Tests", func() {
+
+		It("API call made properly", func() {
+			_, err := hardware.GetHardwareIter(slsession, options)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(slsession.DoRequestCallCount()).To(Equal(1))
+		})
+	})
+})

helpers/virtual/virtual.go

@@ -17,13 +17,14 @@
 package virtual
 
 import (
-	"time"
-
+	"fmt"
 	"github.com/softlayer/softlayer-go/datatypes"
 	"github.com/softlayer/softlayer-go/helpers/product"
 	"github.com/softlayer/softlayer-go/services"
 	"github.com/softlayer/softlayer-go/session"
 	"github.com/softlayer/softlayer-go/sl"
+	"sync"
+	"time"
 )
 
 // Upgrade a virtual guest to a specified set of features (e.g. cpu, ram).
@@ -159,3 +160,36 @@ func UpgradeVirtualGuestWithPreset(
 	orderService := services.GetProductOrderService(sess)
 	return orderService.PlaceOrder(&order, sl.Bool(false))
 }
+
+// Use go-routines to iterate through all virtual guest results.
+// optoins should be any Mask or Filter you need, and a Limit if the default is too large.
+// Any error in the subsequent API calls will be logged, but largely ignored
+func GetVirtualGuestsIter(session session.SLSession, options *sl.Options) (resp []datatypes.Virtual_Guest, err error) {
+
+	options.SetOffset(0)
+	limit := options.ValidateLimit()
+
+	// Can't call service.GetVirtualGuests because it passes a copy of options, not the address to options sadly.
+	err = session.DoRequest("SoftLayer_Account", "getVirtualGuests", nil, options, &resp)
+	if err != nil {
+		return
+	}
+	apicalls := options.GetRemainingAPICalls()
+	var wg sync.WaitGroup
+	for x := 1; x <= apicalls; x++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			offset := i * limit
+			this_resp := []datatypes.Virtual_Guest{}
+			options.Offset = &offset
+			err = session.DoRequest("SoftLayer_Account", "getVirtualGuests", nil, options, &this_resp)
+			if err != nil {
+				fmt.Printf("[ERROR] %v\n", err)
+			}
+			resp = append(resp, this_resp...)
+		}(x)
+	}
+	wg.Wait()
+	return resp, err
+}

helpers/virtual/virtual_test.go

@@ -0,0 +1,40 @@
+package virtual_test
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	// "github.com/softlayer/softlayer-go/datatypes"
+
+	"github.com/softlayer/softlayer-go/helpers/virtual"
+	"github.com/softlayer/softlayer-go/session/sessionfakes"
+	"github.com/softlayer/softlayer-go/sl"
+	"testing"
+)
+
+func TestServices(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Helper Virtual Tests")
+}
+
+var _ = Describe("Helper Virtual Tests", func() {
+	var slsession *sessionfakes.FakeSLSession
+	var options *sl.Options
+	BeforeEach(func() {
+		limit := 10
+		slsession = &sessionfakes.FakeSLSession{}
+		options = &sl.Options{
+			Mask:   "mask[id,hostname]",
+			Filter: "",
+			Limit:  &limit,
+		}
+	})
+
+	Context("GetVirtualGuestsIter Tests", func() {
+
+		It("API call made properly", func() {
+			_, err := virtual.GetVirtualGuestsIter(slsession, options)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(slsession.DoRequestCallCount()).To(Equal(1))
+		})
+	})
+})

session/rest.go

@@ -283,9 +283,19 @@ func makeHTTPRequest(
 	defer resp.Body.Close()
 
 	responseBody, err := ioutil.ReadAll(resp.Body)
+
 	if err != nil {
 		return nil, resp.StatusCode, err
 	}
+	if resp.Header["Softlayer-Total-Items"] != nil && len(resp.Header["Softlayer-Total-Items"]) == 1 {
+		var str_err error
+		var total_items int
+		total_items, str_err = strconv.Atoi(resp.Header["Softlayer-Total-Items"][0])
+		if str_err != nil {
+			log.Println("[Error] Unable to convert Softlayer-Total-Items to int: ", str_err)
+		}
+		options.SetTotalItems(total_items)
+	}
 
 	if session.Debug {
 		log.Println("[DEBUG] Status Code: ", resp.StatusCode)

session/session.go

@@ -262,7 +262,11 @@ func (r *Session) DoRequest(service string, method string, args []interface{}, o
 		r.TransportHandler = getDefaultTransport(r.Endpoint)
 	}
 
-	return r.TransportHandler.DoRequest(r, service, method, args, options, pResult)
+	err := r.TransportHandler.DoRequest(r, service, method, args, options, pResult)
+	if err != nil {
+		return err
+	}
+	return err
 }
 
 // SetTimeout creates a copy of the session and sets the passed timeout into it