@@ 29,8 29,8 @@ type Result struct {
type Client struct {
Capabilities selenium.Capabilities
URL string
+ startTime time.Time
driver selenium.WebDriver
- count uint
}
func (c *Client) reset() {
@@ 38,7 38,6 @@ func (c *Client) reset() {
c.driver.Quit()
c.driver = nil
}
- c.count = 0
}
func (c *Client) Connect() selenium.WebDriver {
@@ 49,11 48,11 @@ func (c *Client) Connect() selenium.WebD
c.driver, err = selenium.NewRemote(c.Capabilities, c.URL)
if err != nil {
- logger.Println("Error connection to WD", c.URL, err)
+ logger.Println("Error connecting to WD", c.URL, err)
time.Sleep(30 * time.Second)
} else {
- logger.Println("connected")
- c.count = 0
+ logger.Println("Connected to", c.URL)
+ c.startTime = time.Now()
break
}
}
@@ 66,7 65,8 @@ func (c *Client) Connect() selenium.WebD
// Download url re-using the previous Selenium session if count < 100
//
func (c *Client) Get(url string) (duration time.Duration, err error) {
- if c.driver != nil && c.count > 100 {
+ // Re-connect every 10 minutes
+ if c.driver != nil && time.Since(c.startTime) > 10*time.Minute {
c.reset()
}
@@ 134,8 134,8 @@ func ScanLines(input io.Reader) <-chan s
//
// All the element sent to the returned channel will be written to `output`
//
-func CSVWriter(output io.Writer) chan<- Result {
- var c = make(chan Result)
+func CSVWriter(output io.Writer, workers int) chan<- Result {
+ var c = make(chan Result, workers)
go func() {
w := csv.NewWriter(output)
@@ 156,31 156,31 @@ func CSVWriter(output io.Writer) chan<-
func main() {
var args struct {
- Workers uint `long:"workers" default:"4" description:"Number of worker routines"`
- Arg struct {
- URL string `description:"URL to the Selenium server"`
+ Workers uint `short:"w" long:"workers" default:"4" description:"Number of workers"`
+ Arg struct {
+ URL string `description:"URL to the Selenium server"`
} `positional-args:"yes" required:"yes"`
}
parser := flags.NewParser(&args, flags.Default)
extra, err := parser.ParseArgs(os.Args[1:])
- if err != nil {
- // FIXME go-flags outputs the error in stderr in some cases, check it
- // does it for all errors
- os.Exit(1)
- }
- if len(extra) != 0 {
- logger.Fatalln("Extra arguments:", extra)
- }
+ if err != nil {
+ // FIXME go-flags outputs the error in stderr in some cases, check it
+ // does it for all errors
+ os.Exit(1)
+ }
+ if len(extra) != 0 {
+ logger.Fatalln("Extra arguments:", extra)
+ }
caps := selenium.Capabilities{
- "browserName": "chrome",
- "platform": "Linux",
- "version": "48.0",
+ "idleTimeout": 1000,
+ "browserName": "firefox",
+ "platform": "linux",
}
+ var workers = int(args.Workers)
var input = ScanLines(os.Stdin)
- var output = CSVWriter(os.Stdout)
- var workers = int(args.Workers)
+ var output = CSVWriter(os.Stdout, workers)
var url = args.Arg.URL
var wg sync.WaitGroup