[Experimental] Add table driven scenario support (#1202)

* extract version from semver

* parse table under scenarios as data table, #111

* resolve dynamic params using scenario datatable, #111

* refactor: remove redundant ParamResolver type

* refactor: pull up params resolver to parser

* refactored param resolution to not be tied to specexecutor

some go lint fixes as well

* reorganized resolver specs

* scenario data table params are now resolved in the execution, #111

* execute table driven scenarios individually

* updated proto messages for table driven scenario

* map and mule datatable context from execution to result

* add toggle for table_driven_scenario #111

* honour scenario datatable toggles for execution, #111

* fix unit tests for scenario datatable

* fix scenario executor, teardown need to be executed separately

* fix review: use an if instead of switch for single condition check

api/lang/server.go

@@ -223,7 +223,7 @@ func (h *LangHandler) Handle(ctx context.Context, conn jsonrpc2.JSONRPC2, req *j
 		}
 		return val, err
 	case "gauge/executionStatus":
-		val, err := execution.ReadExecutionStatus()
+		val, err := execution.ReadLastExecutionResult()
 		if err != nil {
 			logDebug(req, err.Error())
 		}

build/npm/src/index.js

@@ -27,5 +27,5 @@ var downloadAndExtract = async function(version) {
 }
 
 install.getVersion(packageJsonPath)
-.then((v) => downloadAndExtract(v))
+.then((v) => downloadAndExtract(v.split('-')[0]))
 .catch((e) => console.error(e));

env/env.go

@@ -21,25 +21,36 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strconv"
 
 	"regexp"
 	"strings"
 
 	"github.com/dmotylev/goproperties"
 	"github.com/getgauge/common"
 	"github.com/getgauge/gauge/config"
+	"github.com/getgauge/gauge/logger"
 )
 
 const (
-	SpecsDir            = "gauge_specs_dir"
-	GaugeReportsDir     = "gauge_reports_dir"
-	LogsDirectory       = "logs_directory"
-	OverwriteReports    = "overwrite_reports"
+	// SpecsDir holds the location of spec files
+	SpecsDir = "gauge_specs_dir"
+	// GaugeReportsDir holds the location of reports
+	GaugeReportsDir = "gauge_reports_dir"
+	// LogsDirectory holds the location of log files
+	LogsDirectory = "logs_directory"
+	// OverwriteReports = false will create a new directory for reports
+	// for every run.
+	OverwriteReports = "overwrite_reports"
+	// ScreenshotOnFailure indicates if failure should invoke screenshot
 	ScreenshotOnFailure = "screenshot_on_failure"
-	SaveExecutionResult = "save_execution_result" // determines if last run result should be saved
-	CsvDelimiter        = "csv_delimiter"
-	AllowMultilineStep  = "allow_multiline_step"
-	useTestGA           = "use_test_ga"
+	saveExecutionResult = "save_execution_result"
+	// CsvDelimiter holds delimiter used to parse csv files
+	CsvDelimiter           = "csv_delimiter"
+	allowMultilineStep     = "allow_multiline_step"
+	allowScenarioDatatable = "allow_scenario_datatable"
+	enableMultithreading   = "enable_multithreading"
+	useTestGA              = "use_test_ga"
 )
 
 var envVars map[string]string
@@ -98,9 +109,10 @@ func loadDefaultEnvVars() {
 	addEnvVar(LogsDirectory, "logs")
 	addEnvVar(OverwriteReports, "true")
 	addEnvVar(ScreenshotOnFailure, "true")
-	addEnvVar(SaveExecutionResult, "false")
+	addEnvVar(saveExecutionResult, "false")
 	addEnvVar(CsvDelimiter, ",")
-	addEnvVar(AllowMultilineStep, "false")
+	addEnvVar(allowMultilineStep, "false")
+	addEnvVar(allowScenarioDatatable, "false")
 	addEnvVar(useTestGA, "false")
 }
 
@@ -154,7 +166,7 @@ func substituteEnvVars() error {
 				envKey, property := match[0], match[1]
 				// error if env property is not found
 				if !isPropertySet(property) {
-					return fmt.Errorf("'%s' env variable was not set.", property)
+					return fmt.Errorf("'%s' env variable was not set", property)
 				}
 				// get env var from system
 				propertyValue := os.Getenv(property)
@@ -199,6 +211,38 @@ func CurrentEnv() string {
 	return currentEnv
 }
 
+func convertToBool(property string, defaultValue bool) bool {
+	v := os.Getenv(property)
+	boolValue, err := strconv.ParseBool(strings.TrimSpace(v))
+	if err != nil {
+		logger.Warningf(true, "Incorrect value for %s in property file. Cannot convert %s to boolean.", property, v)
+		logger.Warningf(true, "Using default value %v for property %s.", defaultValue, property)
+		return defaultValue
+	}
+	return boolValue
+}
+
+// AllowScenarioDatatable -feature toggle for datatables in scenario
+var AllowScenarioDatatable = func() bool {
+	return convertToBool(allowScenarioDatatable, false)
+}
+
+// AllowMultiLineStep - feature toggle for newline in step text
+var AllowMultiLineStep = func() bool {
+	return convertToBool(allowMultilineStep, false)
+}
+
+// SaveExecutionResult determines if last run result should be saved
+var SaveExecutionResult = func() bool {
+	return convertToBool(saveExecutionResult, false)
+}
+
+// EnableMultiThreadedExecution determines if threads should be used instead of process
+// for each parallel stream
+var EnableMultiThreadedExecution = func() bool {
+	return convertToBool(enableMultithreading, false)
+}
+
 // UseTestGA checks if test google analytics account needs to be used
 var UseTestGA = func() bool {
 	return strings.ToLower(os.Getenv(useTestGA)) == "true"

env/env_test.go

@@ -216,5 +216,5 @@ func (s *MySuite) TestLoadDefaultEnvWithSubstitutedVariables(c *C) {
 func (s *MySuite) TestLoadDefaultEnvWithInvalidSubstitutedVariable(c *C) {
 	config.ProjectRoot = "_testdata/proj3"
 	e := LoadEnv("default")
-	c.Assert(e, ErrorMatches, ".*env variable was not set.")
+	c.Assert(e, ErrorMatches, ".*env variable was not set")
 }

execution/execute.go

@@ -15,7 +15,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Gauge.  If not, see <http://www.gnu.org/licenses/>.
 
-/*
+/*Package execution handles gauge's execution of spec/scenario/steps
    Execution can be of two types
 	- Simple execution
 	- Paralell execution
@@ -63,7 +63,6 @@ import (
 	"github.com/getgauge/gauge/plugin/install"
 	"github.com/getgauge/gauge/reporter"
 	"github.com/getgauge/gauge/runner"
-	"github.com/getgauge/gauge/util"
 	"github.com/getgauge/gauge/validation"
 )
 
@@ -148,13 +147,13 @@ var ExecuteSpecs = func(specDirs []string) int {
 	wg := &sync.WaitGroup{}
 	reporter.ListenExecutionEvents(wg)
 	rerun.ListenFailedScenarios(wg, specDirs)
-	if util.ConvertToBool(os.Getenv(env.SaveExecutionResult), env.SaveExecutionResult, false) {
+	if env.SaveExecutionResult() {
 		ListenSuiteEndAndSaveResult(wg)
 	}
 	defer wg.Wait()
 	ei := newExecutionInfo(res.SpecCollection, res.Runner, nil, res.ErrMap, InParallel, 0)
 	e := newExecution(ei)
-	return printExecutionStatus(e.run(), res.ParseOk)
+	return printExecutionResult(e.run(), res.ParseOk)
 }
 
 func newExecution(executionInfo *executionInfo) suiteExecutor {
@@ -202,7 +201,7 @@ func statusJSON(executedSpecs, passedSpecs, failedSpecs, skippedSpecs, executedS
 	return s
 }
 
-func writeExecutionStatus(content string) {
+func writeExecutionResult(content string) {
 	executionStatusFile := filepath.Join(config.ProjectRoot, common.DotGauge, executionStatusFile)
 	dotGaugeDir := filepath.Join(config.ProjectRoot, common.DotGauge)
 	if err := os.MkdirAll(dotGaugeDir, common.NewDirectoryPermissions); err != nil {
@@ -214,7 +213,9 @@ func writeExecutionStatus(content string) {
 	}
 }
 
-func ReadExecutionStatus() (interface{}, error) {
+// ReadLastExecutionResult returns the result of previous execution in JSON format
+// This is stored in $GAUGE_PROJECT_ROOT/.gauge/executionStatus.json file after every execution
+func ReadLastExecutionResult() (interface{}, error) {
 	contents, err := common.ReadFileContents(filepath.Join(config.ProjectRoot, common.DotGauge, executionStatusFile))
 	if err != nil {
 		logger.Fatalf(true, "Failed to read execution status information. Reason: %s", err.Error())
@@ -227,7 +228,7 @@ func ReadExecutionStatus() (interface{}, error) {
 	return meta, nil
 }
 
-func printExecutionStatus(suiteResult *result.SuiteResult, isParsingOk bool) int {
+func printExecutionResult(suiteResult *result.SuiteResult, isParsingOk bool) int {
 	nSkippedSpecs := suiteResult.SpecsSkippedCount
 	var nExecutedSpecs int
 	if len(suiteResult.SpecResults) != 0 {
@@ -259,7 +260,7 @@ func printExecutionStatus(suiteResult *result.SuiteResult, isParsingOk bool) int
 	logger.Infof(true, "Specifications:\t%d executed\t%d passed\t%d failed\t%d skipped", nExecutedSpecs, nPassedSpecs, nFailedSpecs, nSkippedSpecs)
 	logger.Infof(true, "Scenarios:\t%d executed\t%d passed\t%d failed\t%d skipped", nExecutedScenarios, nPassedScenarios, nFailedScenarios, nSkippedScenarios)
 	logger.Infof(true, "\nTotal time taken: %s", time.Millisecond*time.Duration(suiteResult.ExecutionTime))
-	writeExecutionStatus(s)
+	writeExecutionResult(s)
 
 	if !isParsingOk {
 		return ParseFailed
@@ -275,10 +276,10 @@ func validateFlags() error {
 		return nil
 	}
 	if NumberOfExecutionStreams < 1 {
-		return fmt.Errorf("Invalid input(%s) to --n flag.", strconv.Itoa(NumberOfExecutionStreams))
+		return fmt.Errorf("invalid input(%s) to --n flag", strconv.Itoa(NumberOfExecutionStreams))
 	}
 	if !isValidStrategy(Strategy) {
-		return fmt.Errorf("Invalid input(%s) to --strategy flag.", Strategy)
+		return fmt.Errorf("invalid input(%s) to --strategy flag", Strategy)
 	}
 	return nil
 }

execution/execute_test.go

@@ -80,12 +80,12 @@ func (s *MySuite) TestValidateFlagsWithInvalidStrategy(c *C) {
 	Strategy = "sdf"
 	NumberOfExecutionStreams = 1
 	err := validateFlags()
-	c.Assert(err.Error(), Equals, "Invalid input(sdf) to --strategy flag.")
+	c.Assert(err.Error(), Equals, "invalid input(sdf) to --strategy flag")
 }
 
 func (s *MySuite) TestValidateFlagsWithInvalidStream(c *C) {
 	InParallel = true
 	NumberOfExecutionStreams = -1
 	err := validateFlags()
-	c.Assert(err.Error(), Equals, "Invalid input(-1) to --n flag.")
+	c.Assert(err.Error(), Equals, "invalid input(-1) to --n flag")
 }

execution/parallelExecution.go

@@ -30,6 +30,7 @@ import (
 	"github.com/getgauge/common"
 	"github.com/getgauge/gauge/config"
 	"github.com/getgauge/gauge/conn"
+	"github.com/getgauge/gauge/env"
 	"github.com/getgauge/gauge/execution/event"
 	"github.com/getgauge/gauge/execution/result"
 	"github.com/getgauge/gauge/filter"
@@ -40,17 +41,16 @@ import (
 	"github.com/getgauge/gauge/plugin"
 	"github.com/getgauge/gauge/reporter"
 	"github.com/getgauge/gauge/runner"
-	"github.com/getgauge/gauge/util"
 )
 
+// Strategy for execution, can be either 'Eager' or 'Lazy'
 var Strategy string
 
 // Eager is a parallelization strategy for execution. In this case tests are distributed before execution, thus making them an equal number based distribution.
 const Eager string = "eager"
 
 // Lazy is a parallelization strategy for execution. In this case tests assignment will be dynamic during execution, i.e. assign the next spec in line to the stream that has completed it’s previous execution and is waiting for more work.
 const Lazy string = "lazy"
-const enableMultithreadingEnv = "enable_multithreading"
 
 type parallelExecution struct {
 	wg                       sync.WaitGroup
@@ -273,8 +273,7 @@ func isValidStrategy(strategy string) bool {
 }
 
 func (e *parallelExecution) isMultithreaded() bool {
-	value := util.ConvertToBool(os.Getenv(enableMultithreadingEnv), enableMultithreadingEnv, false)
-	if !value {
+	if !env.EnableMultiThreadedExecution() {
 		return false
 	}
 	if !e.runner.IsMultithreaded() {

execution/parallelExecution_test.go

@@ -21,8 +21,8 @@ import (
 	"testing"
 
 	"net"
-	"os"
 
+	"github.com/getgauge/gauge/env"
 	"github.com/getgauge/gauge/execution/result"
 	"github.com/getgauge/gauge/gauge"
 	"github.com/getgauge/gauge/gauge_messages"
@@ -115,37 +115,25 @@ func (s *MySuite) TestAggregationOfSuiteResultWithHook(c *C) {
 func (s *MySuite) TestIsMultiThreadedWithEnvSetToFalse(c *C) {
 	e := parallelExecution{errMaps: getValidationErrorMap()}
 
-	os.Setenv(enableMultithreadingEnv, "false")
+	env.EnableMultiThreadedExecution = func() bool { return false }
 
-	multithreaded := e.isMultithreaded()
-
-	os.Setenv(enableMultithreadingEnv, "")
-
-	c.Assert(false, Equals, multithreaded)
+	c.Assert(false, Equals, e.isMultithreaded())
 }
 
 func (s *MySuite) TestIsMultiThreadedWithRunnerWhenSupportsMultithreading(c *C) {
 	e := parallelExecution{errMaps: getValidationErrorMap(), runner: &fakeRunner{isMultiThreaded: true}}
 
-	os.Setenv(enableMultithreadingEnv, "true")
-
-	multithreaded := e.isMultithreaded()
+	env.EnableMultiThreadedExecution = func() bool { return true }
 
-	os.Setenv(enableMultithreadingEnv, "")
-
-	c.Assert(true, Equals, multithreaded)
+	c.Assert(true, Equals, e.isMultithreaded())
 }
 
 func (s *MySuite) TestIsMultiThreadedWithRunnerWhenDoesNotSupportMultithreading(c *C) {
 	e := parallelExecution{errMaps: getValidationErrorMap(), runner: &fakeRunner{isMultiThreaded: false}}
 
-	os.Setenv(enableMultithreadingEnv, "true")
-
-	multithreaded := e.isMultithreaded()
-
-	os.Setenv(enableMultithreadingEnv, "")
+	env.EnableMultiThreadedExecution = func() bool { return true }
 
-	c.Assert(false, Equals, multithreaded)
+	c.Assert(false, Equals, e.isMultithreaded())
 }
 
 type fakeRunner struct {